legacy: Arduino preprocess subroutine refactorization (part 5)

arduino/builder/cpp.go → arduino/builder/cpp/cpp.go

@@ -13,29 +13,63 @@
 // Arduino software without disclosing the source code of your own applications.
 // To purchase a commercial license, send an email to [email protected].
 
-package builder
+package cpp
 
 import (
+	"strconv"
 	"strings"
 	"unicode/utf8"
+
+	"github.com/arduino/go-paths-helper"
 )
 
-// QuoteCppString returns the given string as a quoted string for use with the C
+// QuoteString returns the given string as a quoted string for use with the C
 // preprocessor. This adds double quotes around it and escapes any
 // double quotes and backslashes in the string.
-func QuoteCppString(str string) string {
+func QuoteString(str string) string {
 	str = strings.Replace(str, "\\", "\\\\", -1)
 	str = strings.Replace(str, "\"", "\\\"", -1)
 	return "\"" + str + "\""
 }
 
-// ParseCppString parse a C-preprocessor string as emitted by the preprocessor. This
+// ParseLineMarker parses the given line as a gcc line marker and returns the contained
+// filename.
+func ParseLineMarker(line string) *paths.Path {
+	// A line marker contains the line number and filename and looks like:
+	// # 123 /path/to/file.cpp
+	// It can be followed by zero or more flag number that indicate the
+	// preprocessor state and can be ignored.
+	// For exact details on this format, see:
+	// https://github.com/gcc-mirror/gcc/blob/edd716b6b1caa1a5cb320a8cd7f626f30198e098/gcc/c-family/c-ppoutput.c#L413-L415
+
+	split := strings.SplitN(line, " ", 3)
+	if len(split) < 3 || len(split[0]) == 0 || split[0][0] != '#' {
+		return nil
+	}
+
+	_, err := strconv.Atoi(split[1])
+	if err != nil {
+		return nil
+	}
+
+	// If we get here, we found a # followed by a line number, so
+	// assume this is a line marker and see if the rest of the line
+	// starts with a string containing the filename
+	str, rest, ok := ParseString(split[2])
+
+	if ok && (rest == "" || rest[0] == ' ') {
+		return paths.New(str)
+	}
+	return nil
+}
+
+// ParseString parse a string as emitted by the preprocessor. This
 // is a string contained in double quotes, with any backslashes or
 // quotes escaped with a backslash. If a valid string was present at the
 // start of the given line, returns the unquoted string contents, the
 // remainder of the line (everything after the closing "), and true.
 // Otherwise, returns the empty string, the entire line and false.
-func ParseCppString(line string) (string, string, bool) {
+func ParseString(line string) (string, string, bool) {
 	// For details about how these strings are output by gcc, see:
 	// https://github.com/gcc-mirror/gcc/blob/a588355ab948cf551bc9d2b89f18e5ae5140f52c/libcpp/macro.c#L491-L511
 	// Note that the documentation suggests all non-printable

arduino/builder/cpp/cpp_test.go

@@ -0,0 +1,74 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package cpp_test
+
+import (
+	"testing"
+
+	"github.com/arduino/arduino-cli/arduino/builder/cpp"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseString(t *testing.T) {
+	_, _, ok := cpp.ParseString(`foo`)
+	require.Equal(t, false, ok)
+
+	_, _, ok = cpp.ParseString(`"foo`)
+	require.Equal(t, false, ok)
+
+	str, rest, ok := cpp.ParseString(`"foo"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `foo`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = cpp.ParseString(`"foo\\bar"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `foo\bar`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = cpp.ParseString(`"foo \"is\" quoted and \\\\bar\"\" escaped\\" and "then" some`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `foo "is" quoted and \\bar"" escaped\`, str)
+	require.Equal(t, ` and "then" some`, rest)
+
+	str, rest, ok = cpp.ParseString(`" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = cpp.ParseString(`"/home/ççç/"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `/home/ççç/`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = cpp.ParseString(`"/home/ççç/ /$sdsdd\\"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `/home/ççç/ /$sdsdd\`, str)
+	require.Equal(t, ``, rest)
+}
+
+func TestQuoteString(t *testing.T) {
+	cases := map[string]string{
+		`foo`:                                  `"foo"`,
+		`foo\bar`:                              `"foo\\bar"`,
+		`foo "is" quoted and \\bar"" escaped\`: `"foo \"is\" quoted and \\\\bar\"\" escaped\\"`,
+		// ASCII 0x20 - 0x7e, excluding `
+		` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`: `" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`,
+	}
+	for input, expected := range cases {
+		require.Equal(t, expected, cpp.QuoteString(input))
+	}
+}

arduino/builder/preprocessor/ctags.go

@@ -16,10 +16,17 @@
 package preprocessor
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"fmt"
+	"io"
+	"strconv"
 	"strings"
 
+	"github.com/arduino/arduino-cli/arduino/builder/cpp"
+	"github.com/arduino/arduino-cli/arduino/builder/preprocessor/internal/ctags"
+	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/executils"
 	"github.com/arduino/arduino-cli/i18n"
 	"github.com/arduino/go-paths-helper"
@@ -29,6 +36,145 @@ import (
 
 var tr = i18n.Tr
 
+// DebugPreprocessor when set to true the CTags preprocessor will output debugging info to stdout
+// this is useful for unit-testing to provide more infos
+var DebugPreprocessor bool
+
+// PreprocessSketchWithCtags performs preprocessing of the arduino sketch using CTags.
+func PreprocessSketchWithCtags(sketch *sketch.Sketch, buildPath *paths.Path, includes paths.PathList, lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase bool) ([]byte, []byte, error) {
+	// Create a temporary working directory
+	tmpDir, err := paths.MkTempDir("", "")
+	if err != nil {
+		return nil, nil, err
+	}
+	defer tmpDir.RemoveAll()
+	ctagsTarget := tmpDir.Join("sketch_merged.cpp")
+
+	normalOutput := &bytes.Buffer{}
+	verboseOutput := &bytes.Buffer{}
+
+	// Run GCC preprocessor
+	sourceFile := buildPath.Join("sketch", sketch.MainFile.Base()+".cpp")
+	gccStdout, gccStderr, err := GCC(sourceFile, ctagsTarget, includes, buildProperties)
+	verboseOutput.Write(gccStdout)
+	verboseOutput.Write(gccStderr)
+	normalOutput.Write(gccStderr)
+	if err != nil {
+		if !onlyUpdateCompilationDatabase {
+			return normalOutput.Bytes(), verboseOutput.Bytes(), errors.WithStack(err)
+		}
+
+		// Do not bail out if we are generating the compile commands database
+		normalOutput.WriteString(fmt.Sprintf("%s: %s",
+			tr("An error occurred adding prototypes"),
+			tr("the compilation database may be incomplete or inaccurate")))
+		if err := sourceFile.CopyTo(ctagsTarget); err != nil {
+			return normalOutput.Bytes(), verboseOutput.Bytes(), errors.WithStack(err)
+		}
+	}
+
+	if src, err := ctagsTarget.ReadFile(); err == nil {
+		filteredSource := filterSketchSource(sketch, bytes.NewReader(src), false)
+		if err := ctagsTarget.WriteFile([]byte(filteredSource)); err != nil {
+			return normalOutput.Bytes(), verboseOutput.Bytes(), err
+		}
+	} else {
+		return normalOutput.Bytes(), verboseOutput.Bytes(), err
+	}
+
+	// Run CTags on gcc-preprocessed source
+	ctagsOutput, ctagsStdErr, err := RunCTags(ctagsTarget, buildProperties)
+	verboseOutput.Write(ctagsStdErr)
+	if err != nil {
+		return normalOutput.Bytes(), verboseOutput.Bytes(), err
+	}
+
+	// Parse CTags output
+	parser := &ctags.Parser{}
+	prototypes, firstFunctionLine := parser.Parse(ctagsOutput, sketch.MainFile)
+	if firstFunctionLine == -1 {
+		firstFunctionLine = 0
+	}
+
+	// Add prototypes to the original sketch source
+	var source string
+	if sourceData, err := sourceFile.ReadFile(); err == nil {
+		source = string(sourceData)
+	} else {
+		return normalOutput.Bytes(), verboseOutput.Bytes(), err
+	}
+	source = strings.Replace(source, "\r\n", "\n", -1)
+	source = strings.Replace(source, "\r", "\n", -1)
+	sourceRows := strings.Split(source, "\n")
+	if isFirstFunctionOutsideOfSource(firstFunctionLine, sourceRows) {
+		return normalOutput.Bytes(), verboseOutput.Bytes(), nil
+	}
+
+	insertionLine := firstFunctionLine + lineOffset - 1
+	firstFunctionChar := len(strings.Join(sourceRows[:insertionLine], "\n")) + 1
+	prototypeSection := composePrototypeSection(firstFunctionLine, prototypes)
+	preprocessedSource := source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:]
+
+	if DebugPreprocessor {
+		fmt.Println("#PREPROCESSED SOURCE")
+		prototypesRows := strings.Split(prototypeSection, "\n")
+		prototypesRows = prototypesRows[:len(prototypesRows)-1]
+		for i := 0; i < len(sourceRows)+len(prototypesRows); i++ {
+			if i < insertionLine {
+				fmt.Printf("   |%s\n", sourceRows[i])
+			} else if i < insertionLine+len(prototypesRows) {
+				fmt.Printf("PRO|%s\n", prototypesRows[i-insertionLine])
+			} else {
+				fmt.Printf("   |%s\n", sourceRows[i-len(prototypesRows)])
+			}
+		}
+		fmt.Println("#END OF PREPROCESSED SOURCE")
+	}
+
+	// Write back arduino-preprocess output to the sourceFile
+	err = sourceFile.WriteFile([]byte(preprocessedSource))
+	return normalOutput.Bytes(), verboseOutput.Bytes(), err
+}
+
+func composePrototypeSection(line int, prototypes []*ctags.Prototype) string {
+	if len(prototypes) == 0 {
+		return ""
+	}
+
+	str := joinPrototypes(prototypes)
+	str += "\n#line "
+	str += strconv.Itoa(line)
+	str += " " + cpp.QuoteString(prototypes[0].File)
+	str += "\n"
+
+	return str
+}
+
+func joinPrototypes(prototypes []*ctags.Prototype) string {
+	prototypesSlice := []string{}
+	for _, proto := range prototypes {
+		if signatureContainsaDefaultArg(proto) {
+			continue
+		}
+		prototypesSlice = append(prototypesSlice, "#line "+strconv.Itoa(proto.Line)+" "+cpp.QuoteString(proto.File))
+		prototypeParts := []string{}
+		if proto.Modifiers != "" {
+			prototypeParts = append(prototypeParts, proto.Modifiers)
+		}
+		prototypeParts = append(prototypeParts, proto.Prototype)
+		prototypesSlice = append(prototypesSlice, strings.Join(prototypeParts, " "))
+	}
+	return strings.Join(prototypesSlice, "\n")
+}
+
+func signatureContainsaDefaultArg(proto *ctags.Prototype) bool {
+	return strings.Contains(proto.Prototype, "=")
+}
+
+func isFirstFunctionOutsideOfSource(firstFunctionLine int, sourceRows []string) bool {
+	return firstFunctionLine > len(sourceRows)-1
+}
+
 // RunCTags performs a run of ctags on the given source file. Returns the ctags output and the stderr contents.
 func RunCTags(sourceFile *paths.Path, buildProperties *properties.Map) ([]byte, []byte, error) {
 	ctagsBuildProperties := properties.NewMap()
@@ -60,3 +206,29 @@ func RunCTags(sourceFile *paths.Path, buildProperties *properties.Map) ([]byte,
 	stderr = append([]byte(args), stderr...)
 	return stdout, stderr, err
 }
+
+func filterSketchSource(sketch *sketch.Sketch, source io.Reader, removeLineMarkers bool) string {
+	fileNames := paths.NewPathList()
+	fileNames.Add(sketch.MainFile)
+	fileNames.AddAll(sketch.OtherSketchFiles)
+
+	inSketch := false
+	filtered := ""
+
+	scanner := bufio.NewScanner(source)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if filename := cpp.ParseLineMarker(line); filename != nil {
+			inSketch = fileNames.Contains(filename)
+			if inSketch && removeLineMarkers {
+				continue
+			}
+		}
+
+		if inSketch {
+			filtered += line + "\n"
+		}
+	}
+
+	return filtered
+}