diff --git a/arduino/builder/preprocessor/gcc.go b/arduino/builder/preprocessor/gcc.go new file mode 100644 index 00000000000..0345d475900 --- /dev/null +++ b/arduino/builder/preprocessor/gcc.go @@ -0,0 +1,81 @@ +// This file is part of arduino-cli. +// +// Copyright 2023 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package preprocessor + +import ( + "context" + "fmt" + "strings" + + "github.com/arduino/arduino-cli/executils" + f "github.com/arduino/arduino-cli/internal/algorithms" + "github.com/arduino/arduino-cli/legacy/builder/utils" + "github.com/arduino/go-paths-helper" + "github.com/arduino/go-properties-orderedmap" + "github.com/pkg/errors" +) + +// GCC performs a run of the gcc preprocess (macro/includes expansion). The function outputs the result +// to targetFilePath. Returns the stdout/stderr of gcc if any. +func GCC(sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList, buildProperties *properties.Map) ([]byte, []byte, error) { + gccBuildProperties := properties.NewMap() + gccBuildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") + gccBuildProperties.Merge(buildProperties) + gccBuildProperties.Set("build.library_discovery_phase", "1") + gccBuildProperties.SetPath("source_file", sourceFilePath) + gccBuildProperties.SetPath("preprocessed_file_path", targetFilePath) + + includesStrings := f.Map(includes.AsStrings(), utils.WrapWithHyphenI) + gccBuildProperties.Set("includes", strings.Join(includesStrings, " ")) + + const gccPreprocRecipeProperty = "recipe.preproc.macros" + if gccBuildProperties.Get(gccPreprocRecipeProperty) == "" { + // autogenerate preprocess macros recipe from compile recipe + preprocPattern := gccBuildProperties.Get("recipe.cpp.o.pattern") + // add {preproc.macros.flags} to {compiler.cpp.flags} + preprocPattern = strings.Replace(preprocPattern, "{compiler.cpp.flags}", "{compiler.cpp.flags} {preproc.macros.flags}", 1) + // replace "{object_file}" with "{preprocessed_file_path}" + preprocPattern = strings.Replace(preprocPattern, "{object_file}", "{preprocessed_file_path}", 1) + + gccBuildProperties.Set(gccPreprocRecipeProperty, preprocPattern) + } + + pattern := gccBuildProperties.Get(gccPreprocRecipeProperty) + if pattern == "" { + return nil, nil, errors.Errorf(tr("%s pattern is missing"), gccPreprocRecipeProperty) + } + + commandLine := gccBuildProperties.ExpandPropsInString(pattern) + args, err := properties.SplitQuotedString(commandLine, `"'`, false) + if err != nil { + return nil, nil, err + } + + // Remove -MMD argument if present. Leaving it will make gcc try + // to create a /dev/null.d dependency file, which won't work. + args = f.Filter(args, f.NotEquals("-MMD")) + + proc, err := executils.NewProcess(nil, args...) + if err != nil { + return nil, nil, err + } + stdout, stderr, err := proc.RunAndCaptureOutput(context.Background()) + + // Append gcc arguments to stdout + stdout = append([]byte(fmt.Sprintln(strings.Join(args, " "))), stdout...) + + return stdout, stderr, err +} diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index a372abb9250..288bb03e1f4 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -40,8 +40,11 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { // Run preprocessor sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") - stderr, err := GCCPreprocRunner(ctx, sourceFile, targetFilePath, ctx.IncludeFolders) - ctx.WriteStderr(stderr) + gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFilePath, ctx.IncludeFolders, ctx.BuildProperties) + if ctx.Verbose { + ctx.WriteStdout(gccStdout) + ctx.WriteStderr(gccStderr) + } if err != nil { if !ctx.OnlyUpdateCompilationDatabase { return errors.WithStack(err) diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go index 824c5ddb73b..e7b8cbe0994 100644 --- a/legacy/builder/container_find_includes.go +++ b/legacy/builder/container_find_includes.go @@ -98,6 +98,7 @@ import ( "os/exec" "time" + "github.com/arduino/arduino-cli/arduino/builder/preprocessor" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" @@ -369,7 +370,12 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQu ctx.Info(tr("Using cached library dependencies for file: %[1]s", sourcePath)) } } else { - preproc_stderr, preproc_err = GCCPreprocRunner(ctx, sourcePath, targetFilePath, includes) + var preproc_stdout []byte + preproc_stdout, preproc_stderr, preproc_err = preprocessor.GCC(sourcePath, targetFilePath, includes, ctx.BuildProperties) + if ctx.Verbose { + ctx.WriteStdout(preproc_stdout) + ctx.WriteStdout(preproc_stderr) + } // Unwrap error and see if it is an ExitError. _, is_exit_error := errors.Cause(preproc_err).(*exec.ExitError) if preproc_err == nil { @@ -397,11 +403,13 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQu library := ResolveLibrary(ctx, include) if library == nil { // Library could not be resolved, show error - // err := runCommand(ctx, &GCCPreprocRunner{SourceFilePath: sourcePath, TargetFileName: paths.New(constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E), Includes: includes}) - // return errors.WithStack(err) if preproc_err == nil || preproc_stderr == nil { // Filename came from cache, so run preprocessor to obtain error to show - preproc_stderr, preproc_err = GCCPreprocRunner(ctx, sourcePath, targetFilePath, includes) + var preproc_stdout []byte + preproc_stdout, preproc_stderr, preproc_err = preprocessor.GCC(sourcePath, targetFilePath, includes, ctx.BuildProperties) + if ctx.Verbose { + ctx.WriteStdout(preproc_stdout) + } if preproc_err == nil { // If there is a missing #include in the cache, but running // gcc does not reproduce that, there is something wrong. diff --git a/legacy/builder/ctags/ctags_has_issues.go b/legacy/builder/ctags/ctags_has_issues.go index 003aaa035da..a61bf6c6c46 100644 --- a/legacy/builder/ctags/ctags_has_issues.go +++ b/legacy/builder/ctags/ctags_has_issues.go @@ -19,27 +19,20 @@ import ( "bufio" "os" "strings" + + "golang.org/x/exp/slices" ) -func (p *CTagsParser) FixCLinkageTagsDeclarations() { +func (p *CTagsParser) fixCLinkageTagsDeclarations() { linesMap := p.FindCLinkageLines(p.tags) for i := range p.tags { - if sliceContainsInt(linesMap[p.tags[i].Filename], p.tags[i].Line) && + if slices.Contains(linesMap[p.tags[i].Filename], p.tags[i].Line) && !strings.Contains(p.tags[i].PrototypeModifiers, EXTERN) { p.tags[i].PrototypeModifiers = p.tags[i].PrototypeModifiers + " " + EXTERN } } } -func sliceContainsInt(s []int, e int) bool { - for _, a := range s { - if a == e { - return true - } - } - return false -} - func (p *CTagsParser) prototypeAndCodeDontMatch(tag *CTag) bool { if tag.SkipMe { return true diff --git a/legacy/builder/ctags/ctags_parser.go b/legacy/builder/ctags/ctags_parser.go index 61076101be5..5836f7bf337 100644 --- a/legacy/builder/ctags/ctags_parser.go +++ b/legacy/builder/ctags/ctags_parser.go @@ -58,7 +58,7 @@ type CTag struct { PrototypeModifiers string } -func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) []*CTag { +func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) ([]*Prototype, int) { rows := strings.Split(string(ctagsOutput), "\n") rows = removeEmpty(rows) @@ -74,8 +74,9 @@ func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) []*CTag { p.removeDefinedProtypes() p.skipDuplicates() p.skipTagsWhere(p.prototypeAndCodeDontMatch) + p.fixCLinkageTagsDeclarations() - return p.tags + return p.toPrototypes(), p.findLineWhereToInsertPrototypes() } func (p *CTagsParser) addPrototypes() { diff --git a/legacy/builder/ctags/ctags_parser_test.go b/legacy/builder/ctags/ctags_parser_test.go index d3c4abe6b54..97d5c85a4c9 100644 --- a/legacy/builder/ctags/ctags_parser_test.go +++ b/legacy/builder/ctags/ctags_parser_test.go @@ -20,6 +20,7 @@ import ( "path/filepath" "testing" + "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" ) @@ -28,7 +29,8 @@ func produceTags(t *testing.T, filename string) []*CTag { require.NoError(t, err) parser := CTagsParser{} - return parser.Parse(bytes, nil) + parser.Parse(bytes, paths.New("sketch.ino")) + return parser.tags } func TestCTagsParserShouldListPrototypes(t *testing.T) { diff --git a/legacy/builder/ctags/ctags_to_prototypes.go b/legacy/builder/ctags/ctags_to_prototypes.go index d5f750d2da7..71cc072a6fa 100644 --- a/legacy/builder/ctags/ctags_to_prototypes.go +++ b/legacy/builder/ctags/ctags_to_prototypes.go @@ -32,10 +32,6 @@ func (proto *Prototype) String() string { return proto.Modifiers + " " + proto.Prototype + " @ " + strconv.Itoa(proto.Line) } -func (p *CTagsParser) GeneratePrototypes() ([]*Prototype, int) { - return p.toPrototypes(), p.findLineWhereToInsertPrototypes() -} - func (p *CTagsParser) findLineWhereToInsertPrototypes() int { firstFunctionLine := p.firstFunctionAtLine() firstFunctionPointerAsArgument := p.firstFunctionPointerUsedAsArgument() diff --git a/legacy/builder/ctags/ctags_to_prototypes_test.go b/legacy/builder/ctags/ctags_to_prototypes_test.go index 2e36c343b53..98455c22202 100644 --- a/legacy/builder/ctags/ctags_to_prototypes_test.go +++ b/legacy/builder/ctags/ctags_to_prototypes_test.go @@ -29,8 +29,7 @@ func producePrototypes(t *testing.T, filename string, mainFile string) ([]*Proto require.NoError(t, err) parser := &CTagsParser{} - parser.Parse(bytes, paths.New(mainFile)) - return parser.GeneratePrototypes() + return parser.Parse(bytes, paths.New(mainFile)) } func TestCTagsToPrototypesShouldListPrototypes(t *testing.T) { diff --git a/legacy/builder/gcc_preproc_runner.go b/legacy/builder/gcc_preproc_runner.go deleted file mode 100644 index 10876f5d08b..00000000000 --- a/legacy/builder/gcc_preproc_runner.go +++ /dev/null @@ -1,72 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "os/exec" - "strings" - - f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" - "github.com/arduino/go-paths-helper" - properties "github.com/arduino/go-properties-orderedmap" - "github.com/pkg/errors" -) - -func GCCPreprocRunner(ctx *types.Context, sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList) ([]byte, error) { - cmd, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) - if err != nil { - return nil, err - } - _, stderr, err := utils.ExecCommand(ctx, cmd, utils.ShowIfVerbose /* stdout */, utils.Capture /* stderr */) - return stderr, err -} - -func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList) (*exec.Cmd, error) { - buildProperties := properties.NewMap() - buildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") - buildProperties.Merge(ctx.BuildProperties) - buildProperties.Set("build.library_discovery_phase", "1") - buildProperties.SetPath("source_file", sourceFilePath) - buildProperties.SetPath("preprocessed_file_path", targetFilePath) - - includesStrings := f.Map(includes.AsStrings(), utils.WrapWithHyphenI) - buildProperties.Set("includes", strings.Join(includesStrings, " ")) - - if buildProperties.Get("recipe.preproc.macros") == "" { - // autogenerate preprocess macros recipe from compile recipe - preprocPattern := buildProperties.Get("recipe.cpp.o.pattern") - // add {preproc.macros.flags} to {compiler.cpp.flags} - preprocPattern = strings.Replace(preprocPattern, "{compiler.cpp.flags}", "{compiler.cpp.flags} {preproc.macros.flags}", 1) - // replace "{object_file}" with "{preprocessed_file_path}" - preprocPattern = strings.Replace(preprocPattern, "{object_file}", "{preprocessed_file_path}", 1) - - buildProperties.Set("recipe.preproc.macros", preprocPattern) - } - - cmd, err := builder_utils.PrepareCommandForRecipe(buildProperties, "recipe.preproc.macros", true, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) - if err != nil { - return nil, errors.WithStack(err) - } - - // Remove -MMD argument if present. Leaving it will make gcc try - // to create a /dev/null.d dependency file, which won't work. - cmd.Args = f.Filter(cmd.Args, f.NotEquals("-MMD")) - - return cmd, nil -} diff --git a/legacy/builder/preprocess_sketch.go b/legacy/builder/preprocess_sketch.go index efd6721847d..7f8aaccec38 100644 --- a/legacy/builder/preprocess_sketch.go +++ b/legacy/builder/preprocess_sketch.go @@ -23,6 +23,7 @@ import ( "runtime" bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/preprocessor" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" properties "github.com/arduino/go-properties-orderedmap" @@ -36,11 +37,15 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error { sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") targetFile := ctx.PreprocPath.Join("sketch_merged.cpp") - stderr, err := GCCPreprocRunner(ctx, sourceFile, targetFile, ctx.IncludeFolders) - ctx.WriteStderr(stderr) + gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFile, ctx.IncludeFolders, ctx.BuildProperties) + if ctx.Verbose { + ctx.WriteStdout(gccStdout) + ctx.WriteStderr(gccStderr) + } if err != nil { return err } + buildProperties := properties.NewMap() buildProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}") buildProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor") diff --git a/legacy/builder/prototypes_adder.go b/legacy/builder/prototypes_adder.go index fba8dd94e4c..e62a8486fa8 100644 --- a/legacy/builder/prototypes_adder.go +++ b/legacy/builder/prototypes_adder.go @@ -30,10 +30,7 @@ var DebugPreprocessor bool func PrototypesAdder(sketch *sketch.Sketch, source string, ctagsStdout []byte, lineOffset int) string { parser := &ctags.CTagsParser{} - parser.Parse(ctagsStdout, sketch.MainFile) - parser.FixCLinkageTagsDeclarations() - - prototypes, firstFunctionLine := parser.GeneratePrototypes() + prototypes, firstFunctionLine := parser.Parse(ctagsStdout, sketch.MainFile) if firstFunctionLine == -1 { firstFunctionLine = 0 }