Added 'source override' feature to compile command. (#1099)

This feature allows to selectively "override" the content of a sketch.
Overridden files will not be read from disk as usual but fetched directly
from gRPC paramaters (if the cli is running as daemon) or from a .json
file containing the new source code (if running from command line).

Author: cmaglie

arduino/builder/sketch.go

@@ -216,12 +216,23 @@ func SketchLoad(sketchPath, buildPath string) (*sketch.Sketch, error) {
 }
 
 // SketchMergeSources merges all the source files included in a sketch
-func SketchMergeSources(sketch *sketch.Sketch) (int, string, error) {
+func SketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, string, error) {
 	lineOffset := 0
 	mergedSource := ""
 
+	getSource := func(i *sketch.Item) (string, error) {
+		path, err := filepath.Rel(sk.LocationPath, i.Path)
+		if err != nil {
+			return "", errors.Wrap(err, "unable to compute relative path to the sketch for the item")
+		}
+		if override, ok := overrides[path]; ok {
+			return override, nil
+		}
+		return i.GetSourceStr()
+	}
+
 	// add Arduino.h inclusion directive if missing
-	mainSrc, err := sketch.MainFile.GetSourceStr()
+	mainSrc, err := getSource(sk.MainFile)
 	if err != nil {
 		return 0, "", err
 	}
@@ -230,12 +241,12 @@ func SketchMergeSources(sketch *sketch.Sketch) (int, string, error) {
 		lineOffset++
 	}
 
-	mergedSource += "#line 1 " + QuoteCppString(sketch.MainFile.Path) + "\n"
+	mergedSource += "#line 1 " + QuoteCppString(sk.MainFile.Path) + "\n"
 	mergedSource += mainSrc + "\n"
 	lineOffset++
 
-	for _, item := range sketch.OtherSketchFiles {
-		src, err := item.GetSourceStr()
+	for _, item := range sk.OtherSketchFiles {
+		src, err := getSource(item)
 		if err != nil {
 			return 0, "", err
 		}
@@ -248,7 +259,7 @@ func SketchMergeSources(sketch *sketch.Sketch) (int, string, error) {
 
 // SketchCopyAdditionalFiles copies the additional files for a sketch to the
 // specified destination directory.
-func SketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath string) error {
+func SketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath string, overrides map[string]string) error {
 	if err := os.MkdirAll(destPath, os.FileMode(0755)); err != nil {
 		return errors.Wrap(err, "unable to create a folder to save the sketch files")
 	}
@@ -265,7 +276,20 @@ func SketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath string) error {
 			return errors.Wrap(err, "unable to create the folder containing the item")
 		}
 
-		err = writeIfDifferent(item.Path, targetPath)
+		var sourceBytes []byte
+		if override, ok := overrides[relpath]; ok {
+			// use override source
+			sourceBytes = []byte(override)
+		} else {
+			// read the source file
+			s, err := item.GetSourceBytes()
+			if err != nil {
+				return errors.Wrap(err, "unable to read contents of the source item")
+			}
+			sourceBytes = s
+		}
+
+		err = writeIfDifferent(sourceBytes, targetPath)
 		if err != nil {
 			return errors.Wrap(err, "unable to write to destination file")
 		}
@@ -274,18 +298,12 @@ func SketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath string) error {
 	return nil
 }
 
-func writeIfDifferent(sourcePath, destPath string) error {
-	// read the source file
-	newbytes, err := ioutil.ReadFile(sourcePath)
-	if err != nil {
-		return errors.Wrap(err, "unable to read contents of the source item")
-	}
-
+func writeIfDifferent(source []byte, destPath string) error {
 	// check whether the destination file exists
-	_, err = os.Stat(destPath)
+	_, err := os.Stat(destPath)
 	if os.IsNotExist(err) {
 		// write directly
-		return ioutil.WriteFile(destPath, newbytes, os.FileMode(0644))
+		return ioutil.WriteFile(destPath, source, os.FileMode(0644))
 	}
 
 	// read the destination file if it ex
@@ -295,8 +313,8 @@ func writeIfDifferent(sourcePath, destPath string) error {
 	}
 
 	// overwrite if contents are different
-	if bytes.Compare(existingBytes, newbytes) != 0 {
-		return ioutil.WriteFile(destPath, newbytes, os.FileMode(0644))
+	if bytes.Compare(existingBytes, source) != 0 {
+		return ioutil.WriteFile(destPath, source, os.FileMode(0644))
 	}
 
 	// source and destination are the same, don't write anything

arduino/builder/sketch_test.go

@@ -180,7 +180,7 @@ func TestMergeSketchSources(t *testing.T) {
 		t.Fatalf("unable to read golden file %s: %v", mergedPath, err)
 	}
 
-	offset, source, err := builder.SketchMergeSources(s)
+	offset, source, err := builder.SketchMergeSources(s, nil)
 	require.Nil(t, err)
 	require.Equal(t, 2, offset)
 	require.Equal(t, string(mergedBytes), source)
@@ -192,7 +192,7 @@ func TestMergeSketchSourcesArduinoIncluded(t *testing.T) {
 	require.NotNil(t, s)
 
 	// ensure not to include Arduino.h when it's already there
-	_, source, err := builder.SketchMergeSources(s)
+	_, source, err := builder.SketchMergeSources(s, nil)
 	require.Nil(t, err)
 	require.Equal(t, 1, strings.Count(source, "<Arduino.h>"))
 }
@@ -208,7 +208,7 @@ func TestCopyAdditionalFiles(t *testing.T) {
 
 	// copy the sketch over, create a fake main file we don't care about it
 	// but we need it for `SketchLoad` to succeed later
-	err = builder.SketchCopyAdditionalFiles(s1, tmp)
+	err = builder.SketchCopyAdditionalFiles(s1, tmp, nil)
 	require.Nil(t, err)
 	fakeIno := filepath.Join(tmp, fmt.Sprintf("%s.ino", filepath.Base(tmp)))
 	require.Nil(t, ioutil.WriteFile(fakeIno, []byte{}, os.FileMode(0644)))
@@ -223,7 +223,7 @@ func TestCopyAdditionalFiles(t *testing.T) {
 	require.Nil(t, err)
 
 	// copy again
-	err = builder.SketchCopyAdditionalFiles(s1, tmp)
+	err = builder.SketchCopyAdditionalFiles(s1, tmp, nil)
 	require.Nil(t, err)
 
 	// verify file hasn't changed

cli/compile/compile.go

@@ -18,6 +18,7 @@ package compile
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"os"
 
 	"github.com/arduino/arduino-cli/cli/feedback"
@@ -55,6 +56,7 @@ var (
 	clean                   bool     // Cleanup the build folder and do not use any cached build
 	exportBinaries          bool     // Copies compiled binaries to sketch folder when true
 	compilationDatabaseOnly bool     // Only create compilation database without actually compiling
+	sourceOverrides         string   // Path to a .json file that contains a set of replacements of the sketch source code.
 )
 
 // NewCommand created a new `compile` command
@@ -102,6 +104,8 @@ func NewCommand() *cobra.Command {
 	// This must be done because the value is set when the binding is accessed from viper. Accessing from cobra would only
 	// read the value if the flag is set explicitly by the user.
 	command.Flags().BoolP("export-binaries", "e", false, "If set built binaries will be exported to the sketch folder.")
+	command.Flags().StringVar(&sourceOverrides, "source-override", "", "Optional. Path to a .json file that contains a set of replacements of the sketch source code.")
+	command.Flag("source-override").Hidden = true
 
 	configuration.Settings.BindPFlag("sketch.always_export_binaries", command.Flags().Lookup("export-binaries"))
 
@@ -128,6 +132,23 @@ func run(cmd *cobra.Command, args []string) {
 	// the config file and the env vars.
 	exportBinaries = configuration.Settings.GetBool("sketch.always_export_binaries")
 
+	var overrides map[string]string
+	if sourceOverrides != "" {
+		data, err := paths.New(sourceOverrides).ReadFile()
+		if err != nil {
+			feedback.Errorf("Error opening source code overrides data file: %v", err)
+			os.Exit(errorcodes.ErrGeneric)
+		}
+		var o struct {
+			Overrides map[string]string `json:"overrides"`
+		}
+		if err := json.Unmarshal(data, &o); err != nil {
+			feedback.Errorf("Error: invalid source code overrides data file: %v", err)
+			os.Exit(errorcodes.ErrGeneric)
+		}
+		overrides = o.Overrides
+	}
+
 	compileReq := &rpc.CompileReq{
 		Instance:                      inst,
 		Fqbn:                          fqbn,
@@ -147,6 +168,7 @@ func run(cmd *cobra.Command, args []string) {
 		Clean:                         clean,
 		ExportBinaries:                exportBinaries,
 		CreateCompilationDatabaseOnly: compilationDatabaseOnly,
+		SourceOverride:                overrides,
 	}
 	compileOut := new(bytes.Buffer)
 	compileErr := new(bytes.Buffer)

commands/compile/compile.go

@@ -192,6 +192,8 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 	builderCtx.Clean = req.GetClean()
 	builderCtx.OnlyUpdateCompilationDatabase = req.GetCreateCompilationDatabaseOnly()
 
+	builderCtx.SourceOverride = req.GetSourceOverride()
+
 	// Use defer() to create an rpc.CompileResp with all the information available at the
 	// moment of return.
 	defer func() {

legacy/builder/container_merge_copy_sketch_files.go

@@ -28,7 +28,7 @@ func (s *ContainerMergeCopySketchFiles) Run(ctx *types.Context) error {
 	if sk == nil {
 		return errors.New("unable to convert legacy sketch to the new type")
 	}
-	offset, source, err := bldr.SketchMergeSources(sk)
+	offset, source, err := bldr.SketchMergeSources(sk, ctx.SourceOverride)
 	if err != nil {
 		return err
 	}
@@ -39,7 +39,7 @@ func (s *ContainerMergeCopySketchFiles) Run(ctx *types.Context) error {
 		return errors.WithStack(err)
 	}
 
-	if err := bldr.SketchCopyAdditionalFiles(sk, ctx.SketchBuildPath.String()); err != nil {
+	if err := bldr.SketchCopyAdditionalFiles(sk, ctx.SketchBuildPath.String(), ctx.SourceOverride); err != nil {
 		return errors.WithStack(err)
 	}
 

legacy/builder/types/context.go

@@ -169,6 +169,11 @@ type Context struct {
 	CompilationDatabase *builder.CompilationDatabase
 	// Set to true to skip build and produce only Compilation Database
 	OnlyUpdateCompilationDatabase bool
+
+	// Source code overrides (filename -> content map).
+	// The provided source data is used instead of reading it from disk.
+	// The keys of the map are paths relative to sketch folder.
+	SourceOverride map[string]string
 }
 
 // ExecutableSectionSize represents a section of the executable output file