diff --git a/.travis.yml b/.travis.yml index 910c086e..665f1418 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ install: - go get github.com/wadey/gocovmerge - go get github.com/arduino/go-properties-map - go get github.com/arduino/go-timeutils + - go get github.com/marcinbor85/gohex script: - go get github.com/arduino/arduino-builder/arduino-builder diff --git a/merge_sketch_with_bootloader.go b/merge_sketch_with_bootloader.go index ebefa77e..f27ac426 100644 --- a/merge_sketch_with_bootloader.go +++ b/merge_sketch_with_bootloader.go @@ -30,13 +30,17 @@ package builder import ( - "github.com/arduino/arduino-builder/constants" - "github.com/arduino/arduino-builder/i18n" - "github.com/arduino/arduino-builder/types" - "github.com/arduino/arduino-builder/utils" + "errors" + "math" "os" "path/filepath" "strings" + "strconv" + + "github.com/arduino/arduino-builder/constants" + "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" + "github.com/marcinbor85/gohex" ) type MergeSketchWithBootloader struct{} @@ -80,70 +84,95 @@ func (s *MergeSketchWithBootloader) Run(ctx *types.Context) error { mergedSketchPath := filepath.Join(filepath.Dir(builtSketchPath), sketchFileName+".with_bootloader.hex") - err := merge(builtSketchPath, bootloaderPath, mergedSketchPath) - - return err -} - -func hexLineOnlyContainsFF(line string) bool { - //:206FE000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFB1 - if len(line) <= 11 { - return false + // Ignore merger errors for the first iteration + maximumBinSize := 16000000 + if uploadMaxSize, ok := ctx.BuildProperties[constants.PROPERTY_UPLOAD_MAX_SIZE]; ok { + maximumBinSize, _ = strconv.Atoi(uploadMaxSize) + maximumBinSize *= 2 } - byteArray := []byte(line) - for _, char := range byteArray[9:(len(byteArray) - 2)] { - if char != 'F' { - return false - } + err := merge(builtSketchPath, bootloaderPath, mergedSketchPath, maximumBinSize) + if err != nil { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_WARN, err.Error()) } - return true -} -func extractActualBootloader(bootloader []string) []string { + return nil +} - var realBootloader []string +func merge(builtSketchPath, bootloaderPath, mergedSketchPath string, maximumBinSize int) error { - // skip until we find a line full of FFFFFF (except address and checksum) - for i, row := range bootloader { - if hexLineOnlyContainsFF(row) { - realBootloader = bootloader[i:len(bootloader)] - break - } + if filepath.Ext(bootloaderPath) == ".bin" { + bootloaderPath = strings.TrimSuffix(bootloaderPath, ".bin") + ".hex" } - // drop all "empty" lines - for i, row := range realBootloader { - if !hexLineOnlyContainsFF(row) { - realBootloader = realBootloader[i:len(realBootloader)] - break - } + bootFile, err := os.Open(bootloaderPath) + if err != nil { + return err } + defer bootFile.Close() - if len(realBootloader) == 0 { - // we didn't find any line full of FFFF, thus it's a standalone bootloader - realBootloader = bootloader + mem_boot := gohex.NewMemory() + err = mem_boot.ParseIntelHex(bootFile) + if err != nil { + return errors.New(bootFile.Name() + " " + err.Error()) } - return realBootloader -} + buildFile, err := os.Open(builtSketchPath) + if err != nil { + return err + } + defer buildFile.Close() -func merge(builtSketchPath, bootloaderPath, mergedSketchPath string) error { - sketch, err := utils.ReadFileToRows(builtSketchPath) + mem_sketch := gohex.NewMemory() + err = mem_sketch.ParseIntelHex(buildFile) if err != nil { - return i18n.WrapError(err) + return errors.New(buildFile.Name() + " " + err.Error()) } - sketch = sketch[:len(sketch)-2] - bootloader, err := utils.ReadFileToRows(bootloaderPath) + mem_merge := gohex.NewMemory() + initial_address := uint32(math.MaxUint32) + last_address := uint32(0) + + for _, segment := range mem_boot.GetDataSegments() { + err = mem_merge.AddBinary(segment.Address, segment.Data) + if err != nil { + continue + } else { + if segment.Address < initial_address { + initial_address = segment.Address + } + if segment.Address+uint32(len(segment.Data)) > last_address { + last_address = segment.Address + uint32(len(segment.Data)) + } + } + } + for _, segment := range mem_sketch.GetDataSegments() { + err = mem_merge.AddBinary(segment.Address, segment.Data) + if err != nil { + continue + } + if segment.Address < initial_address { + initial_address = segment.Address + } + if segment.Address+uint32(len(segment.Data)) > last_address { + last_address = segment.Address + uint32(len(segment.Data)) + } + } + + mergeFile, err := os.Create(mergedSketchPath) if err != nil { - return i18n.WrapError(err) + return err } + defer mergeFile.Close() + + mem_merge.DumpIntelHex(mergeFile, 16) - realBootloader := extractActualBootloader(bootloader) + mergedSketchPathBin := strings.TrimSuffix(mergedSketchPath, ".hex") + ".bin" - for _, row := range realBootloader { - sketch = append(sketch, row) + size := last_address - initial_address + if (size > uint32(maximumBinSize)) { + return nil } - return utils.WriteFile(mergedSketchPath, strings.Join(sketch, "\n")) + bytes := mem_merge.ToBinary(initial_address, last_address-initial_address, 0xFF) + return utils.WriteFile(mergedSketchPathBin, string(bytes)) } diff --git a/test/merge_sketch_with_bootloader_test.go b/test/merge_sketch_with_bootloader_test.go index 5d6776cf..db03dad8 100644 --- a/test/merge_sketch_with_bootloader_test.go +++ b/test/merge_sketch_with_bootloader_test.go @@ -30,16 +30,17 @@ package test import ( - "github.com/arduino/arduino-builder" - "github.com/arduino/arduino-builder/constants" - "github.com/arduino/arduino-builder/types" - "github.com/arduino/arduino-builder/utils" - "github.com/stretchr/testify/require" "io/ioutil" "os" "path/filepath" "strings" "testing" + + "github.com/arduino/arduino-builder" + "github.com/arduino/arduino-builder/constants" + "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" + "github.com/stretchr/testify/require" ) func TestMergeSketchWithBootloader(t *testing.T) { @@ -61,8 +62,35 @@ func TestMergeSketchWithBootloader(t *testing.T) { err := utils.EnsureFolderExists(filepath.Join(buildPath, "sketch")) NoError(t, err) - fakeSketchHex := "row 1\n" + - "row 2\n" + fakeSketchHex := ":100000000C9434000C9446000C9446000C9446006A\n" + + ":100010000C9446000C9446000C9446000C94460048\n" + + ":100020000C9446000C9446000C9446000C94460038\n" + + ":100030000C9446000C9446000C9446000C94460028\n" + + ":100040000C9448000C9446000C9446000C94460016\n" + + ":100050000C9446000C9446000C9446000C94460008\n" + + ":100060000C9446000C94460011241FBECFEFD8E03C\n" + + ":10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC\n" + + ":10008000B207E1F70E9492000C94DC000C9400008F\n" + + ":100090001F920F920FB60F9211242F933F938F93BD\n" + + ":1000A0009F93AF93BF938091050190910601A0911A\n" + + ":1000B0000701B09108013091040123E0230F2D378F\n" + + ":1000C00020F40196A11DB11D05C026E8230F02965C\n" + + ":1000D000A11DB11D20930401809305019093060199\n" + + ":1000E000A0930701B0930801809100019091010154\n" + + ":1000F000A0910201B09103010196A11DB11D809351\n" + + ":10010000000190930101A0930201B0930301BF91FC\n" + + ":10011000AF919F918F913F912F910F900FBE0F90B4\n" + + ":100120001F901895789484B5826084BD84B58160F1\n" + + ":1001300084BD85B5826085BD85B5816085BD8091B2\n" + + ":100140006E00816080936E0010928100809181002A\n" + + ":100150008260809381008091810081608093810022\n" + + ":10016000809180008160809380008091B1008460E4\n" + + ":100170008093B1008091B00081608093B000809145\n" + + ":100180007A00846080937A0080917A008260809304\n" + + ":100190007A0080917A00816080937A0080917A0061\n" + + ":1001A000806880937A001092C100C0E0D0E0209770\n" + + ":0C01B000F1F30E940000FBCFF894FFCF99\n" + + ":00000001FF\n" err = utils.WriteFile(filepath.Join(buildPath, "sketch", "sketch.ino.hex"), fakeSketchHex) NoError(t, err) @@ -80,8 +108,8 @@ func TestMergeSketchWithBootloader(t *testing.T) { NoError(t, err) mergedSketchHex := string(bytes) - require.True(t, strings.HasPrefix(mergedSketchHex, "row 1\n:107E0000112484B714BE81FFF0D085E080938100F7\n")) - require.True(t, strings.HasSuffix(mergedSketchHex, ":0400000300007E007B\n:00000001FF\n")) + require.True(t, strings.HasPrefix(mergedSketchHex, ":100000000C9434000C9446000C9446000C9446006A\n")) + require.True(t, strings.HasSuffix(mergedSketchHex, ":00000001FF\n")) } func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) { @@ -103,8 +131,35 @@ func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) { err := utils.EnsureFolderExists(filepath.Join(buildPath, "sketch")) NoError(t, err) - fakeSketchHex := "row 1\n" + - "row 2\n" + fakeSketchHex := ":100000000C9434000C9446000C9446000C9446006A\n" + + ":100010000C9446000C9446000C9446000C94460048\n" + + ":100020000C9446000C9446000C9446000C94460038\n" + + ":100030000C9446000C9446000C9446000C94460028\n" + + ":100040000C9448000C9446000C9446000C94460016\n" + + ":100050000C9446000C9446000C9446000C94460008\n" + + ":100060000C9446000C94460011241FBECFEFD8E03C\n" + + ":10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC\n" + + ":10008000B207E1F70E9492000C94DC000C9400008F\n" + + ":100090001F920F920FB60F9211242F933F938F93BD\n" + + ":1000A0009F93AF93BF938091050190910601A0911A\n" + + ":1000B0000701B09108013091040123E0230F2D378F\n" + + ":1000C00020F40196A11DB11D05C026E8230F02965C\n" + + ":1000D000A11DB11D20930401809305019093060199\n" + + ":1000E000A0930701B0930801809100019091010154\n" + + ":1000F000A0910201B09103010196A11DB11D809351\n" + + ":10010000000190930101A0930201B0930301BF91FC\n" + + ":10011000AF919F918F913F912F910F900FBE0F90B4\n" + + ":100120001F901895789484B5826084BD84B58160F1\n" + + ":1001300084BD85B5826085BD85B5816085BD8091B2\n" + + ":100140006E00816080936E0010928100809181002A\n" + + ":100150008260809381008091810081608093810022\n" + + ":10016000809180008160809380008091B1008460E4\n" + + ":100170008093B1008091B00081608093B000809145\n" + + ":100180007A00846080937A0080917A008260809304\n" + + ":100190007A0080917A00816080937A0080917A0061\n" + + ":1001A000806880937A001092C100C0E0D0E0209770\n" + + ":0C01B000F1F30E940000FBCFF894FFCF99\n" + + ":00000001FF\n" err = utils.WriteFile(filepath.Join(buildPath, "sketch.ino.hex"), fakeSketchHex) NoError(t, err) @@ -122,8 +177,8 @@ func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) { NoError(t, err) mergedSketchHex := string(bytes) - require.True(t, strings.HasPrefix(mergedSketchHex, "row 1\n:107E0000112484B714BE81FFF0D085E080938100F7\n")) - require.True(t, strings.HasSuffix(mergedSketchHex, ":0400000300007E007B\n:00000001FF\n")) + require.True(t, strings.HasPrefix(mergedSketchHex, ":100000000C9434000C9446000C9446000C9446006A\n")) + require.True(t, strings.HasSuffix(mergedSketchHex, ":00000001FF\n")) } func TestMergeSketchWithBootloaderWhenNoBootloaderAvailable(t *testing.T) { @@ -183,8 +238,35 @@ func TestMergeSketchWithBootloaderPathIsParameterized(t *testing.T) { err := utils.EnsureFolderExists(filepath.Join(buildPath, "sketch")) NoError(t, err) - fakeSketchHex := "row 1\n" + - "row 2\n" + fakeSketchHex := ":100000000C9434000C9446000C9446000C9446006A\n" + + ":100010000C9446000C9446000C9446000C94460048\n" + + ":100020000C9446000C9446000C9446000C94460038\n" + + ":100030000C9446000C9446000C9446000C94460028\n" + + ":100040000C9448000C9446000C9446000C94460016\n" + + ":100050000C9446000C9446000C9446000C94460008\n" + + ":100060000C9446000C94460011241FBECFEFD8E03C\n" + + ":10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC\n" + + ":10008000B207E1F70E9492000C94DC000C9400008F\n" + + ":100090001F920F920FB60F9211242F933F938F93BD\n" + + ":1000A0009F93AF93BF938091050190910601A0911A\n" + + ":1000B0000701B09108013091040123E0230F2D378F\n" + + ":1000C00020F40196A11DB11D05C026E8230F02965C\n" + + ":1000D000A11DB11D20930401809305019093060199\n" + + ":1000E000A0930701B0930801809100019091010154\n" + + ":1000F000A0910201B09103010196A11DB11D809351\n" + + ":10010000000190930101A0930201B0930301BF91FC\n" + + ":10011000AF919F918F913F912F910F900FBE0F90B4\n" + + ":100120001F901895789484B5826084BD84B58160F1\n" + + ":1001300084BD85B5826085BD85B5816085BD8091B2\n" + + ":100140006E00816080936E0010928100809181002A\n" + + ":100150008260809381008091810081608093810022\n" + + ":10016000809180008160809380008091B1008460E4\n" + + ":100170008093B1008091B00081608093B000809145\n" + + ":100180007A00846080937A0080917A008260809304\n" + + ":100190007A0080917A00816080937A0080917A0061\n" + + ":1001A000806880937A001092C100C0E0D0E0209770\n" + + ":0C01B000F1F30E940000FBCFF894FFCF99\n" + + ":00000001FF\n" err = utils.WriteFile(filepath.Join(buildPath, "sketch", "sketch.ino.hex"), fakeSketchHex) NoError(t, err) @@ -202,6 +284,6 @@ func TestMergeSketchWithBootloaderPathIsParameterized(t *testing.T) { NoError(t, err) mergedSketchHex := string(bytes) - require.True(t, strings.HasPrefix(mergedSketchHex, "row 1\n:020000023000CC")) - require.True(t, strings.HasSuffix(mergedSketchHex, ":040000033000E000E9\n:00000001FF\n")) + require.True(t, strings.HasPrefix(mergedSketchHex, ":100000000C9434000C9446000C9446000C9446006A\n")) + require.True(t, strings.HasSuffix(mergedSketchHex, ":00000001FF\n")) }