From 1707aec9fcdceb7d7261a0cebcd838fc067de21a Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Fri, 14 Sep 2018 15:54:16 +0200 Subject: [PATCH 1/5] Start introducing a proper hex merger Uses github.com/marcinbor85/gohex as additiona dependency --- merge_sketch_with_bootloader.go | 90 ++++++++++++++------------------- 1 file changed, 37 insertions(+), 53 deletions(-) diff --git a/merge_sketch_with_bootloader.go b/merge_sketch_with_bootloader.go index ebefa77e..c00cf7dd 100644 --- a/merge_sketch_with_bootloader.go +++ b/merge_sketch_with_bootloader.go @@ -30,13 +30,13 @@ package builder import ( + "os" + "path/filepath" + "github.com/arduino/arduino-builder/constants" - "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" - "os" - "path/filepath" - "strings" + "github.com/marcinbor85/gohex" ) type MergeSketchWithBootloader struct{} @@ -80,70 +80,54 @@ func (s *MergeSketchWithBootloader) Run(ctx *types.Context) error { mergedSketchPath := filepath.Join(filepath.Dir(builtSketchPath), sketchFileName+".with_bootloader.hex") - err := merge(builtSketchPath, bootloaderPath, mergedSketchPath) + // Ignore merger errors for the first iteration + merge(builtSketchPath, bootloaderPath, mergedSketchPath) - return err + return nil } -func hexLineOnlyContainsFF(line string) bool { - //:206FE000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFB1 - if len(line) <= 11 { - return false - } - byteArray := []byte(line) - for _, char := range byteArray[9:(len(byteArray) - 2)] { - if char != 'F' { - return false - } +func merge(builtSketchPath, bootloaderPath, mergedSketchPath string) error { + bootFile, err := os.Open(bootloaderPath) + if err != nil { + return err } - return true -} - -func extractActualBootloader(bootloader []string) []string { - - var realBootloader []string + defer bootFile.Close() - // skip until we find a line full of FFFFFF (except address and checksum) - for i, row := range bootloader { - if hexLineOnlyContainsFF(row) { - realBootloader = bootloader[i:len(bootloader)] - break - } + mem_boot := gohex.NewMemory() + err = mem_boot.ParseIntelHex(bootFile) + if err != nil { + return err } - // drop all "empty" lines - for i, row := range realBootloader { - if !hexLineOnlyContainsFF(row) { - realBootloader = realBootloader[i:len(realBootloader)] - break - } + buildFile, err := os.Open(builtSketchPath) + if err != nil { + return err } + defer buildFile.Close() - if len(realBootloader) == 0 { - // we didn't find any line full of FFFF, thus it's a standalone bootloader - realBootloader = bootloader + mem_sketch := gohex.NewMemory() + err = mem_sketch.ParseIntelHex(buildFile) + if err != nil { + return err } - return realBootloader -} + mem_merge := gohex.NewMemory() -func merge(builtSketchPath, bootloaderPath, mergedSketchPath string) error { - sketch, err := utils.ReadFileToRows(builtSketchPath) - if err != nil { - return i18n.WrapError(err) + for _, segment := range mem_boot.GetDataSegments() { + err = mem_merge.AddBinary(segment.Address, segment.Data) } - sketch = sketch[:len(sketch)-2] - - bootloader, err := utils.ReadFileToRows(bootloaderPath) - if err != nil { - return i18n.WrapError(err) + for _, segment := range mem_sketch.GetDataSegments() { + err = mem_merge.AddBinary(segment.Address, segment.Data) } - realBootloader := extractActualBootloader(bootloader) - - for _, row := range realBootloader { - sketch = append(sketch, row) + mergeFile, err := os.Create(mergedSketchPath) + if err != nil { + return err } + defer mergeFile.Close() - return utils.WriteFile(mergedSketchPath, strings.Join(sketch, "\n")) + mem_merge.DumpIntelHex(mergeFile, 32) + return nil + //bytes := mem_merge.ToBinary(0, len, 0xFF) + //return utils.WriteFile(mergedSketchPath, string(bytes)) } From 7a6bad655c9be5113d416fc38e09da312b5f3f56 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Thu, 20 Sep 2018 16:34:54 +0200 Subject: [PATCH 2/5] Properly handle hex<->bin operations --- merge_sketch_with_bootloader.go | 48 ++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/merge_sketch_with_bootloader.go b/merge_sketch_with_bootloader.go index c00cf7dd..fb42c43a 100644 --- a/merge_sketch_with_bootloader.go +++ b/merge_sketch_with_bootloader.go @@ -30,8 +30,11 @@ package builder import ( + "errors" + "math" "os" "path/filepath" + "strings" "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/types" @@ -81,12 +84,20 @@ func (s *MergeSketchWithBootloader) Run(ctx *types.Context) error { mergedSketchPath := filepath.Join(filepath.Dir(builtSketchPath), sketchFileName+".with_bootloader.hex") // Ignore merger errors for the first iteration - merge(builtSketchPath, bootloaderPath, mergedSketchPath) + err := merge(builtSketchPath, bootloaderPath, mergedSketchPath) + if err != nil { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_WARN, err.Error()) + } return nil } func merge(builtSketchPath, bootloaderPath, mergedSketchPath string) error { + + if filepath.Ext(bootloaderPath) == ".bin" { + bootloaderPath = strings.TrimSuffix(bootloaderPath, ".bin") + ".hex" + } + bootFile, err := os.Open(bootloaderPath) if err != nil { return err @@ -96,7 +107,7 @@ func merge(builtSketchPath, bootloaderPath, mergedSketchPath string) error { mem_boot := gohex.NewMemory() err = mem_boot.ParseIntelHex(bootFile) if err != nil { - return err + return errors.New(bootFile.Name() + " " + err.Error()) } buildFile, err := os.Open(builtSketchPath) @@ -108,16 +119,37 @@ func merge(builtSketchPath, bootloaderPath, mergedSketchPath string) error { mem_sketch := gohex.NewMemory() err = mem_sketch.ParseIntelHex(buildFile) if err != nil { - return err + return errors.New(buildFile.Name() + " " + err.Error()) } mem_merge := gohex.NewMemory() + initial_address := uint32(math.MaxUint32) + last_address := uint32(0) for _, segment := range mem_boot.GetDataSegments() { err = mem_merge.AddBinary(segment.Address, segment.Data) + if err != nil { + continue + } else { + if segment.Address < initial_address { + initial_address = segment.Address + } + if segment.Address+uint32(len(segment.Data)) > last_address { + last_address = segment.Address + uint32(len(segment.Data)) + } + } } for _, segment := range mem_sketch.GetDataSegments() { err = mem_merge.AddBinary(segment.Address, segment.Data) + if err != nil { + continue + } + if segment.Address < initial_address { + initial_address = segment.Address + } + if segment.Address+uint32(len(segment.Data)) > last_address { + last_address = segment.Address + uint32(len(segment.Data)) + } } mergeFile, err := os.Create(mergedSketchPath) @@ -126,8 +158,10 @@ func merge(builtSketchPath, bootloaderPath, mergedSketchPath string) error { } defer mergeFile.Close() - mem_merge.DumpIntelHex(mergeFile, 32) - return nil - //bytes := mem_merge.ToBinary(0, len, 0xFF) - //return utils.WriteFile(mergedSketchPath, string(bytes)) + mem_merge.DumpIntelHex(mergeFile, 16) + + mergedSketchPathBin := strings.TrimSuffix(mergedSketchPath, ".hex") + ".bin" + + bytes := mem_merge.ToBinary(initial_address, last_address-initial_address, 0xFF) + return utils.WriteFile(mergedSketchPathBin, string(bytes)) } From 9a61ec1910cdbdb00c6d6688d952fbccd909d3ed Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Thu, 20 Sep 2018 16:35:09 +0200 Subject: [PATCH 3/5] Fix merge bootloader test Since we have a proper hex merger now, let's fix the dummy tests We should also add some new tests (for overlapping sections, for example) --- test/merge_sketch_with_bootloader_test.go | 116 ++++++++++++++++++---- 1 file changed, 99 insertions(+), 17 deletions(-) diff --git a/test/merge_sketch_with_bootloader_test.go b/test/merge_sketch_with_bootloader_test.go index 5d6776cf..db03dad8 100644 --- a/test/merge_sketch_with_bootloader_test.go +++ b/test/merge_sketch_with_bootloader_test.go @@ -30,16 +30,17 @@ package test import ( - "github.com/arduino/arduino-builder" - "github.com/arduino/arduino-builder/constants" - "github.com/arduino/arduino-builder/types" - "github.com/arduino/arduino-builder/utils" - "github.com/stretchr/testify/require" "io/ioutil" "os" "path/filepath" "strings" "testing" + + "github.com/arduino/arduino-builder" + "github.com/arduino/arduino-builder/constants" + "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" + "github.com/stretchr/testify/require" ) func TestMergeSketchWithBootloader(t *testing.T) { @@ -61,8 +62,35 @@ func TestMergeSketchWithBootloader(t *testing.T) { err := utils.EnsureFolderExists(filepath.Join(buildPath, "sketch")) NoError(t, err) - fakeSketchHex := "row 1\n" + - "row 2\n" + fakeSketchHex := ":100000000C9434000C9446000C9446000C9446006A\n" + + ":100010000C9446000C9446000C9446000C94460048\n" + + ":100020000C9446000C9446000C9446000C94460038\n" + + ":100030000C9446000C9446000C9446000C94460028\n" + + ":100040000C9448000C9446000C9446000C94460016\n" + + ":100050000C9446000C9446000C9446000C94460008\n" + + ":100060000C9446000C94460011241FBECFEFD8E03C\n" + + ":10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC\n" + + ":10008000B207E1F70E9492000C94DC000C9400008F\n" + + ":100090001F920F920FB60F9211242F933F938F93BD\n" + + ":1000A0009F93AF93BF938091050190910601A0911A\n" + + ":1000B0000701B09108013091040123E0230F2D378F\n" + + ":1000C00020F40196A11DB11D05C026E8230F02965C\n" + + ":1000D000A11DB11D20930401809305019093060199\n" + + ":1000E000A0930701B0930801809100019091010154\n" + + ":1000F000A0910201B09103010196A11DB11D809351\n" + + ":10010000000190930101A0930201B0930301BF91FC\n" + + ":10011000AF919F918F913F912F910F900FBE0F90B4\n" + + ":100120001F901895789484B5826084BD84B58160F1\n" + + ":1001300084BD85B5826085BD85B5816085BD8091B2\n" + + ":100140006E00816080936E0010928100809181002A\n" + + ":100150008260809381008091810081608093810022\n" + + ":10016000809180008160809380008091B1008460E4\n" + + ":100170008093B1008091B00081608093B000809145\n" + + ":100180007A00846080937A0080917A008260809304\n" + + ":100190007A0080917A00816080937A0080917A0061\n" + + ":1001A000806880937A001092C100C0E0D0E0209770\n" + + ":0C01B000F1F30E940000FBCFF894FFCF99\n" + + ":00000001FF\n" err = utils.WriteFile(filepath.Join(buildPath, "sketch", "sketch.ino.hex"), fakeSketchHex) NoError(t, err) @@ -80,8 +108,8 @@ func TestMergeSketchWithBootloader(t *testing.T) { NoError(t, err) mergedSketchHex := string(bytes) - require.True(t, strings.HasPrefix(mergedSketchHex, "row 1\n:107E0000112484B714BE81FFF0D085E080938100F7\n")) - require.True(t, strings.HasSuffix(mergedSketchHex, ":0400000300007E007B\n:00000001FF\n")) + require.True(t, strings.HasPrefix(mergedSketchHex, ":100000000C9434000C9446000C9446000C9446006A\n")) + require.True(t, strings.HasSuffix(mergedSketchHex, ":00000001FF\n")) } func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) { @@ -103,8 +131,35 @@ func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) { err := utils.EnsureFolderExists(filepath.Join(buildPath, "sketch")) NoError(t, err) - fakeSketchHex := "row 1\n" + - "row 2\n" + fakeSketchHex := ":100000000C9434000C9446000C9446000C9446006A\n" + + ":100010000C9446000C9446000C9446000C94460048\n" + + ":100020000C9446000C9446000C9446000C94460038\n" + + ":100030000C9446000C9446000C9446000C94460028\n" + + ":100040000C9448000C9446000C9446000C94460016\n" + + ":100050000C9446000C9446000C9446000C94460008\n" + + ":100060000C9446000C94460011241FBECFEFD8E03C\n" + + ":10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC\n" + + ":10008000B207E1F70E9492000C94DC000C9400008F\n" + + ":100090001F920F920FB60F9211242F933F938F93BD\n" + + ":1000A0009F93AF93BF938091050190910601A0911A\n" + + ":1000B0000701B09108013091040123E0230F2D378F\n" + + ":1000C00020F40196A11DB11D05C026E8230F02965C\n" + + ":1000D000A11DB11D20930401809305019093060199\n" + + ":1000E000A0930701B0930801809100019091010154\n" + + ":1000F000A0910201B09103010196A11DB11D809351\n" + + ":10010000000190930101A0930201B0930301BF91FC\n" + + ":10011000AF919F918F913F912F910F900FBE0F90B4\n" + + ":100120001F901895789484B5826084BD84B58160F1\n" + + ":1001300084BD85B5826085BD85B5816085BD8091B2\n" + + ":100140006E00816080936E0010928100809181002A\n" + + ":100150008260809381008091810081608093810022\n" + + ":10016000809180008160809380008091B1008460E4\n" + + ":100170008093B1008091B00081608093B000809145\n" + + ":100180007A00846080937A0080917A008260809304\n" + + ":100190007A0080917A00816080937A0080917A0061\n" + + ":1001A000806880937A001092C100C0E0D0E0209770\n" + + ":0C01B000F1F30E940000FBCFF894FFCF99\n" + + ":00000001FF\n" err = utils.WriteFile(filepath.Join(buildPath, "sketch.ino.hex"), fakeSketchHex) NoError(t, err) @@ -122,8 +177,8 @@ func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) { NoError(t, err) mergedSketchHex := string(bytes) - require.True(t, strings.HasPrefix(mergedSketchHex, "row 1\n:107E0000112484B714BE81FFF0D085E080938100F7\n")) - require.True(t, strings.HasSuffix(mergedSketchHex, ":0400000300007E007B\n:00000001FF\n")) + require.True(t, strings.HasPrefix(mergedSketchHex, ":100000000C9434000C9446000C9446000C9446006A\n")) + require.True(t, strings.HasSuffix(mergedSketchHex, ":00000001FF\n")) } func TestMergeSketchWithBootloaderWhenNoBootloaderAvailable(t *testing.T) { @@ -183,8 +238,35 @@ func TestMergeSketchWithBootloaderPathIsParameterized(t *testing.T) { err := utils.EnsureFolderExists(filepath.Join(buildPath, "sketch")) NoError(t, err) - fakeSketchHex := "row 1\n" + - "row 2\n" + fakeSketchHex := ":100000000C9434000C9446000C9446000C9446006A\n" + + ":100010000C9446000C9446000C9446000C94460048\n" + + ":100020000C9446000C9446000C9446000C94460038\n" + + ":100030000C9446000C9446000C9446000C94460028\n" + + ":100040000C9448000C9446000C9446000C94460016\n" + + ":100050000C9446000C9446000C9446000C94460008\n" + + ":100060000C9446000C94460011241FBECFEFD8E03C\n" + + ":10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC\n" + + ":10008000B207E1F70E9492000C94DC000C9400008F\n" + + ":100090001F920F920FB60F9211242F933F938F93BD\n" + + ":1000A0009F93AF93BF938091050190910601A0911A\n" + + ":1000B0000701B09108013091040123E0230F2D378F\n" + + ":1000C00020F40196A11DB11D05C026E8230F02965C\n" + + ":1000D000A11DB11D20930401809305019093060199\n" + + ":1000E000A0930701B0930801809100019091010154\n" + + ":1000F000A0910201B09103010196A11DB11D809351\n" + + ":10010000000190930101A0930201B0930301BF91FC\n" + + ":10011000AF919F918F913F912F910F900FBE0F90B4\n" + + ":100120001F901895789484B5826084BD84B58160F1\n" + + ":1001300084BD85B5826085BD85B5816085BD8091B2\n" + + ":100140006E00816080936E0010928100809181002A\n" + + ":100150008260809381008091810081608093810022\n" + + ":10016000809180008160809380008091B1008460E4\n" + + ":100170008093B1008091B00081608093B000809145\n" + + ":100180007A00846080937A0080917A008260809304\n" + + ":100190007A0080917A00816080937A0080917A0061\n" + + ":1001A000806880937A001092C100C0E0D0E0209770\n" + + ":0C01B000F1F30E940000FBCFF894FFCF99\n" + + ":00000001FF\n" err = utils.WriteFile(filepath.Join(buildPath, "sketch", "sketch.ino.hex"), fakeSketchHex) NoError(t, err) @@ -202,6 +284,6 @@ func TestMergeSketchWithBootloaderPathIsParameterized(t *testing.T) { NoError(t, err) mergedSketchHex := string(bytes) - require.True(t, strings.HasPrefix(mergedSketchHex, "row 1\n:020000023000CC")) - require.True(t, strings.HasSuffix(mergedSketchHex, ":040000033000E000E9\n:00000001FF\n")) + require.True(t, strings.HasPrefix(mergedSketchHex, ":100000000C9434000C9446000C9446000C9446006A\n")) + require.True(t, strings.HasSuffix(mergedSketchHex, ":00000001FF\n")) } From d6536f4bd81885dcb5cbabcf75db8ad64849d28d Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Thu, 20 Sep 2018 16:36:50 +0200 Subject: [PATCH 4/5] Travis: add dependency from gohex --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 910c086e..665f1418 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ install: - go get github.com/wadey/gocovmerge - go get github.com/arduino/go-properties-map - go get github.com/arduino/go-timeutils + - go get github.com/marcinbor85/gohex script: - go get github.com/arduino/arduino-builder/arduino-builder From d158e152bbe69fc8d5192b49af809b6de2aadce5 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Thu, 7 Mar 2019 10:59:40 +0100 Subject: [PATCH 5/5] Limit _with_bootloader bin file to 16MB or 2*max_upload_size --- merge_sketch_with_bootloader.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/merge_sketch_with_bootloader.go b/merge_sketch_with_bootloader.go index fb42c43a..f27ac426 100644 --- a/merge_sketch_with_bootloader.go +++ b/merge_sketch_with_bootloader.go @@ -35,6 +35,7 @@ import ( "os" "path/filepath" "strings" + "strconv" "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/types" @@ -84,7 +85,12 @@ func (s *MergeSketchWithBootloader) Run(ctx *types.Context) error { mergedSketchPath := filepath.Join(filepath.Dir(builtSketchPath), sketchFileName+".with_bootloader.hex") // Ignore merger errors for the first iteration - err := merge(builtSketchPath, bootloaderPath, mergedSketchPath) + maximumBinSize := 16000000 + if uploadMaxSize, ok := ctx.BuildProperties[constants.PROPERTY_UPLOAD_MAX_SIZE]; ok { + maximumBinSize, _ = strconv.Atoi(uploadMaxSize) + maximumBinSize *= 2 + } + err := merge(builtSketchPath, bootloaderPath, mergedSketchPath, maximumBinSize) if err != nil { logger.Fprintln(os.Stdout, constants.LOG_LEVEL_WARN, err.Error()) } @@ -92,7 +98,7 @@ func (s *MergeSketchWithBootloader) Run(ctx *types.Context) error { return nil } -func merge(builtSketchPath, bootloaderPath, mergedSketchPath string) error { +func merge(builtSketchPath, bootloaderPath, mergedSketchPath string, maximumBinSize int) error { if filepath.Ext(bootloaderPath) == ".bin" { bootloaderPath = strings.TrimSuffix(bootloaderPath, ".bin") + ".hex" @@ -162,6 +168,11 @@ func merge(builtSketchPath, bootloaderPath, mergedSketchPath string) error { mergedSketchPathBin := strings.TrimSuffix(mergedSketchPath, ".hex") + ".bin" + size := last_address - initial_address + if (size > uint32(maximumBinSize)) { + return nil + } + bytes := mem_merge.ToBinary(initial_address, last_address-initial_address, 0xFF) return utils.WriteFile(mergedSketchPathBin, string(bytes)) }