From 7112bc2a5ae600081ffa3aba4f61f06d0cfd3063 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Thu, 31 Jan 2019 17:11:31 +0100 Subject: [PATCH 01/26] Change the directory of the configuration files If you install the arduino cli, the default directory of the configuration file is the location of the executable This is not an ideal configuration for linux or windows With this commit the configuration file is searched in the default configuration folder for each system --- Gopkg.lock | 9 + README.md | 2 +- commands/config/init.go | 9 +- commands/root/root.go | 2 +- configs/directories.go | 24 ++- vendor/github.com/shibukawa/configdir/LICENSE | 21 +++ .../github.com/shibukawa/configdir/README.rst | 111 ++++++++++++ .../github.com/shibukawa/configdir/config.go | 160 ++++++++++++++++++ .../shibukawa/configdir/config_darwin.go | 8 + .../shibukawa/configdir/config_windows.go | 8 + .../shibukawa/configdir/config_xdg.go | 34 ++++ 11 files changed, 377 insertions(+), 11 deletions(-) create mode 100644 vendor/github.com/shibukawa/configdir/LICENSE create mode 100644 vendor/github.com/shibukawa/configdir/README.rst create mode 100644 vendor/github.com/shibukawa/configdir/config.go create mode 100644 vendor/github.com/shibukawa/configdir/config_darwin.go create mode 100644 vendor/github.com/shibukawa/configdir/config_windows.go create mode 100644 vendor/github.com/shibukawa/configdir/config_xdg.go diff --git a/Gopkg.lock b/Gopkg.lock index f32b87a7702..4580a45936f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -266,6 +266,14 @@ revision = "55d61fa8aa702f59229e6cff85793c22e580eaf5" version = "v1.5.1" +[[projects]] + branch = "master" + digest = "1:8209ed8bf2336848aa760c11e809f93ba96fffa64d3c2948e970a362a46e534b" + name = "github.com/shibukawa/configdir" + packages = ["."] + pruneopts = "UT" + revision = "e180dbdc8da04c4fa04272e875ce64949f38bd3e" + [[projects]] digest = "1:9e9193aa51197513b3abcb108970d831fbcf40ef96aa845c4f03276e1fa316d2" name = "github.com/sirupsen/logrus" @@ -441,6 +449,7 @@ "github.com/mitchellh/go-homedir", "github.com/pkg/errors", "github.com/pmylund/sortutil", + "github.com/shibukawa/configdir", "github.com/sirupsen/logrus", "github.com/spf13/cobra", "github.com/spf13/cobra/doc", diff --git a/README.md b/README.md index 59356a7ac67..ebfa59f0e5b 100644 --- a/README.md +++ b/README.md @@ -295,7 +295,7 @@ Flags: -h, --help help for core Global Flags: - --config-file string The custom config file (if not specified ./.cli-config.yml will be used). (default "/home/megabug/Workspace/go/src/github.com/arduino/arduino-cli/.cli-config.yml") + --config-file string The custom config file (if not specified the default one will be used). (example "/home/megabug/.config/arduino/arduino-cli/.cli-config.yml") --debug Enables debug output (super verbose, used to debug the CLI). --format string The output format, can be [text|json]. (default "text") diff --git a/commands/config/init.go b/commands/config/init.go index 1b1774fcb67..914ee57b081 100644 --- a/commands/config/init.go +++ b/commands/config/init.go @@ -65,7 +65,14 @@ func runInitCommand(cmd *cobra.Command, args []string) { if filepath == "" { filepath = commands.Config.ConfigFile.String() } - err := commands.Config.SaveToYAML(filepath) + + err := os.MkdirAll(commands.Config.ConfigFile.Parent().String(), 0766) + if err != nil { + formatter.PrintError(err, "Cannot create config file.") + os.Exit(commands.ErrGeneric) + } + + err = commands.Config.SaveToYAML(filepath) if err != nil { formatter.PrintError(err, "Cannot create config file.") os.Exit(commands.ErrGeneric) diff --git a/commands/root/root.go b/commands/root/root.go index 779acddb7b1..640238917f8 100644 --- a/commands/root/root.go +++ b/commands/root/root.go @@ -56,7 +56,7 @@ func Init() *cobra.Command { } command.PersistentFlags().BoolVar(&commands.GlobalFlags.Debug, "debug", false, "Enables debug output (super verbose, used to debug the CLI).") command.PersistentFlags().StringVar(&commands.GlobalFlags.Format, "format", "text", "The output format, can be [text|json].") - command.PersistentFlags().StringVar(&yamlConfigFile, "config-file", "", "The custom config file (if not specified ./.cli-config.yml will be used).") + command.PersistentFlags().StringVar(&yamlConfigFile, "config-file", "", "The custom config file (if not specified the default will be used).") command.AddCommand(board.InitCommand()) command.AddCommand(compile.InitCommand()) command.AddCommand(config.InitCommand()) diff --git a/configs/directories.go b/configs/directories.go index 6c187c6c88b..43e295e17da 100644 --- a/configs/directories.go +++ b/configs/directories.go @@ -19,23 +19,31 @@ package configs import ( "fmt" - "os" "os/user" "runtime" "github.com/arduino/go-paths-helper" - "github.com/arduino/go-win32-utils" + "github.com/shibukawa/configdir" ) -// getDefaultConfigFilePath returns the default path for .cli-config.yml, -// this is the directory where the arduino-cli executable resides. +// getDefaultConfigFilePath returns the default path for .cli-config.yml. It searches the following directories for an existing .cli-config.yml file: +// - User level configuration folder(e.g. $HOME/.config///setting.json in Linux) +// - System level configuration folder(e.g. /etc/xdg///setting.json in Linux) +// If it doesn't find one, it defaults to the user level configuration folder func getDefaultConfigFilePath() *paths.Path { - executablePath, err := os.Executable() - if err != nil { - executablePath = "." + configDirs := configdir.New("arduino", "arduino-cli") + + // Search for a suitable configuration file + path := configDirs.QueryFolderContainsFile(".cli-config.yml") + if path != nil { + return paths.New(path.Path, ".cli-config.yml") } - return paths.New(executablePath).Parent().Join(".cli-config.yml") + // Default to the global configuration + locals := configDirs.QueryFolders(configdir.Global) + return paths.New(locals[0].Path, ".cli-config.yml") + + return nil } func getDefaultArduinoDataDir() (*paths.Path, error) { diff --git a/vendor/github.com/shibukawa/configdir/LICENSE b/vendor/github.com/shibukawa/configdir/LICENSE new file mode 100644 index 00000000000..b20af456a19 --- /dev/null +++ b/vendor/github.com/shibukawa/configdir/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 shibukawa + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/shibukawa/configdir/README.rst b/vendor/github.com/shibukawa/configdir/README.rst new file mode 100644 index 00000000000..99906697da3 --- /dev/null +++ b/vendor/github.com/shibukawa/configdir/README.rst @@ -0,0 +1,111 @@ +configdir for Golang +===================== + +Multi platform library of configuration directory for Golang. + +This library helps to get regular directories for configuration files or cache files that matches target operationg system's convention. + +It assumes the following folders are standard paths of each environment: + +.. list-table:: + :header-rows: 1 + + - * + * Windows: + * Linux/BSDs: + * MacOSX: + - * System level configuration folder + * ``%PROGRAMDATA%`` (``C:\\ProgramData``) + * ``${XDG_CONFIG_DIRS}`` (``/etc/xdg``) + * ``/Library/Application Support`` + - * User level configuration folder + * ``%APPDATA%`` (``C:\\Users\\\\AppData\\Roaming``) + * ``${XDG_CONFIG_HOME}`` (``${HOME}/.config``) + * ``${HOME}/Library/Application Support`` + - * User wide cache folder + * ``%LOCALAPPDATA%`` ``(C:\\Users\\\\AppData\\Local)`` + * ``${XDG_CACHE_HOME}`` (``${HOME}/.cache``) + * ``${HOME}/Library/Caches`` + +Examples +------------ + +Getting Configuration +~~~~~~~~~~~~~~~~~~~~~~~~ + +``configdir.ConfigDir.QueryFolderContainsFile()`` searches files in the following order: + +* Local path (if you add the path via LocalPath parameter) +* User level configuration folder(e.g. ``$HOME/.config///setting.json`` in Linux) +* System level configuration folder(e.g. ``/etc/xdg///setting.json`` in Linux) + +``configdir.Config`` provides some convenient methods(``ReadFile``, ``WriteFile`` and so on). + +.. code-block:: go + + var config Config + + configDirs := configdir.New("vendor-name", "application-name") + // optional: local path has the highest priority + configDirs.LocalPath, _ = filepath.Abs(".") + folder := configDirs.QueryFolderContainsFile("setting.json") + if folder != nil { + data, _ := folder.ReadFile("setting.json") + json.Unmarshal(data, &config) + } else { + config = DefaultConfig + } + +Write Configuration +~~~~~~~~~~~~~~~~~~~~~~ + +When storing configuration, get configuration folder by using ``configdir.ConfigDir.QueryFolders()`` method. + +.. code-block:: go + + configDirs := configdir.New("vendor-name", "application-name") + + var config Config + data, _ := json.Marshal(&config) + + // Stores to local folder + folders := configDirs.QueryFolders(configdir.Local) + folders[0].WriteFile("setting.json", data) + + // Stores to user folder + folders = configDirs.QueryFolders(configdir.Global) + folders[0].WriteFile("setting.json", data) + + // Stores to system folder + folders = configDirs.QueryFolders(configdir.System) + folders[0].WriteFile("setting.json", data) + +Getting Cache Folder +~~~~~~~~~~~~~~~~~~~~~~ + +It is similar to the above example, but returns cache folder. + +.. code-block:: go + + configDirs := configdir.New("vendor-name", "application-name") + cache := configDirs.QueryCacheFolder() + + resp, err := http.Get("http://examples.com/sdk.zip") + if err != nil { + log.Fatal(err) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + + cache.WriteFile("sdk.zip", body) + +Document +------------ + +https://godoc.org/github.com/shibukawa/configdir + +License +------------ + +MIT + diff --git a/vendor/github.com/shibukawa/configdir/config.go b/vendor/github.com/shibukawa/configdir/config.go new file mode 100644 index 00000000000..8a20e54b59a --- /dev/null +++ b/vendor/github.com/shibukawa/configdir/config.go @@ -0,0 +1,160 @@ +// configdir provides access to configuration folder in each platforms. +// +// System wide configuration folders: +// +// - Windows: %PROGRAMDATA% (C:\ProgramData) +// - Linux/BSDs: ${XDG_CONFIG_DIRS} (/etc/xdg) +// - MacOSX: "/Library/Application Support" +// +// User wide configuration folders: +// +// - Windows: %APPDATA% (C:\Users\\AppData\Roaming) +// - Linux/BSDs: ${XDG_CONFIG_HOME} (${HOME}/.config) +// - MacOSX: "${HOME}/Library/Application Support" +// +// User wide cache folders: +// +// - Windows: %LOCALAPPDATA% (C:\Users\\AppData\Local) +// - Linux/BSDs: ${XDG_CACHE_HOME} (${HOME}/.cache) +// - MacOSX: "${HOME}/Library/Caches" +// +// configdir returns paths inside the above folders. + +package configdir + +import ( + "io/ioutil" + "os" + "path/filepath" +) + +type ConfigType int + +const ( + System ConfigType = iota + Global + All + Existing + Local + Cache +) + +// Config represents each folder +type Config struct { + Path string + Type ConfigType +} + +func (c Config) Open(fileName string) (*os.File, error) { + return os.Open(filepath.Join(c.Path, fileName)) +} + +func (c Config) Create(fileName string) (*os.File, error) { + err := c.CreateParentDir(fileName) + if err != nil { + return nil, err + } + return os.Create(filepath.Join(c.Path, fileName)) +} + +func (c Config) ReadFile(fileName string) ([]byte, error) { + return ioutil.ReadFile(filepath.Join(c.Path, fileName)) +} + +// CreateParentDir creates the parent directory of fileName inside c. fileName +// is a relative path inside c, containing zero or more path separators. +func (c Config) CreateParentDir(fileName string) error { + return os.MkdirAll(filepath.Dir(filepath.Join(c.Path, fileName)), 0755) +} + +func (c Config) WriteFile(fileName string, data []byte) error { + err := c.CreateParentDir(fileName) + if err != nil { + return err + } + return ioutil.WriteFile(filepath.Join(c.Path, fileName), data, 0644) +} + +func (c Config) MkdirAll() error { + return os.MkdirAll(c.Path, 0755) +} + +func (c Config) Exists(fileName string) bool { + _, err := os.Stat(filepath.Join(c.Path, fileName)) + return !os.IsNotExist(err) +} + +// ConfigDir keeps setting for querying folders. +type ConfigDir struct { + VendorName string + ApplicationName string + LocalPath string +} + +func New(vendorName, applicationName string) ConfigDir { + return ConfigDir{ + VendorName: vendorName, + ApplicationName: applicationName, + } +} + +func (c ConfigDir) joinPath(root string) string { + if c.VendorName != "" && hasVendorName { + return filepath.Join(root, c.VendorName, c.ApplicationName) + } + return filepath.Join(root, c.ApplicationName) +} + +func (c ConfigDir) QueryFolders(configType ConfigType) []*Config { + if configType == Cache { + return []*Config{c.QueryCacheFolder()} + } + var result []*Config + if c.LocalPath != "" && configType != System && configType != Global { + result = append(result, &Config{ + Path: c.LocalPath, + Type: Local, + }) + } + if configType != System && configType != Local { + result = append(result, &Config{ + Path: c.joinPath(globalSettingFolder), + Type: Global, + }) + } + if configType != Global && configType != Local { + for _, root := range systemSettingFolders { + result = append(result, &Config{ + Path: c.joinPath(root), + Type: System, + }) + } + } + if configType != Existing { + return result + } + var existing []*Config + for _, entry := range result { + if _, err := os.Stat(entry.Path); !os.IsNotExist(err) { + existing = append(existing, entry) + } + } + return existing +} + +func (c ConfigDir) QueryFolderContainsFile(fileName string) *Config { + configs := c.QueryFolders(Existing) + for _, config := range configs { + if _, err := os.Stat(filepath.Join(config.Path, fileName)); !os.IsNotExist(err) { + return config + } + } + return nil +} + +func (c ConfigDir) QueryCacheFolder() *Config { + return &Config{ + Path: c.joinPath(cacheFolder), + Type: Cache, + } +} diff --git a/vendor/github.com/shibukawa/configdir/config_darwin.go b/vendor/github.com/shibukawa/configdir/config_darwin.go new file mode 100644 index 00000000000..d668507a7e3 --- /dev/null +++ b/vendor/github.com/shibukawa/configdir/config_darwin.go @@ -0,0 +1,8 @@ +package configdir + +import "os" + +var hasVendorName = true +var systemSettingFolders = []string{"/Library/Application Support"} +var globalSettingFolder = os.Getenv("HOME") + "/Library/Application Support" +var cacheFolder = os.Getenv("HOME") + "/Library/Caches" diff --git a/vendor/github.com/shibukawa/configdir/config_windows.go b/vendor/github.com/shibukawa/configdir/config_windows.go new file mode 100644 index 00000000000..0984821778d --- /dev/null +++ b/vendor/github.com/shibukawa/configdir/config_windows.go @@ -0,0 +1,8 @@ +package configdir + +import "os" + +var hasVendorName = true +var systemSettingFolders = []string{os.Getenv("PROGRAMDATA")} +var globalSettingFolder = os.Getenv("APPDATA") +var cacheFolder = os.Getenv("LOCALAPPDATA") diff --git a/vendor/github.com/shibukawa/configdir/config_xdg.go b/vendor/github.com/shibukawa/configdir/config_xdg.go new file mode 100644 index 00000000000..026ca68a0bb --- /dev/null +++ b/vendor/github.com/shibukawa/configdir/config_xdg.go @@ -0,0 +1,34 @@ +// +build !windows,!darwin + +package configdir + +import ( + "os" + "path/filepath" + "strings" +) + +// https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + +var hasVendorName = true +var systemSettingFolders []string +var globalSettingFolder string +var cacheFolder string + +func init() { + if os.Getenv("XDG_CONFIG_HOME") != "" { + globalSettingFolder = os.Getenv("XDG_CONFIG_HOME") + } else { + globalSettingFolder = filepath.Join(os.Getenv("HOME"), ".config") + } + if os.Getenv("XDG_CONFIG_DIRS") != "" { + systemSettingFolders = strings.Split(os.Getenv("XDG_CONFIG_DIRS"), ":") + } else { + systemSettingFolders = []string{"/etc/xdg"} + } + if os.Getenv("XDG_CACHE_HOME") != "" { + cacheFolder = os.Getenv("XDG_CACHE_HOME") + } else { + cacheFolder = filepath.Join(os.Getenv("HOME"), ".cache") + } +} From f5d7eae969d4e01bd0fb5ec2b17e1013f2d65b28 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Fri, 1 Feb 2019 10:55:37 +0100 Subject: [PATCH 02/26] Stub new function configs.Navigate This function will navigate the filesystem scraping the fs searching for config files. It will search in $pwd navigating through parent folders stopping at $root --- configs/navigate.go | 5 +++++ configs/navigate_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 configs/navigate.go create mode 100644 configs/navigate_test.go diff --git a/configs/navigate.go b/configs/navigate.go new file mode 100644 index 00000000000..128af0f8859 --- /dev/null +++ b/configs/navigate.go @@ -0,0 +1,5 @@ +package configs + +func Navigate(root, pwd string) Configuration { + return Configuration{} +} diff --git a/configs/navigate_test.go b/configs/navigate_test.go new file mode 100644 index 00000000000..9a61756ab2c --- /dev/null +++ b/configs/navigate_test.go @@ -0,0 +1,27 @@ +package configs_test + +import ( + "reflect" + "testing" +) + +func TestNavigate(t *testing.T) { + type args struct { + root string + pwd string + } + tests := []struct { + name string + args args + want Configuration + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Navigate(tt.args.root, tt.args.pwd); !reflect.DeepEqual(got, tt.want) { + t.Errorf("Navigate() = %v, want %v", got, tt.want) + } + }) + } +} From 4a73a8e3121665ca6e7e0ee4224b8ccea9703544 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Mon, 11 Feb 2019 16:04:23 +0100 Subject: [PATCH 03/26] Navigate returns the default configuration --- configs/navigate.go | 15 +++++- configs/navigate_test.go | 54 +++++++++++++------ .../navigate/noconfig/first/second/.gitkeep | 0 .../testdata/navigate/noconfig/golden.yaml | 4 ++ 4 files changed, 57 insertions(+), 16 deletions(-) create mode 100644 configs/testdata/navigate/noconfig/first/second/.gitkeep create mode 100644 configs/testdata/navigate/noconfig/golden.yaml diff --git a/configs/navigate.go b/configs/navigate.go index 128af0f8859..2f9181b7acf 100644 --- a/configs/navigate.go +++ b/configs/navigate.go @@ -1,5 +1,18 @@ package configs +import ( + paths "github.com/arduino/go-paths-helper" + homedir "github.com/mitchellh/go-homedir" +) + func Navigate(root, pwd string) Configuration { - return Configuration{} + home, err := homedir.Dir() + if err != nil { + panic(err) // Should never happen + } + + return Configuration{ + SketchbookDir: paths.New(home, "Arduino"), + DataDir: paths.New(home, ".arduino15"), + } } diff --git a/configs/navigate_test.go b/configs/navigate_test.go index 9a61756ab2c..c7e82023444 100644 --- a/configs/navigate_test.go +++ b/configs/navigate_test.go @@ -1,27 +1,51 @@ package configs_test import ( - "reflect" + "io/ioutil" + "path/filepath" + "strings" "testing" + + "github.com/arduino/arduino-cli/configs" + homedir "github.com/mitchellh/go-homedir" + "github.com/sergi/go-diff/diffmatchpatch" ) func TestNavigate(t *testing.T) { - type args struct { - root string - pwd string - } - tests := []struct { - name string - args args - want Configuration - }{ - // TODO: Add test cases. + tests := []string{ + "noconfig", } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := Navigate(tt.args.root, tt.args.pwd); !reflect.DeepEqual(got, tt.want) { - t.Errorf("Navigate() = %v, want %v", got, tt.want) - } + t.Run(tt, func(t *testing.T) { + root := filepath.Join("testdata", "navigate", tt) + pwd := filepath.Join("testdata", "navigate", tt, "first", "second") + golden := filepath.Join("testdata", "navigate", tt, "golden.yaml") + + got := configs.Navigate(root, pwd) + data, _ := got.SerializeToYAML() + + diff(t, data, golden) }) } } + +func diff(t *testing.T, data []byte, goldenFile string) { + golden, err := ioutil.ReadFile(goldenFile) + if err != nil { + t.Error(err) + return + } + + dataStr := strings.TrimSpace(string(data)) + goldenStr := strings.TrimSpace(string(golden)) + + // Substitute home folder + homedir, _ := homedir.Dir() + dataStr = strings.Replace(dataStr, homedir, "$HOME", -1) + + if dataStr != goldenStr { + dmp := diffmatchpatch.New() + diffs := dmp.DiffMain(goldenStr, dataStr, false) + t.Errorf(dmp.DiffPrettyText(diffs)) + } +} diff --git a/configs/testdata/navigate/noconfig/first/second/.gitkeep b/configs/testdata/navigate/noconfig/first/second/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/configs/testdata/navigate/noconfig/golden.yaml b/configs/testdata/navigate/noconfig/golden.yaml new file mode 100644 index 00000000000..bad5a4031c4 --- /dev/null +++ b/configs/testdata/navigate/noconfig/golden.yaml @@ -0,0 +1,4 @@ +proxy_type: "" +sketchbook_path: $HOME/Arduino +arduino_data: $HOME/.arduino15 +board_manager: null \ No newline at end of file From 74f9f6a741f17d5a35e28e3765e648393eee5ce0 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Mon, 11 Feb 2019 17:05:06 +0100 Subject: [PATCH 04/26] Search for config file in local directory --- configs/navigate.go | 11 ++++++++++- configs/navigate_test.go | 1 + configs/testdata/navigate/local/first/second/.gitkeep | 0 .../navigate/local/first/second/arduino-cli.yaml | 4 ++++ configs/testdata/navigate/local/golden.yaml | 6 ++++++ configs/yaml_serializer.go | 10 +++------- 6 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 configs/testdata/navigate/local/first/second/.gitkeep create mode 100644 configs/testdata/navigate/local/first/second/arduino-cli.yaml create mode 100644 configs/testdata/navigate/local/golden.yaml diff --git a/configs/navigate.go b/configs/navigate.go index 2f9181b7acf..b61c1c846dc 100644 --- a/configs/navigate.go +++ b/configs/navigate.go @@ -1,18 +1,27 @@ package configs import ( + "fmt" + paths "github.com/arduino/go-paths-helper" homedir "github.com/mitchellh/go-homedir" ) func Navigate(root, pwd string) Configuration { + fmt.Println("Navigate", root, pwd) home, err := homedir.Dir() if err != nil { panic(err) // Should never happen } - return Configuration{ + // Default configuration + config := Configuration{ SketchbookDir: paths.New(home, "Arduino"), DataDir: paths.New(home, ".arduino15"), } + + // Search for arduino-cli.yaml in current folder + _ = config.LoadFromYAML(paths.New(pwd, "arduino-cli.yaml")) + + return config } diff --git a/configs/navigate_test.go b/configs/navigate_test.go index c7e82023444..925ae434fe6 100644 --- a/configs/navigate_test.go +++ b/configs/navigate_test.go @@ -14,6 +14,7 @@ import ( func TestNavigate(t *testing.T) { tests := []string{ "noconfig", + "local", } for _, tt := range tests { t.Run(tt, func(t *testing.T) { diff --git a/configs/testdata/navigate/local/first/second/.gitkeep b/configs/testdata/navigate/local/first/second/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/configs/testdata/navigate/local/first/second/arduino-cli.yaml b/configs/testdata/navigate/local/first/second/arduino-cli.yaml new file mode 100644 index 00000000000..d122d502b1f --- /dev/null +++ b/configs/testdata/navigate/local/first/second/arduino-cli.yaml @@ -0,0 +1,4 @@ +board_manager: + additional_urls: + - https://downloads.arduino.cc/package_index_mraa.json + \ No newline at end of file diff --git a/configs/testdata/navigate/local/golden.yaml b/configs/testdata/navigate/local/golden.yaml new file mode 100644 index 00000000000..de1412d6620 --- /dev/null +++ b/configs/testdata/navigate/local/golden.yaml @@ -0,0 +1,6 @@ +proxy_type: "" +sketchbook_path: $HOME/Arduino +arduino_data: $HOME/.arduino15 +board_manager: + additional_urls: + - https://downloads.arduino.cc/package_index_mraa.json \ No newline at end of file diff --git a/configs/yaml_serializer.go b/configs/yaml_serializer.go index d5897b5ee8d..1428f283853 100644 --- a/configs/yaml_serializer.go +++ b/configs/yaml_serializer.go @@ -23,7 +23,6 @@ import ( "net/url" paths "github.com/arduino/go-paths-helper" - "github.com/sirupsen/logrus" yaml "gopkg.in/yaml.v2" ) @@ -48,16 +47,13 @@ type yamlProxyConfig struct { // LoadFromYAML loads the configs from a yaml file. func (config *Configuration) LoadFromYAML(path *paths.Path) error { - logrus.Info("Unserializing configurations from ", path) content, err := path.ReadFile() if err != nil { - logrus.WithError(err).Warn("Error reading config, using default configuration") return err } var ret yamlConfig err = yaml.Unmarshal(content, &ret) if err != nil { - logrus.WithError(err).Warn("Error parsing config, using default configuration") return err } @@ -84,12 +80,12 @@ func (config *Configuration) LoadFromYAML(path *paths.Path) error { for _, rawurl := range ret.BoardsManager.AdditionalURLS { url, err := url.Parse(rawurl) if err != nil { - logrus.WithError(err).Warn("Error parsing config") continue } config.BoardManagerAdditionalUrls = append(config.BoardManagerAdditionalUrls, url) } } + return nil } @@ -113,9 +109,9 @@ func (config *Configuration) SerializeToYAML() ([]byte, error) { Password: config.ProxyPassword, } } - if len(config.BoardManagerAdditionalUrls) > 1 { + if len(config.BoardManagerAdditionalUrls) > 0 { c.BoardsManager = &yamlBoardsManagerConfig{AdditionalURLS: []string{}} - for _, URL := range config.BoardManagerAdditionalUrls[1:] { + for _, URL := range config.BoardManagerAdditionalUrls { c.BoardsManager.AdditionalURLS = append(c.BoardsManager.AdditionalURLS, URL.String()) } } From f785e1ceefb54f998ed4dfa614c9252085689825 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Mon, 11 Feb 2019 17:35:20 +0100 Subject: [PATCH 05/26] Inherit configuration from parent folders It starts from the root and for every folder until $PWD it parses the yaml. Files closer to $PWD take precedence --- configs/navigate.go | 19 +++++++++++++++---- configs/navigate_test.go | 1 + .../inheritance/first/arduino-cli.yaml | 5 +++++ .../inheritance/first/second/.gitkeep | 0 .../inheritance/first/second/arduino-cli.yaml | 4 ++++ .../testdata/navigate/inheritance/golden.yaml | 6 ++++++ configs/yaml_serializer.go | 11 ++++++++++- 7 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 configs/testdata/navigate/inheritance/first/arduino-cli.yaml create mode 100644 configs/testdata/navigate/inheritance/first/second/.gitkeep create mode 100644 configs/testdata/navigate/inheritance/first/second/arduino-cli.yaml create mode 100644 configs/testdata/navigate/inheritance/golden.yaml diff --git a/configs/navigate.go b/configs/navigate.go index b61c1c846dc..977cef48bad 100644 --- a/configs/navigate.go +++ b/configs/navigate.go @@ -1,14 +1,19 @@ package configs import ( - "fmt" + "path/filepath" + "strings" paths "github.com/arduino/go-paths-helper" homedir "github.com/mitchellh/go-homedir" ) func Navigate(root, pwd string) Configuration { - fmt.Println("Navigate", root, pwd) + relativePath, err := filepath.Rel(root, pwd) + if err != nil { + panic(err) + } + home, err := homedir.Dir() if err != nil { panic(err) // Should never happen @@ -20,8 +25,14 @@ func Navigate(root, pwd string) Configuration { DataDir: paths.New(home, ".arduino15"), } - // Search for arduino-cli.yaml in current folder - _ = config.LoadFromYAML(paths.New(pwd, "arduino-cli.yaml")) + // From the root to the current folder, search for arduino-cli.yaml files + parts := strings.Split(relativePath, string(filepath.Separator)) + for i := range parts { + path := paths.New(root) + path = path.Join(parts[:i+1]...) + path = path.Join("arduino-cli.yaml") + _ = config.LoadFromYAML(path) + } return config } diff --git a/configs/navigate_test.go b/configs/navigate_test.go index 925ae434fe6..360e4b2c82f 100644 --- a/configs/navigate_test.go +++ b/configs/navigate_test.go @@ -15,6 +15,7 @@ func TestNavigate(t *testing.T) { tests := []string{ "noconfig", "local", + "inheritance", } for _, tt := range tests { t.Run(tt, func(t *testing.T) { diff --git a/configs/testdata/navigate/inheritance/first/arduino-cli.yaml b/configs/testdata/navigate/inheritance/first/arduino-cli.yaml new file mode 100644 index 00000000000..82fd1a30bf6 --- /dev/null +++ b/configs/testdata/navigate/inheritance/first/arduino-cli.yaml @@ -0,0 +1,5 @@ +sketchbook_path: /tmp +board_manager: + additional_urls: + - https://downloads.arduino.cc/package_index_mraa.json + \ No newline at end of file diff --git a/configs/testdata/navigate/inheritance/first/second/.gitkeep b/configs/testdata/navigate/inheritance/first/second/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/configs/testdata/navigate/inheritance/first/second/arduino-cli.yaml b/configs/testdata/navigate/inheritance/first/second/arduino-cli.yaml new file mode 100644 index 00000000000..d122d502b1f --- /dev/null +++ b/configs/testdata/navigate/inheritance/first/second/arduino-cli.yaml @@ -0,0 +1,4 @@ +board_manager: + additional_urls: + - https://downloads.arduino.cc/package_index_mraa.json + \ No newline at end of file diff --git a/configs/testdata/navigate/inheritance/golden.yaml b/configs/testdata/navigate/inheritance/golden.yaml new file mode 100644 index 00000000000..d861c833309 --- /dev/null +++ b/configs/testdata/navigate/inheritance/golden.yaml @@ -0,0 +1,6 @@ +proxy_type: "" +sketchbook_path: /tmp +arduino_data: $HOME/.arduino15 +board_manager: + additional_urls: + - https://downloads.arduino.cc/package_index_mraa.json \ No newline at end of file diff --git a/configs/yaml_serializer.go b/configs/yaml_serializer.go index 1428f283853..c7ff2b9060f 100644 --- a/configs/yaml_serializer.go +++ b/configs/yaml_serializer.go @@ -112,7 +112,7 @@ func (config *Configuration) SerializeToYAML() ([]byte, error) { if len(config.BoardManagerAdditionalUrls) > 0 { c.BoardsManager = &yamlBoardsManagerConfig{AdditionalURLS: []string{}} for _, URL := range config.BoardManagerAdditionalUrls { - c.BoardsManager.AdditionalURLS = append(c.BoardsManager.AdditionalURLS, URL.String()) + c.BoardsManager.AdditionalURLS = appendIfMissing(c.BoardsManager.AdditionalURLS, URL.String()) } } return yaml.Marshal(c) @@ -130,3 +130,12 @@ func (config *Configuration) SaveToYAML(path string) error { } return nil } + +func appendIfMissing(slice []string, i string) []string { + for _, ele := range slice { + if ele == i { + return slice + } + } + return append(slice, i) +} From 471266220c6bd330d00e2dfa632d082333abc80d Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Mon, 11 Feb 2019 18:00:12 +0100 Subject: [PATCH 06/26] Make Navigate a method of Configuration This will make sure it can be chained with other ParseFromYaml --- configs/navigate.go | 18 ++---------------- configs/navigate_test.go | 6 ++++-- .../testdata/navigate/inheritance/golden.yaml | 3 ++- configs/testdata/navigate/local/golden.yaml | 3 ++- configs/testdata/navigate/noconfig/golden.yaml | 6 ++++-- 5 files changed, 14 insertions(+), 22 deletions(-) diff --git a/configs/navigate.go b/configs/navigate.go index 977cef48bad..734c7c1f8f6 100644 --- a/configs/navigate.go +++ b/configs/navigate.go @@ -5,34 +5,20 @@ import ( "strings" paths "github.com/arduino/go-paths-helper" - homedir "github.com/mitchellh/go-homedir" ) -func Navigate(root, pwd string) Configuration { +func (c *Configuration) Navigate(root, pwd string) { relativePath, err := filepath.Rel(root, pwd) if err != nil { panic(err) } - home, err := homedir.Dir() - if err != nil { - panic(err) // Should never happen - } - - // Default configuration - config := Configuration{ - SketchbookDir: paths.New(home, "Arduino"), - DataDir: paths.New(home, ".arduino15"), - } - // From the root to the current folder, search for arduino-cli.yaml files parts := strings.Split(relativePath, string(filepath.Separator)) for i := range parts { path := paths.New(root) path = path.Join(parts[:i+1]...) path = path.Join("arduino-cli.yaml") - _ = config.LoadFromYAML(path) + _ = c.LoadFromYAML(path) } - - return config } diff --git a/configs/navigate_test.go b/configs/navigate_test.go index 360e4b2c82f..e819307d8ef 100644 --- a/configs/navigate_test.go +++ b/configs/navigate_test.go @@ -23,8 +23,10 @@ func TestNavigate(t *testing.T) { pwd := filepath.Join("testdata", "navigate", tt, "first", "second") golden := filepath.Join("testdata", "navigate", tt, "golden.yaml") - got := configs.Navigate(root, pwd) - data, _ := got.SerializeToYAML() + config, _ := configs.NewConfiguration() + + config.Navigate(root, pwd) + data, _ := config.SerializeToYAML() diff(t, data, golden) }) diff --git a/configs/testdata/navigate/inheritance/golden.yaml b/configs/testdata/navigate/inheritance/golden.yaml index d861c833309..6aede490ac9 100644 --- a/configs/testdata/navigate/inheritance/golden.yaml +++ b/configs/testdata/navigate/inheritance/golden.yaml @@ -1,6 +1,7 @@ -proxy_type: "" +proxy_type: auto sketchbook_path: /tmp arduino_data: $HOME/.arduino15 board_manager: additional_urls: + - https://downloads.arduino.cc/packages/package_index.json - https://downloads.arduino.cc/package_index_mraa.json \ No newline at end of file diff --git a/configs/testdata/navigate/local/golden.yaml b/configs/testdata/navigate/local/golden.yaml index de1412d6620..0fc132bfd79 100644 --- a/configs/testdata/navigate/local/golden.yaml +++ b/configs/testdata/navigate/local/golden.yaml @@ -1,6 +1,7 @@ -proxy_type: "" +proxy_type: auto sketchbook_path: $HOME/Arduino arduino_data: $HOME/.arduino15 board_manager: additional_urls: + - https://downloads.arduino.cc/packages/package_index.json - https://downloads.arduino.cc/package_index_mraa.json \ No newline at end of file diff --git a/configs/testdata/navigate/noconfig/golden.yaml b/configs/testdata/navigate/noconfig/golden.yaml index bad5a4031c4..49d1c3ae825 100644 --- a/configs/testdata/navigate/noconfig/golden.yaml +++ b/configs/testdata/navigate/noconfig/golden.yaml @@ -1,4 +1,6 @@ -proxy_type: "" +proxy_type: auto sketchbook_path: $HOME/Arduino arduino_data: $HOME/.arduino15 -board_manager: null \ No newline at end of file +board_manager: + additional_urls: + - https://downloads.arduino.cc/packages/package_index.json \ No newline at end of file From da73a114d7a65020b9252031b0e8de8efe4011ef Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Mon, 11 Feb 2019 18:08:41 +0100 Subject: [PATCH 07/26] Use config.Navigate in command root --- commands/root/root.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/commands/root/root.go b/commands/root/root.go index 640238917f8..d7a574975b0 100644 --- a/commands/root/root.go +++ b/commands/root/root.go @@ -18,8 +18,10 @@ package root import ( + "fmt" "io/ioutil" "os" + "path/filepath" "github.com/arduino/arduino-cli/output" @@ -115,6 +117,7 @@ func preRun(cmd *cobra.Command, args []string) { // initConfigs initializes the configuration from the specified file. func initConfigs() { + // Start with default configuration if conf, err := configs.NewConfiguration(); err != nil { logrus.WithError(err).Error("Error creating default configuration") formatter.PrintError(err, "Error creating default configuration") @@ -123,14 +126,25 @@ func initConfigs() { commands.Config = conf } + // Navigate through folders + pwd, err := filepath.Abs(".") + if err != nil { + logrus.WithError(err).Warn("Did not manage to find current path") + } + + commands.Config.Navigate("/", pwd) + + fmt.Println(yamlConfigFile) + if yamlConfigFile != "" { commands.Config.ConfigFile = paths.New(yamlConfigFile) + if err := commands.Config.LoadFromYAML(commands.Config.ConfigFile); err != nil { + logrus.WithError(err).Warn("Did not manage to get config file, using default configuration") + } } logrus.Info("Initiating configuration") - if err := commands.Config.LoadFromYAML(commands.Config.ConfigFile); err != nil { - logrus.WithError(err).Warn("Did not manage to get config file, using default configuration") - } + if commands.Config.IsBundledInDesktopIDE() { logrus.Info("CLI is bundled into the IDE") err := commands.Config.LoadFromDesktopIDEPreferences() From 5e22ab2a36190f1ff3d5815fe775f60b6ab2ff49 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Mon, 11 Feb 2019 18:12:54 +0100 Subject: [PATCH 08/26] Forgot deps --- Gopkg.lock | 9 + vendor/github.com/sergi/go-diff/AUTHORS | 25 + vendor/github.com/sergi/go-diff/CONTRIBUTORS | 32 + vendor/github.com/sergi/go-diff/LICENSE | 20 + .../sergi/go-diff/diffmatchpatch/diff.go | 1344 +++++++++++++++++ .../go-diff/diffmatchpatch/diffmatchpatch.go | 46 + .../sergi/go-diff/diffmatchpatch/match.go | 160 ++ .../sergi/go-diff/diffmatchpatch/mathutil.go | 23 + .../sergi/go-diff/diffmatchpatch/patch.go | 556 +++++++ .../go-diff/diffmatchpatch/stringutil.go | 88 ++ 10 files changed, 2303 insertions(+) create mode 100644 vendor/github.com/sergi/go-diff/AUTHORS create mode 100644 vendor/github.com/sergi/go-diff/CONTRIBUTORS create mode 100644 vendor/github.com/sergi/go-diff/LICENSE create mode 100644 vendor/github.com/sergi/go-diff/diffmatchpatch/diff.go create mode 100644 vendor/github.com/sergi/go-diff/diffmatchpatch/diffmatchpatch.go create mode 100644 vendor/github.com/sergi/go-diff/diffmatchpatch/match.go create mode 100644 vendor/github.com/sergi/go-diff/diffmatchpatch/mathutil.go create mode 100644 vendor/github.com/sergi/go-diff/diffmatchpatch/patch.go create mode 100644 vendor/github.com/sergi/go-diff/diffmatchpatch/stringutil.go diff --git a/Gopkg.lock b/Gopkg.lock index 4580a45936f..6cf71918bbc 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -266,6 +266,14 @@ revision = "55d61fa8aa702f59229e6cff85793c22e580eaf5" version = "v1.5.1" +[[projects]] + digest = "1:d917313f309bda80d27274d53985bc65651f81a5b66b820749ac7f8ef061fd04" + name = "github.com/sergi/go-diff" + packages = ["diffmatchpatch"] + pruneopts = "UT" + revision = "1744e2970ca51c86172c8190fadad617561ed6e7" + version = "v1.0.0" + [[projects]] branch = "master" digest = "1:8209ed8bf2336848aa760c11e809f93ba96fffa64d3c2948e970a362a46e534b" @@ -449,6 +457,7 @@ "github.com/mitchellh/go-homedir", "github.com/pkg/errors", "github.com/pmylund/sortutil", + "github.com/sergi/go-diff/diffmatchpatch", "github.com/shibukawa/configdir", "github.com/sirupsen/logrus", "github.com/spf13/cobra", diff --git a/vendor/github.com/sergi/go-diff/AUTHORS b/vendor/github.com/sergi/go-diff/AUTHORS new file mode 100644 index 00000000000..2d7bb2bf572 --- /dev/null +++ b/vendor/github.com/sergi/go-diff/AUTHORS @@ -0,0 +1,25 @@ +# This is the official list of go-diff authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as +# Name or Organization +# The email address is not required for organizations. + +# Please keep the list sorted. + +Danny Yoo +James Kolb +Jonathan Amsterdam +Markus Zimmermann +Matt Kovars +Örjan Persson +Osman Masood +Robert Carlsen +Rory Flynn +Sergi Mansilla +Shatrugna Sadhu +Shawn Smith +Stas Maksimov +Tor Arvid Lund +Zac Bergquist diff --git a/vendor/github.com/sergi/go-diff/CONTRIBUTORS b/vendor/github.com/sergi/go-diff/CONTRIBUTORS new file mode 100644 index 00000000000..369e3d55190 --- /dev/null +++ b/vendor/github.com/sergi/go-diff/CONTRIBUTORS @@ -0,0 +1,32 @@ +# This is the official list of people who can contribute +# (and typically have contributed) code to the go-diff +# repository. +# +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, ACME Inc. employees would be listed here +# but not in AUTHORS, because ACME Inc. would hold the copyright. +# +# When adding J Random Contributor's name to this file, +# either J's name or J's organization's name should be +# added to the AUTHORS file. +# +# Names should be added to this file like so: +# Name +# +# Please keep the list sorted. + +Danny Yoo +James Kolb +Jonathan Amsterdam +Markus Zimmermann +Matt Kovars +Örjan Persson +Osman Masood +Robert Carlsen +Rory Flynn +Sergi Mansilla +Shatrugna Sadhu +Shawn Smith +Stas Maksimov +Tor Arvid Lund +Zac Bergquist diff --git a/vendor/github.com/sergi/go-diff/LICENSE b/vendor/github.com/sergi/go-diff/LICENSE new file mode 100644 index 00000000000..937942c2b2c --- /dev/null +++ b/vendor/github.com/sergi/go-diff/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2012-2016 The go-diff Authors. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + diff --git a/vendor/github.com/sergi/go-diff/diffmatchpatch/diff.go b/vendor/github.com/sergi/go-diff/diffmatchpatch/diff.go new file mode 100644 index 00000000000..82ad7bc8f1c --- /dev/null +++ b/vendor/github.com/sergi/go-diff/diffmatchpatch/diff.go @@ -0,0 +1,1344 @@ +// Copyright (c) 2012-2016 The go-diff authors. All rights reserved. +// https://github.com/sergi/go-diff +// See the included LICENSE file for license details. +// +// go-diff is a Go implementation of Google's Diff, Match, and Patch library +// Original library is Copyright (c) 2006 Google Inc. +// http://code.google.com/p/google-diff-match-patch/ + +package diffmatchpatch + +import ( + "bytes" + "errors" + "fmt" + "html" + "math" + "net/url" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +// Operation defines the operation of a diff item. +type Operation int8 + +const ( + // DiffDelete item represents a delete diff. + DiffDelete Operation = -1 + // DiffInsert item represents an insert diff. + DiffInsert Operation = 1 + // DiffEqual item represents an equal diff. + DiffEqual Operation = 0 +) + +// Diff represents one diff operation +type Diff struct { + Type Operation + Text string +} + +func splice(slice []Diff, index int, amount int, elements ...Diff) []Diff { + return append(slice[:index], append(elements, slice[index+amount:]...)...) +} + +// DiffMain finds the differences between two texts. +// If an invalid UTF-8 sequence is encountered, it will be replaced by the Unicode replacement character. +func (dmp *DiffMatchPatch) DiffMain(text1, text2 string, checklines bool) []Diff { + return dmp.DiffMainRunes([]rune(text1), []rune(text2), checklines) +} + +// DiffMainRunes finds the differences between two rune sequences. +// If an invalid UTF-8 sequence is encountered, it will be replaced by the Unicode replacement character. +func (dmp *DiffMatchPatch) DiffMainRunes(text1, text2 []rune, checklines bool) []Diff { + var deadline time.Time + if dmp.DiffTimeout > 0 { + deadline = time.Now().Add(dmp.DiffTimeout) + } + return dmp.diffMainRunes(text1, text2, checklines, deadline) +} + +func (dmp *DiffMatchPatch) diffMainRunes(text1, text2 []rune, checklines bool, deadline time.Time) []Diff { + if runesEqual(text1, text2) { + var diffs []Diff + if len(text1) > 0 { + diffs = append(diffs, Diff{DiffEqual, string(text1)}) + } + return diffs + } + // Trim off common prefix (speedup). + commonlength := commonPrefixLength(text1, text2) + commonprefix := text1[:commonlength] + text1 = text1[commonlength:] + text2 = text2[commonlength:] + + // Trim off common suffix (speedup). + commonlength = commonSuffixLength(text1, text2) + commonsuffix := text1[len(text1)-commonlength:] + text1 = text1[:len(text1)-commonlength] + text2 = text2[:len(text2)-commonlength] + + // Compute the diff on the middle block. + diffs := dmp.diffCompute(text1, text2, checklines, deadline) + + // Restore the prefix and suffix. + if len(commonprefix) != 0 { + diffs = append([]Diff{Diff{DiffEqual, string(commonprefix)}}, diffs...) + } + if len(commonsuffix) != 0 { + diffs = append(diffs, Diff{DiffEqual, string(commonsuffix)}) + } + + return dmp.DiffCleanupMerge(diffs) +} + +// diffCompute finds the differences between two rune slices. Assumes that the texts do not have any common prefix or suffix. +func (dmp *DiffMatchPatch) diffCompute(text1, text2 []rune, checklines bool, deadline time.Time) []Diff { + diffs := []Diff{} + if len(text1) == 0 { + // Just add some text (speedup). + return append(diffs, Diff{DiffInsert, string(text2)}) + } else if len(text2) == 0 { + // Just delete some text (speedup). + return append(diffs, Diff{DiffDelete, string(text1)}) + } + + var longtext, shorttext []rune + if len(text1) > len(text2) { + longtext = text1 + shorttext = text2 + } else { + longtext = text2 + shorttext = text1 + } + + if i := runesIndex(longtext, shorttext); i != -1 { + op := DiffInsert + // Swap insertions for deletions if diff is reversed. + if len(text1) > len(text2) { + op = DiffDelete + } + // Shorter text is inside the longer text (speedup). + return []Diff{ + Diff{op, string(longtext[:i])}, + Diff{DiffEqual, string(shorttext)}, + Diff{op, string(longtext[i+len(shorttext):])}, + } + } else if len(shorttext) == 1 { + // Single character string. + // After the previous speedup, the character can't be an equality. + return []Diff{ + Diff{DiffDelete, string(text1)}, + Diff{DiffInsert, string(text2)}, + } + // Check to see if the problem can be split in two. + } else if hm := dmp.diffHalfMatch(text1, text2); hm != nil { + // A half-match was found, sort out the return data. + text1A := hm[0] + text1B := hm[1] + text2A := hm[2] + text2B := hm[3] + midCommon := hm[4] + // Send both pairs off for separate processing. + diffsA := dmp.diffMainRunes(text1A, text2A, checklines, deadline) + diffsB := dmp.diffMainRunes(text1B, text2B, checklines, deadline) + // Merge the results. + return append(diffsA, append([]Diff{Diff{DiffEqual, string(midCommon)}}, diffsB...)...) + } else if checklines && len(text1) > 100 && len(text2) > 100 { + return dmp.diffLineMode(text1, text2, deadline) + } + return dmp.diffBisect(text1, text2, deadline) +} + +// diffLineMode does a quick line-level diff on both []runes, then rediff the parts for greater accuracy. This speedup can produce non-minimal diffs. +func (dmp *DiffMatchPatch) diffLineMode(text1, text2 []rune, deadline time.Time) []Diff { + // Scan the text on a line-by-line basis first. + text1, text2, linearray := dmp.diffLinesToRunes(text1, text2) + + diffs := dmp.diffMainRunes(text1, text2, false, deadline) + + // Convert the diff back to original text. + diffs = dmp.DiffCharsToLines(diffs, linearray) + // Eliminate freak matches (e.g. blank lines) + diffs = dmp.DiffCleanupSemantic(diffs) + + // Rediff any replacement blocks, this time character-by-character. + // Add a dummy entry at the end. + diffs = append(diffs, Diff{DiffEqual, ""}) + + pointer := 0 + countDelete := 0 + countInsert := 0 + + // NOTE: Rune slices are slower than using strings in this case. + textDelete := "" + textInsert := "" + + for pointer < len(diffs) { + switch diffs[pointer].Type { + case DiffInsert: + countInsert++ + textInsert += diffs[pointer].Text + case DiffDelete: + countDelete++ + textDelete += diffs[pointer].Text + case DiffEqual: + // Upon reaching an equality, check for prior redundancies. + if countDelete >= 1 && countInsert >= 1 { + // Delete the offending records and add the merged ones. + diffs = splice(diffs, pointer-countDelete-countInsert, + countDelete+countInsert) + + pointer = pointer - countDelete - countInsert + a := dmp.diffMainRunes([]rune(textDelete), []rune(textInsert), false, deadline) + for j := len(a) - 1; j >= 0; j-- { + diffs = splice(diffs, pointer, 0, a[j]) + } + pointer = pointer + len(a) + } + + countInsert = 0 + countDelete = 0 + textDelete = "" + textInsert = "" + } + pointer++ + } + + return diffs[:len(diffs)-1] // Remove the dummy entry at the end. +} + +// DiffBisect finds the 'middle snake' of a diff, split the problem in two and return the recursively constructed diff. +// If an invalid UTF-8 sequence is encountered, it will be replaced by the Unicode replacement character. +// See Myers 1986 paper: An O(ND) Difference Algorithm and Its Variations. +func (dmp *DiffMatchPatch) DiffBisect(text1, text2 string, deadline time.Time) []Diff { + // Unused in this code, but retained for interface compatibility. + return dmp.diffBisect([]rune(text1), []rune(text2), deadline) +} + +// diffBisect finds the 'middle snake' of a diff, splits the problem in two and returns the recursively constructed diff. +// See Myers's 1986 paper: An O(ND) Difference Algorithm and Its Variations. +func (dmp *DiffMatchPatch) diffBisect(runes1, runes2 []rune, deadline time.Time) []Diff { + // Cache the text lengths to prevent multiple calls. + runes1Len, runes2Len := len(runes1), len(runes2) + + maxD := (runes1Len + runes2Len + 1) / 2 + vOffset := maxD + vLength := 2 * maxD + + v1 := make([]int, vLength) + v2 := make([]int, vLength) + for i := range v1 { + v1[i] = -1 + v2[i] = -1 + } + v1[vOffset+1] = 0 + v2[vOffset+1] = 0 + + delta := runes1Len - runes2Len + // If the total number of characters is odd, then the front path will collide with the reverse path. + front := (delta%2 != 0) + // Offsets for start and end of k loop. Prevents mapping of space beyond the grid. + k1start := 0 + k1end := 0 + k2start := 0 + k2end := 0 + for d := 0; d < maxD; d++ { + // Bail out if deadline is reached. + if !deadline.IsZero() && time.Now().After(deadline) { + break + } + + // Walk the front path one step. + for k1 := -d + k1start; k1 <= d-k1end; k1 += 2 { + k1Offset := vOffset + k1 + var x1 int + + if k1 == -d || (k1 != d && v1[k1Offset-1] < v1[k1Offset+1]) { + x1 = v1[k1Offset+1] + } else { + x1 = v1[k1Offset-1] + 1 + } + + y1 := x1 - k1 + for x1 < runes1Len && y1 < runes2Len { + if runes1[x1] != runes2[y1] { + break + } + x1++ + y1++ + } + v1[k1Offset] = x1 + if x1 > runes1Len { + // Ran off the right of the graph. + k1end += 2 + } else if y1 > runes2Len { + // Ran off the bottom of the graph. + k1start += 2 + } else if front { + k2Offset := vOffset + delta - k1 + if k2Offset >= 0 && k2Offset < vLength && v2[k2Offset] != -1 { + // Mirror x2 onto top-left coordinate system. + x2 := runes1Len - v2[k2Offset] + if x1 >= x2 { + // Overlap detected. + return dmp.diffBisectSplit(runes1, runes2, x1, y1, deadline) + } + } + } + } + // Walk the reverse path one step. + for k2 := -d + k2start; k2 <= d-k2end; k2 += 2 { + k2Offset := vOffset + k2 + var x2 int + if k2 == -d || (k2 != d && v2[k2Offset-1] < v2[k2Offset+1]) { + x2 = v2[k2Offset+1] + } else { + x2 = v2[k2Offset-1] + 1 + } + var y2 = x2 - k2 + for x2 < runes1Len && y2 < runes2Len { + if runes1[runes1Len-x2-1] != runes2[runes2Len-y2-1] { + break + } + x2++ + y2++ + } + v2[k2Offset] = x2 + if x2 > runes1Len { + // Ran off the left of the graph. + k2end += 2 + } else if y2 > runes2Len { + // Ran off the top of the graph. + k2start += 2 + } else if !front { + k1Offset := vOffset + delta - k2 + if k1Offset >= 0 && k1Offset < vLength && v1[k1Offset] != -1 { + x1 := v1[k1Offset] + y1 := vOffset + x1 - k1Offset + // Mirror x2 onto top-left coordinate system. + x2 = runes1Len - x2 + if x1 >= x2 { + // Overlap detected. + return dmp.diffBisectSplit(runes1, runes2, x1, y1, deadline) + } + } + } + } + } + // Diff took too long and hit the deadline or number of diffs equals number of characters, no commonality at all. + return []Diff{ + Diff{DiffDelete, string(runes1)}, + Diff{DiffInsert, string(runes2)}, + } +} + +func (dmp *DiffMatchPatch) diffBisectSplit(runes1, runes2 []rune, x, y int, + deadline time.Time) []Diff { + runes1a := runes1[:x] + runes2a := runes2[:y] + runes1b := runes1[x:] + runes2b := runes2[y:] + + // Compute both diffs serially. + diffs := dmp.diffMainRunes(runes1a, runes2a, false, deadline) + diffsb := dmp.diffMainRunes(runes1b, runes2b, false, deadline) + + return append(diffs, diffsb...) +} + +// DiffLinesToChars splits two texts into a list of strings, and educes the texts to a string of hashes where each Unicode character represents one line. +// It's slightly faster to call DiffLinesToRunes first, followed by DiffMainRunes. +func (dmp *DiffMatchPatch) DiffLinesToChars(text1, text2 string) (string, string, []string) { + chars1, chars2, lineArray := dmp.DiffLinesToRunes(text1, text2) + return string(chars1), string(chars2), lineArray +} + +// DiffLinesToRunes splits two texts into a list of runes. Each rune represents one line. +func (dmp *DiffMatchPatch) DiffLinesToRunes(text1, text2 string) ([]rune, []rune, []string) { + // '\x00' is a valid character, but various debuggers don't like it. So we'll insert a junk entry to avoid generating a null character. + lineArray := []string{""} // e.g. lineArray[4] == 'Hello\n' + lineHash := map[string]int{} // e.g. lineHash['Hello\n'] == 4 + + chars1 := dmp.diffLinesToRunesMunge(text1, &lineArray, lineHash) + chars2 := dmp.diffLinesToRunesMunge(text2, &lineArray, lineHash) + + return chars1, chars2, lineArray +} + +func (dmp *DiffMatchPatch) diffLinesToRunes(text1, text2 []rune) ([]rune, []rune, []string) { + return dmp.DiffLinesToRunes(string(text1), string(text2)) +} + +// diffLinesToRunesMunge splits a text into an array of strings, and reduces the texts to a []rune where each Unicode character represents one line. +// We use strings instead of []runes as input mainly because you can't use []rune as a map key. +func (dmp *DiffMatchPatch) diffLinesToRunesMunge(text string, lineArray *[]string, lineHash map[string]int) []rune { + // Walk the text, pulling out a substring for each line. text.split('\n') would would temporarily double our memory footprint. Modifying text would create many large strings to garbage collect. + lineStart := 0 + lineEnd := -1 + runes := []rune{} + + for lineEnd < len(text)-1 { + lineEnd = indexOf(text, "\n", lineStart) + + if lineEnd == -1 { + lineEnd = len(text) - 1 + } + + line := text[lineStart : lineEnd+1] + lineStart = lineEnd + 1 + lineValue, ok := lineHash[line] + + if ok { + runes = append(runes, rune(lineValue)) + } else { + *lineArray = append(*lineArray, line) + lineHash[line] = len(*lineArray) - 1 + runes = append(runes, rune(len(*lineArray)-1)) + } + } + + return runes +} + +// DiffCharsToLines rehydrates the text in a diff from a string of line hashes to real lines of text. +func (dmp *DiffMatchPatch) DiffCharsToLines(diffs []Diff, lineArray []string) []Diff { + hydrated := make([]Diff, 0, len(diffs)) + for _, aDiff := range diffs { + chars := aDiff.Text + text := make([]string, len(chars)) + + for i, r := range chars { + text[i] = lineArray[r] + } + + aDiff.Text = strings.Join(text, "") + hydrated = append(hydrated, aDiff) + } + return hydrated +} + +// DiffCommonPrefix determines the common prefix length of two strings. +func (dmp *DiffMatchPatch) DiffCommonPrefix(text1, text2 string) int { + // Unused in this code, but retained for interface compatibility. + return commonPrefixLength([]rune(text1), []rune(text2)) +} + +// DiffCommonSuffix determines the common suffix length of two strings. +func (dmp *DiffMatchPatch) DiffCommonSuffix(text1, text2 string) int { + // Unused in this code, but retained for interface compatibility. + return commonSuffixLength([]rune(text1), []rune(text2)) +} + +// commonPrefixLength returns the length of the common prefix of two rune slices. +func commonPrefixLength(text1, text2 []rune) int { + short, long := text1, text2 + if len(short) > len(long) { + short, long = long, short + } + for i, r := range short { + if r != long[i] { + return i + } + } + return len(short) +} + +// commonSuffixLength returns the length of the common suffix of two rune slices. +func commonSuffixLength(text1, text2 []rune) int { + n := min(len(text1), len(text2)) + for i := 0; i < n; i++ { + if text1[len(text1)-i-1] != text2[len(text2)-i-1] { + return i + } + } + return n + + // TODO research and benchmark this, why is it not activated? https://github.com/sergi/go-diff/issues/54 + // Binary search. + // Performance analysis: http://neil.fraser.name/news/2007/10/09/ + /* + pointermin := 0 + pointermax := math.Min(len(text1), len(text2)) + pointermid := pointermax + pointerend := 0 + for pointermin < pointermid { + if text1[len(text1)-pointermid:len(text1)-pointerend] == + text2[len(text2)-pointermid:len(text2)-pointerend] { + pointermin = pointermid + pointerend = pointermin + } else { + pointermax = pointermid + } + pointermid = math.Floor((pointermax-pointermin)/2 + pointermin) + } + return pointermid + */ +} + +// DiffCommonOverlap determines if the suffix of one string is the prefix of another. +func (dmp *DiffMatchPatch) DiffCommonOverlap(text1 string, text2 string) int { + // Cache the text lengths to prevent multiple calls. + text1Length := len(text1) + text2Length := len(text2) + // Eliminate the null case. + if text1Length == 0 || text2Length == 0 { + return 0 + } + // Truncate the longer string. + if text1Length > text2Length { + text1 = text1[text1Length-text2Length:] + } else if text1Length < text2Length { + text2 = text2[0:text1Length] + } + textLength := int(math.Min(float64(text1Length), float64(text2Length))) + // Quick check for the worst case. + if text1 == text2 { + return textLength + } + + // Start by looking for a single character match and increase length until no match is found. Performance analysis: http://neil.fraser.name/news/2010/11/04/ + best := 0 + length := 1 + for { + pattern := text1[textLength-length:] + found := strings.Index(text2, pattern) + if found == -1 { + break + } + length += found + if found == 0 || text1[textLength-length:] == text2[0:length] { + best = length + length++ + } + } + + return best +} + +// DiffHalfMatch checks whether the two texts share a substring which is at least half the length of the longer text. This speedup can produce non-minimal diffs. +func (dmp *DiffMatchPatch) DiffHalfMatch(text1, text2 string) []string { + // Unused in this code, but retained for interface compatibility. + runeSlices := dmp.diffHalfMatch([]rune(text1), []rune(text2)) + if runeSlices == nil { + return nil + } + + result := make([]string, len(runeSlices)) + for i, r := range runeSlices { + result[i] = string(r) + } + return result +} + +func (dmp *DiffMatchPatch) diffHalfMatch(text1, text2 []rune) [][]rune { + if dmp.DiffTimeout <= 0 { + // Don't risk returning a non-optimal diff if we have unlimited time. + return nil + } + + var longtext, shorttext []rune + if len(text1) > len(text2) { + longtext = text1 + shorttext = text2 + } else { + longtext = text2 + shorttext = text1 + } + + if len(longtext) < 4 || len(shorttext)*2 < len(longtext) { + return nil // Pointless. + } + + // First check if the second quarter is the seed for a half-match. + hm1 := dmp.diffHalfMatchI(longtext, shorttext, int(float64(len(longtext)+3)/4)) + + // Check again based on the third quarter. + hm2 := dmp.diffHalfMatchI(longtext, shorttext, int(float64(len(longtext)+1)/2)) + + hm := [][]rune{} + if hm1 == nil && hm2 == nil { + return nil + } else if hm2 == nil { + hm = hm1 + } else if hm1 == nil { + hm = hm2 + } else { + // Both matched. Select the longest. + if len(hm1[4]) > len(hm2[4]) { + hm = hm1 + } else { + hm = hm2 + } + } + + // A half-match was found, sort out the return data. + if len(text1) > len(text2) { + return hm + } + + return [][]rune{hm[2], hm[3], hm[0], hm[1], hm[4]} +} + +// diffHalfMatchI checks if a substring of shorttext exist within longtext such that the substring is at least half the length of longtext? +// Returns a slice containing the prefix of longtext, the suffix of longtext, the prefix of shorttext, the suffix of shorttext and the common middle, or null if there was no match. +func (dmp *DiffMatchPatch) diffHalfMatchI(l, s []rune, i int) [][]rune { + var bestCommonA []rune + var bestCommonB []rune + var bestCommonLen int + var bestLongtextA []rune + var bestLongtextB []rune + var bestShorttextA []rune + var bestShorttextB []rune + + // Start with a 1/4 length substring at position i as a seed. + seed := l[i : i+len(l)/4] + + for j := runesIndexOf(s, seed, 0); j != -1; j = runesIndexOf(s, seed, j+1) { + prefixLength := commonPrefixLength(l[i:], s[j:]) + suffixLength := commonSuffixLength(l[:i], s[:j]) + + if bestCommonLen < suffixLength+prefixLength { + bestCommonA = s[j-suffixLength : j] + bestCommonB = s[j : j+prefixLength] + bestCommonLen = len(bestCommonA) + len(bestCommonB) + bestLongtextA = l[:i-suffixLength] + bestLongtextB = l[i+prefixLength:] + bestShorttextA = s[:j-suffixLength] + bestShorttextB = s[j+prefixLength:] + } + } + + if bestCommonLen*2 < len(l) { + return nil + } + + return [][]rune{ + bestLongtextA, + bestLongtextB, + bestShorttextA, + bestShorttextB, + append(bestCommonA, bestCommonB...), + } +} + +// DiffCleanupSemantic reduces the number of edits by eliminating semantically trivial equalities. +func (dmp *DiffMatchPatch) DiffCleanupSemantic(diffs []Diff) []Diff { + changes := false + // Stack of indices where equalities are found. + type equality struct { + data int + next *equality + } + var equalities *equality + + var lastequality string + // Always equal to diffs[equalities[equalitiesLength - 1]][1] + var pointer int // Index of current position. + // Number of characters that changed prior to the equality. + var lengthInsertions1, lengthDeletions1 int + // Number of characters that changed after the equality. + var lengthInsertions2, lengthDeletions2 int + + for pointer < len(diffs) { + if diffs[pointer].Type == DiffEqual { + // Equality found. + + equalities = &equality{ + data: pointer, + next: equalities, + } + lengthInsertions1 = lengthInsertions2 + lengthDeletions1 = lengthDeletions2 + lengthInsertions2 = 0 + lengthDeletions2 = 0 + lastequality = diffs[pointer].Text + } else { + // An insertion or deletion. + + if diffs[pointer].Type == DiffInsert { + lengthInsertions2 += len(diffs[pointer].Text) + } else { + lengthDeletions2 += len(diffs[pointer].Text) + } + // Eliminate an equality that is smaller or equal to the edits on both sides of it. + difference1 := int(math.Max(float64(lengthInsertions1), float64(lengthDeletions1))) + difference2 := int(math.Max(float64(lengthInsertions2), float64(lengthDeletions2))) + if len(lastequality) > 0 && + (len(lastequality) <= difference1) && + (len(lastequality) <= difference2) { + // Duplicate record. + insPoint := equalities.data + diffs = append( + diffs[:insPoint], + append([]Diff{Diff{DiffDelete, lastequality}}, diffs[insPoint:]...)...) + + // Change second copy to insert. + diffs[insPoint+1].Type = DiffInsert + // Throw away the equality we just deleted. + equalities = equalities.next + + if equalities != nil { + equalities = equalities.next + } + if equalities != nil { + pointer = equalities.data + } else { + pointer = -1 + } + + lengthInsertions1 = 0 // Reset the counters. + lengthDeletions1 = 0 + lengthInsertions2 = 0 + lengthDeletions2 = 0 + lastequality = "" + changes = true + } + } + pointer++ + } + + // Normalize the diff. + if changes { + diffs = dmp.DiffCleanupMerge(diffs) + } + diffs = dmp.DiffCleanupSemanticLossless(diffs) + // Find any overlaps between deletions and insertions. + // e.g: abcxxxxxxdef + // -> abcxxxdef + // e.g: xxxabcdefxxx + // -> defxxxabc + // Only extract an overlap if it is as big as the edit ahead or behind it. + pointer = 1 + for pointer < len(diffs) { + if diffs[pointer-1].Type == DiffDelete && + diffs[pointer].Type == DiffInsert { + deletion := diffs[pointer-1].Text + insertion := diffs[pointer].Text + overlapLength1 := dmp.DiffCommonOverlap(deletion, insertion) + overlapLength2 := dmp.DiffCommonOverlap(insertion, deletion) + if overlapLength1 >= overlapLength2 { + if float64(overlapLength1) >= float64(len(deletion))/2 || + float64(overlapLength1) >= float64(len(insertion))/2 { + + // Overlap found. Insert an equality and trim the surrounding edits. + diffs = append( + diffs[:pointer], + append([]Diff{Diff{DiffEqual, insertion[:overlapLength1]}}, diffs[pointer:]...)...) + + diffs[pointer-1].Text = + deletion[0 : len(deletion)-overlapLength1] + diffs[pointer+1].Text = insertion[overlapLength1:] + pointer++ + } + } else { + if float64(overlapLength2) >= float64(len(deletion))/2 || + float64(overlapLength2) >= float64(len(insertion))/2 { + // Reverse overlap found. Insert an equality and swap and trim the surrounding edits. + overlap := Diff{DiffEqual, deletion[:overlapLength2]} + diffs = append( + diffs[:pointer], + append([]Diff{overlap}, diffs[pointer:]...)...) + + diffs[pointer-1].Type = DiffInsert + diffs[pointer-1].Text = insertion[0 : len(insertion)-overlapLength2] + diffs[pointer+1].Type = DiffDelete + diffs[pointer+1].Text = deletion[overlapLength2:] + pointer++ + } + } + pointer++ + } + pointer++ + } + + return diffs +} + +// Define some regex patterns for matching boundaries. +var ( + nonAlphaNumericRegex = regexp.MustCompile(`[^a-zA-Z0-9]`) + whitespaceRegex = regexp.MustCompile(`\s`) + linebreakRegex = regexp.MustCompile(`[\r\n]`) + blanklineEndRegex = regexp.MustCompile(`\n\r?\n$`) + blanklineStartRegex = regexp.MustCompile(`^\r?\n\r?\n`) +) + +// diffCleanupSemanticScore computes a score representing whether the internal boundary falls on logical boundaries. +// Scores range from 6 (best) to 0 (worst). Closure, but does not reference any external variables. +func diffCleanupSemanticScore(one, two string) int { + if len(one) == 0 || len(two) == 0 { + // Edges are the best. + return 6 + } + + // Each port of this function behaves slightly differently due to subtle differences in each language's definition of things like 'whitespace'. Since this function's purpose is largely cosmetic, the choice has been made to use each language's native features rather than force total conformity. + rune1, _ := utf8.DecodeLastRuneInString(one) + rune2, _ := utf8.DecodeRuneInString(two) + char1 := string(rune1) + char2 := string(rune2) + + nonAlphaNumeric1 := nonAlphaNumericRegex.MatchString(char1) + nonAlphaNumeric2 := nonAlphaNumericRegex.MatchString(char2) + whitespace1 := nonAlphaNumeric1 && whitespaceRegex.MatchString(char1) + whitespace2 := nonAlphaNumeric2 && whitespaceRegex.MatchString(char2) + lineBreak1 := whitespace1 && linebreakRegex.MatchString(char1) + lineBreak2 := whitespace2 && linebreakRegex.MatchString(char2) + blankLine1 := lineBreak1 && blanklineEndRegex.MatchString(one) + blankLine2 := lineBreak2 && blanklineEndRegex.MatchString(two) + + if blankLine1 || blankLine2 { + // Five points for blank lines. + return 5 + } else if lineBreak1 || lineBreak2 { + // Four points for line breaks. + return 4 + } else if nonAlphaNumeric1 && !whitespace1 && whitespace2 { + // Three points for end of sentences. + return 3 + } else if whitespace1 || whitespace2 { + // Two points for whitespace. + return 2 + } else if nonAlphaNumeric1 || nonAlphaNumeric2 { + // One point for non-alphanumeric. + return 1 + } + return 0 +} + +// DiffCleanupSemanticLossless looks for single edits surrounded on both sides by equalities which can be shifted sideways to align the edit to a word boundary. +// E.g: The cat came. -> The cat came. +func (dmp *DiffMatchPatch) DiffCleanupSemanticLossless(diffs []Diff) []Diff { + pointer := 1 + + // Intentionally ignore the first and last element (don't need checking). + for pointer < len(diffs)-1 { + if diffs[pointer-1].Type == DiffEqual && + diffs[pointer+1].Type == DiffEqual { + + // This is a single edit surrounded by equalities. + equality1 := diffs[pointer-1].Text + edit := diffs[pointer].Text + equality2 := diffs[pointer+1].Text + + // First, shift the edit as far left as possible. + commonOffset := dmp.DiffCommonSuffix(equality1, edit) + if commonOffset > 0 { + commonString := edit[len(edit)-commonOffset:] + equality1 = equality1[0 : len(equality1)-commonOffset] + edit = commonString + edit[:len(edit)-commonOffset] + equality2 = commonString + equality2 + } + + // Second, step character by character right, looking for the best fit. + bestEquality1 := equality1 + bestEdit := edit + bestEquality2 := equality2 + bestScore := diffCleanupSemanticScore(equality1, edit) + + diffCleanupSemanticScore(edit, equality2) + + for len(edit) != 0 && len(equality2) != 0 { + _, sz := utf8.DecodeRuneInString(edit) + if len(equality2) < sz || edit[:sz] != equality2[:sz] { + break + } + equality1 += edit[:sz] + edit = edit[sz:] + equality2[:sz] + equality2 = equality2[sz:] + score := diffCleanupSemanticScore(equality1, edit) + + diffCleanupSemanticScore(edit, equality2) + // The >= encourages trailing rather than leading whitespace on edits. + if score >= bestScore { + bestScore = score + bestEquality1 = equality1 + bestEdit = edit + bestEquality2 = equality2 + } + } + + if diffs[pointer-1].Text != bestEquality1 { + // We have an improvement, save it back to the diff. + if len(bestEquality1) != 0 { + diffs[pointer-1].Text = bestEquality1 + } else { + diffs = splice(diffs, pointer-1, 1) + pointer-- + } + + diffs[pointer].Text = bestEdit + if len(bestEquality2) != 0 { + diffs[pointer+1].Text = bestEquality2 + } else { + diffs = append(diffs[:pointer+1], diffs[pointer+2:]...) + pointer-- + } + } + } + pointer++ + } + + return diffs +} + +// DiffCleanupEfficiency reduces the number of edits by eliminating operationally trivial equalities. +func (dmp *DiffMatchPatch) DiffCleanupEfficiency(diffs []Diff) []Diff { + changes := false + // Stack of indices where equalities are found. + type equality struct { + data int + next *equality + } + var equalities *equality + // Always equal to equalities[equalitiesLength-1][1] + lastequality := "" + pointer := 0 // Index of current position. + // Is there an insertion operation before the last equality. + preIns := false + // Is there a deletion operation before the last equality. + preDel := false + // Is there an insertion operation after the last equality. + postIns := false + // Is there a deletion operation after the last equality. + postDel := false + for pointer < len(diffs) { + if diffs[pointer].Type == DiffEqual { // Equality found. + if len(diffs[pointer].Text) < dmp.DiffEditCost && + (postIns || postDel) { + // Candidate found. + equalities = &equality{ + data: pointer, + next: equalities, + } + preIns = postIns + preDel = postDel + lastequality = diffs[pointer].Text + } else { + // Not a candidate, and can never become one. + equalities = nil + lastequality = "" + } + postIns = false + postDel = false + } else { // An insertion or deletion. + if diffs[pointer].Type == DiffDelete { + postDel = true + } else { + postIns = true + } + + // Five types to be split: + // ABXYCD + // AXCD + // ABXC + // AXCD + // ABXC + var sumPres int + if preIns { + sumPres++ + } + if preDel { + sumPres++ + } + if postIns { + sumPres++ + } + if postDel { + sumPres++ + } + if len(lastequality) > 0 && + ((preIns && preDel && postIns && postDel) || + ((len(lastequality) < dmp.DiffEditCost/2) && sumPres == 3)) { + + insPoint := equalities.data + + // Duplicate record. + diffs = append(diffs[:insPoint], + append([]Diff{Diff{DiffDelete, lastequality}}, diffs[insPoint:]...)...) + + // Change second copy to insert. + diffs[insPoint+1].Type = DiffInsert + // Throw away the equality we just deleted. + equalities = equalities.next + lastequality = "" + + if preIns && preDel { + // No changes made which could affect previous entry, keep going. + postIns = true + postDel = true + equalities = nil + } else { + if equalities != nil { + equalities = equalities.next + } + if equalities != nil { + pointer = equalities.data + } else { + pointer = -1 + } + postIns = false + postDel = false + } + changes = true + } + } + pointer++ + } + + if changes { + diffs = dmp.DiffCleanupMerge(diffs) + } + + return diffs +} + +// DiffCleanupMerge reorders and merges like edit sections. Merge equalities. +// Any edit section can move as long as it doesn't cross an equality. +func (dmp *DiffMatchPatch) DiffCleanupMerge(diffs []Diff) []Diff { + // Add a dummy entry at the end. + diffs = append(diffs, Diff{DiffEqual, ""}) + pointer := 0 + countDelete := 0 + countInsert := 0 + commonlength := 0 + textDelete := []rune(nil) + textInsert := []rune(nil) + + for pointer < len(diffs) { + switch diffs[pointer].Type { + case DiffInsert: + countInsert++ + textInsert = append(textInsert, []rune(diffs[pointer].Text)...) + pointer++ + break + case DiffDelete: + countDelete++ + textDelete = append(textDelete, []rune(diffs[pointer].Text)...) + pointer++ + break + case DiffEqual: + // Upon reaching an equality, check for prior redundancies. + if countDelete+countInsert > 1 { + if countDelete != 0 && countInsert != 0 { + // Factor out any common prefixies. + commonlength = commonPrefixLength(textInsert, textDelete) + if commonlength != 0 { + x := pointer - countDelete - countInsert + if x > 0 && diffs[x-1].Type == DiffEqual { + diffs[x-1].Text += string(textInsert[:commonlength]) + } else { + diffs = append([]Diff{Diff{DiffEqual, string(textInsert[:commonlength])}}, diffs...) + pointer++ + } + textInsert = textInsert[commonlength:] + textDelete = textDelete[commonlength:] + } + // Factor out any common suffixies. + commonlength = commonSuffixLength(textInsert, textDelete) + if commonlength != 0 { + insertIndex := len(textInsert) - commonlength + deleteIndex := len(textDelete) - commonlength + diffs[pointer].Text = string(textInsert[insertIndex:]) + diffs[pointer].Text + textInsert = textInsert[:insertIndex] + textDelete = textDelete[:deleteIndex] + } + } + // Delete the offending records and add the merged ones. + if countDelete == 0 { + diffs = splice(diffs, pointer-countInsert, + countDelete+countInsert, + Diff{DiffInsert, string(textInsert)}) + } else if countInsert == 0 { + diffs = splice(diffs, pointer-countDelete, + countDelete+countInsert, + Diff{DiffDelete, string(textDelete)}) + } else { + diffs = splice(diffs, pointer-countDelete-countInsert, + countDelete+countInsert, + Diff{DiffDelete, string(textDelete)}, + Diff{DiffInsert, string(textInsert)}) + } + + pointer = pointer - countDelete - countInsert + 1 + if countDelete != 0 { + pointer++ + } + if countInsert != 0 { + pointer++ + } + } else if pointer != 0 && diffs[pointer-1].Type == DiffEqual { + // Merge this equality with the previous one. + diffs[pointer-1].Text += diffs[pointer].Text + diffs = append(diffs[:pointer], diffs[pointer+1:]...) + } else { + pointer++ + } + countInsert = 0 + countDelete = 0 + textDelete = nil + textInsert = nil + break + } + } + + if len(diffs[len(diffs)-1].Text) == 0 { + diffs = diffs[0 : len(diffs)-1] // Remove the dummy entry at the end. + } + + // Second pass: look for single edits surrounded on both sides by equalities which can be shifted sideways to eliminate an equality. E.g: ABAC -> ABAC + changes := false + pointer = 1 + // Intentionally ignore the first and last element (don't need checking). + for pointer < (len(diffs) - 1) { + if diffs[pointer-1].Type == DiffEqual && + diffs[pointer+1].Type == DiffEqual { + // This is a single edit surrounded by equalities. + if strings.HasSuffix(diffs[pointer].Text, diffs[pointer-1].Text) { + // Shift the edit over the previous equality. + diffs[pointer].Text = diffs[pointer-1].Text + + diffs[pointer].Text[:len(diffs[pointer].Text)-len(diffs[pointer-1].Text)] + diffs[pointer+1].Text = diffs[pointer-1].Text + diffs[pointer+1].Text + diffs = splice(diffs, pointer-1, 1) + changes = true + } else if strings.HasPrefix(diffs[pointer].Text, diffs[pointer+1].Text) { + // Shift the edit over the next equality. + diffs[pointer-1].Text += diffs[pointer+1].Text + diffs[pointer].Text = + diffs[pointer].Text[len(diffs[pointer+1].Text):] + diffs[pointer+1].Text + diffs = splice(diffs, pointer+1, 1) + changes = true + } + } + pointer++ + } + + // If shifts were made, the diff needs reordering and another shift sweep. + if changes { + diffs = dmp.DiffCleanupMerge(diffs) + } + + return diffs +} + +// DiffXIndex returns the equivalent location in s2. +func (dmp *DiffMatchPatch) DiffXIndex(diffs []Diff, loc int) int { + chars1 := 0 + chars2 := 0 + lastChars1 := 0 + lastChars2 := 0 + lastDiff := Diff{} + for i := 0; i < len(diffs); i++ { + aDiff := diffs[i] + if aDiff.Type != DiffInsert { + // Equality or deletion. + chars1 += len(aDiff.Text) + } + if aDiff.Type != DiffDelete { + // Equality or insertion. + chars2 += len(aDiff.Text) + } + if chars1 > loc { + // Overshot the location. + lastDiff = aDiff + break + } + lastChars1 = chars1 + lastChars2 = chars2 + } + if lastDiff.Type == DiffDelete { + // The location was deleted. + return lastChars2 + } + // Add the remaining character length. + return lastChars2 + (loc - lastChars1) +} + +// DiffPrettyHtml converts a []Diff into a pretty HTML report. +// It is intended as an example from which to write one's own display functions. +func (dmp *DiffMatchPatch) DiffPrettyHtml(diffs []Diff) string { + var buff bytes.Buffer + for _, diff := range diffs { + text := strings.Replace(html.EscapeString(diff.Text), "\n", "¶
", -1) + switch diff.Type { + case DiffInsert: + _, _ = buff.WriteString("") + _, _ = buff.WriteString(text) + _, _ = buff.WriteString("") + case DiffDelete: + _, _ = buff.WriteString("") + _, _ = buff.WriteString(text) + _, _ = buff.WriteString("") + case DiffEqual: + _, _ = buff.WriteString("") + _, _ = buff.WriteString(text) + _, _ = buff.WriteString("") + } + } + return buff.String() +} + +// DiffPrettyText converts a []Diff into a colored text report. +func (dmp *DiffMatchPatch) DiffPrettyText(diffs []Diff) string { + var buff bytes.Buffer + for _, diff := range diffs { + text := diff.Text + + switch diff.Type { + case DiffInsert: + _, _ = buff.WriteString("\x1b[32m") + _, _ = buff.WriteString(text) + _, _ = buff.WriteString("\x1b[0m") + case DiffDelete: + _, _ = buff.WriteString("\x1b[31m") + _, _ = buff.WriteString(text) + _, _ = buff.WriteString("\x1b[0m") + case DiffEqual: + _, _ = buff.WriteString(text) + } + } + + return buff.String() +} + +// DiffText1 computes and returns the source text (all equalities and deletions). +func (dmp *DiffMatchPatch) DiffText1(diffs []Diff) string { + //StringBuilder text = new StringBuilder() + var text bytes.Buffer + + for _, aDiff := range diffs { + if aDiff.Type != DiffInsert { + _, _ = text.WriteString(aDiff.Text) + } + } + return text.String() +} + +// DiffText2 computes and returns the destination text (all equalities and insertions). +func (dmp *DiffMatchPatch) DiffText2(diffs []Diff) string { + var text bytes.Buffer + + for _, aDiff := range diffs { + if aDiff.Type != DiffDelete { + _, _ = text.WriteString(aDiff.Text) + } + } + return text.String() +} + +// DiffLevenshtein computes the Levenshtein distance that is the number of inserted, deleted or substituted characters. +func (dmp *DiffMatchPatch) DiffLevenshtein(diffs []Diff) int { + levenshtein := 0 + insertions := 0 + deletions := 0 + + for _, aDiff := range diffs { + switch aDiff.Type { + case DiffInsert: + insertions += len(aDiff.Text) + case DiffDelete: + deletions += len(aDiff.Text) + case DiffEqual: + // A deletion and an insertion is one substitution. + levenshtein += max(insertions, deletions) + insertions = 0 + deletions = 0 + } + } + + levenshtein += max(insertions, deletions) + return levenshtein +} + +// DiffToDelta crushes the diff into an encoded string which describes the operations required to transform text1 into text2. +// E.g. =3\t-2\t+ing -> Keep 3 chars, delete 2 chars, insert 'ing'. Operations are tab-separated. Inserted text is escaped using %xx notation. +func (dmp *DiffMatchPatch) DiffToDelta(diffs []Diff) string { + var text bytes.Buffer + for _, aDiff := range diffs { + switch aDiff.Type { + case DiffInsert: + _, _ = text.WriteString("+") + _, _ = text.WriteString(strings.Replace(url.QueryEscape(aDiff.Text), "+", " ", -1)) + _, _ = text.WriteString("\t") + break + case DiffDelete: + _, _ = text.WriteString("-") + _, _ = text.WriteString(strconv.Itoa(utf8.RuneCountInString(aDiff.Text))) + _, _ = text.WriteString("\t") + break + case DiffEqual: + _, _ = text.WriteString("=") + _, _ = text.WriteString(strconv.Itoa(utf8.RuneCountInString(aDiff.Text))) + _, _ = text.WriteString("\t") + break + } + } + delta := text.String() + if len(delta) != 0 { + // Strip off trailing tab character. + delta = delta[0 : utf8.RuneCountInString(delta)-1] + delta = unescaper.Replace(delta) + } + return delta +} + +// DiffFromDelta given the original text1, and an encoded string which describes the operations required to transform text1 into text2, comAdde the full diff. +func (dmp *DiffMatchPatch) DiffFromDelta(text1 string, delta string) (diffs []Diff, err error) { + i := 0 + runes := []rune(text1) + + for _, token := range strings.Split(delta, "\t") { + if len(token) == 0 { + // Blank tokens are ok (from a trailing \t). + continue + } + + // Each token begins with a one character parameter which specifies the operation of this token (delete, insert, equality). + param := token[1:] + + switch op := token[0]; op { + case '+': + // Decode would Diff all "+" to " " + param = strings.Replace(param, "+", "%2b", -1) + param, err = url.QueryUnescape(param) + if err != nil { + return nil, err + } + if !utf8.ValidString(param) { + return nil, fmt.Errorf("invalid UTF-8 token: %q", param) + } + + diffs = append(diffs, Diff{DiffInsert, param}) + case '=', '-': + n, err := strconv.ParseInt(param, 10, 0) + if err != nil { + return nil, err + } else if n < 0 { + return nil, errors.New("Negative number in DiffFromDelta: " + param) + } + + i += int(n) + // Break out if we are out of bounds, go1.6 can't handle this very well + if i > len(runes) { + break + } + // Remember that string slicing is by byte - we want by rune here. + text := string(runes[i-int(n) : i]) + + if op == '=' { + diffs = append(diffs, Diff{DiffEqual, text}) + } else { + diffs = append(diffs, Diff{DiffDelete, text}) + } + default: + // Anything else is an error. + return nil, errors.New("Invalid diff operation in DiffFromDelta: " + string(token[0])) + } + } + + if i != len(runes) { + return nil, fmt.Errorf("Delta length (%v) is different from source text length (%v)", i, len(text1)) + } + + return diffs, nil +} diff --git a/vendor/github.com/sergi/go-diff/diffmatchpatch/diffmatchpatch.go b/vendor/github.com/sergi/go-diff/diffmatchpatch/diffmatchpatch.go new file mode 100644 index 00000000000..d3acc32ce13 --- /dev/null +++ b/vendor/github.com/sergi/go-diff/diffmatchpatch/diffmatchpatch.go @@ -0,0 +1,46 @@ +// Copyright (c) 2012-2016 The go-diff authors. All rights reserved. +// https://github.com/sergi/go-diff +// See the included LICENSE file for license details. +// +// go-diff is a Go implementation of Google's Diff, Match, and Patch library +// Original library is Copyright (c) 2006 Google Inc. +// http://code.google.com/p/google-diff-match-patch/ + +// Package diffmatchpatch offers robust algorithms to perform the operations required for synchronizing plain text. +package diffmatchpatch + +import ( + "time" +) + +// DiffMatchPatch holds the configuration for diff-match-patch operations. +type DiffMatchPatch struct { + // Number of seconds to map a diff before giving up (0 for infinity). + DiffTimeout time.Duration + // Cost of an empty edit operation in terms of edit characters. + DiffEditCost int + // How far to search for a match (0 = exact location, 1000+ = broad match). A match this many characters away from the expected location will add 1.0 to the score (0.0 is a perfect match). + MatchDistance int + // When deleting a large block of text (over ~64 characters), how close do the contents have to be to match the expected contents. (0.0 = perfection, 1.0 = very loose). Note that MatchThreshold controls how closely the end points of a delete need to match. + PatchDeleteThreshold float64 + // Chunk size for context length. + PatchMargin int + // The number of bits in an int. + MatchMaxBits int + // At what point is no match declared (0.0 = perfection, 1.0 = very loose). + MatchThreshold float64 +} + +// New creates a new DiffMatchPatch object with default parameters. +func New() *DiffMatchPatch { + // Defaults. + return &DiffMatchPatch{ + DiffTimeout: time.Second, + DiffEditCost: 4, + MatchThreshold: 0.5, + MatchDistance: 1000, + PatchDeleteThreshold: 0.5, + PatchMargin: 4, + MatchMaxBits: 32, + } +} diff --git a/vendor/github.com/sergi/go-diff/diffmatchpatch/match.go b/vendor/github.com/sergi/go-diff/diffmatchpatch/match.go new file mode 100644 index 00000000000..17374e109fe --- /dev/null +++ b/vendor/github.com/sergi/go-diff/diffmatchpatch/match.go @@ -0,0 +1,160 @@ +// Copyright (c) 2012-2016 The go-diff authors. All rights reserved. +// https://github.com/sergi/go-diff +// See the included LICENSE file for license details. +// +// go-diff is a Go implementation of Google's Diff, Match, and Patch library +// Original library is Copyright (c) 2006 Google Inc. +// http://code.google.com/p/google-diff-match-patch/ + +package diffmatchpatch + +import ( + "math" +) + +// MatchMain locates the best instance of 'pattern' in 'text' near 'loc'. +// Returns -1 if no match found. +func (dmp *DiffMatchPatch) MatchMain(text, pattern string, loc int) int { + // Check for null inputs not needed since null can't be passed in C#. + + loc = int(math.Max(0, math.Min(float64(loc), float64(len(text))))) + if text == pattern { + // Shortcut (potentially not guaranteed by the algorithm) + return 0 + } else if len(text) == 0 { + // Nothing to match. + return -1 + } else if loc+len(pattern) <= len(text) && text[loc:loc+len(pattern)] == pattern { + // Perfect match at the perfect spot! (Includes case of null pattern) + return loc + } + // Do a fuzzy compare. + return dmp.MatchBitap(text, pattern, loc) +} + +// MatchBitap locates the best instance of 'pattern' in 'text' near 'loc' using the Bitap algorithm. +// Returns -1 if no match was found. +func (dmp *DiffMatchPatch) MatchBitap(text, pattern string, loc int) int { + // Initialise the alphabet. + s := dmp.MatchAlphabet(pattern) + + // Highest score beyond which we give up. + scoreThreshold := dmp.MatchThreshold + // Is there a nearby exact match? (speedup) + bestLoc := indexOf(text, pattern, loc) + if bestLoc != -1 { + scoreThreshold = math.Min(dmp.matchBitapScore(0, bestLoc, loc, + pattern), scoreThreshold) + // What about in the other direction? (speedup) + bestLoc = lastIndexOf(text, pattern, loc+len(pattern)) + if bestLoc != -1 { + scoreThreshold = math.Min(dmp.matchBitapScore(0, bestLoc, loc, + pattern), scoreThreshold) + } + } + + // Initialise the bit arrays. + matchmask := 1 << uint((len(pattern) - 1)) + bestLoc = -1 + + var binMin, binMid int + binMax := len(pattern) + len(text) + lastRd := []int{} + for d := 0; d < len(pattern); d++ { + // Scan for the best match; each iteration allows for one more error. Run a binary search to determine how far from 'loc' we can stray at this error level. + binMin = 0 + binMid = binMax + for binMin < binMid { + if dmp.matchBitapScore(d, loc+binMid, loc, pattern) <= scoreThreshold { + binMin = binMid + } else { + binMax = binMid + } + binMid = (binMax-binMin)/2 + binMin + } + // Use the result from this iteration as the maximum for the next. + binMax = binMid + start := int(math.Max(1, float64(loc-binMid+1))) + finish := int(math.Min(float64(loc+binMid), float64(len(text))) + float64(len(pattern))) + + rd := make([]int, finish+2) + rd[finish+1] = (1 << uint(d)) - 1 + + for j := finish; j >= start; j-- { + var charMatch int + if len(text) <= j-1 { + // Out of range. + charMatch = 0 + } else if _, ok := s[text[j-1]]; !ok { + charMatch = 0 + } else { + charMatch = s[text[j-1]] + } + + if d == 0 { + // First pass: exact match. + rd[j] = ((rd[j+1] << 1) | 1) & charMatch + } else { + // Subsequent passes: fuzzy match. + rd[j] = ((rd[j+1]<<1)|1)&charMatch | (((lastRd[j+1] | lastRd[j]) << 1) | 1) | lastRd[j+1] + } + if (rd[j] & matchmask) != 0 { + score := dmp.matchBitapScore(d, j-1, loc, pattern) + // This match will almost certainly be better than any existing match. But check anyway. + if score <= scoreThreshold { + // Told you so. + scoreThreshold = score + bestLoc = j - 1 + if bestLoc > loc { + // When passing loc, don't exceed our current distance from loc. + start = int(math.Max(1, float64(2*loc-bestLoc))) + } else { + // Already passed loc, downhill from here on in. + break + } + } + } + } + if dmp.matchBitapScore(d+1, loc, loc, pattern) > scoreThreshold { + // No hope for a (better) match at greater error levels. + break + } + lastRd = rd + } + return bestLoc +} + +// matchBitapScore computes and returns the score for a match with e errors and x location. +func (dmp *DiffMatchPatch) matchBitapScore(e, x, loc int, pattern string) float64 { + accuracy := float64(e) / float64(len(pattern)) + proximity := math.Abs(float64(loc - x)) + if dmp.MatchDistance == 0 { + // Dodge divide by zero error. + if proximity == 0 { + return accuracy + } + + return 1.0 + } + return accuracy + (proximity / float64(dmp.MatchDistance)) +} + +// MatchAlphabet initialises the alphabet for the Bitap algorithm. +func (dmp *DiffMatchPatch) MatchAlphabet(pattern string) map[byte]int { + s := map[byte]int{} + charPattern := []byte(pattern) + for _, c := range charPattern { + _, ok := s[c] + if !ok { + s[c] = 0 + } + } + i := 0 + + for _, c := range charPattern { + value := s[c] | int(uint(1)< y { + return x + } + return y +} diff --git a/vendor/github.com/sergi/go-diff/diffmatchpatch/patch.go b/vendor/github.com/sergi/go-diff/diffmatchpatch/patch.go new file mode 100644 index 00000000000..223c43c4268 --- /dev/null +++ b/vendor/github.com/sergi/go-diff/diffmatchpatch/patch.go @@ -0,0 +1,556 @@ +// Copyright (c) 2012-2016 The go-diff authors. All rights reserved. +// https://github.com/sergi/go-diff +// See the included LICENSE file for license details. +// +// go-diff is a Go implementation of Google's Diff, Match, and Patch library +// Original library is Copyright (c) 2006 Google Inc. +// http://code.google.com/p/google-diff-match-patch/ + +package diffmatchpatch + +import ( + "bytes" + "errors" + "math" + "net/url" + "regexp" + "strconv" + "strings" +) + +// Patch represents one patch operation. +type Patch struct { + diffs []Diff + Start1 int + Start2 int + Length1 int + Length2 int +} + +// String emulates GNU diff's format. +// Header: @@ -382,8 +481,9 @@ +// Indices are printed as 1-based, not 0-based. +func (p *Patch) String() string { + var coords1, coords2 string + + if p.Length1 == 0 { + coords1 = strconv.Itoa(p.Start1) + ",0" + } else if p.Length1 == 1 { + coords1 = strconv.Itoa(p.Start1 + 1) + } else { + coords1 = strconv.Itoa(p.Start1+1) + "," + strconv.Itoa(p.Length1) + } + + if p.Length2 == 0 { + coords2 = strconv.Itoa(p.Start2) + ",0" + } else if p.Length2 == 1 { + coords2 = strconv.Itoa(p.Start2 + 1) + } else { + coords2 = strconv.Itoa(p.Start2+1) + "," + strconv.Itoa(p.Length2) + } + + var text bytes.Buffer + _, _ = text.WriteString("@@ -" + coords1 + " +" + coords2 + " @@\n") + + // Escape the body of the patch with %xx notation. + for _, aDiff := range p.diffs { + switch aDiff.Type { + case DiffInsert: + _, _ = text.WriteString("+") + case DiffDelete: + _, _ = text.WriteString("-") + case DiffEqual: + _, _ = text.WriteString(" ") + } + + _, _ = text.WriteString(strings.Replace(url.QueryEscape(aDiff.Text), "+", " ", -1)) + _, _ = text.WriteString("\n") + } + + return unescaper.Replace(text.String()) +} + +// PatchAddContext increases the context until it is unique, but doesn't let the pattern expand beyond MatchMaxBits. +func (dmp *DiffMatchPatch) PatchAddContext(patch Patch, text string) Patch { + if len(text) == 0 { + return patch + } + + pattern := text[patch.Start2 : patch.Start2+patch.Length1] + padding := 0 + + // Look for the first and last matches of pattern in text. If two different matches are found, increase the pattern length. + for strings.Index(text, pattern) != strings.LastIndex(text, pattern) && + len(pattern) < dmp.MatchMaxBits-2*dmp.PatchMargin { + padding += dmp.PatchMargin + maxStart := max(0, patch.Start2-padding) + minEnd := min(len(text), patch.Start2+patch.Length1+padding) + pattern = text[maxStart:minEnd] + } + // Add one chunk for good luck. + padding += dmp.PatchMargin + + // Add the prefix. + prefix := text[max(0, patch.Start2-padding):patch.Start2] + if len(prefix) != 0 { + patch.diffs = append([]Diff{Diff{DiffEqual, prefix}}, patch.diffs...) + } + // Add the suffix. + suffix := text[patch.Start2+patch.Length1 : min(len(text), patch.Start2+patch.Length1+padding)] + if len(suffix) != 0 { + patch.diffs = append(patch.diffs, Diff{DiffEqual, suffix}) + } + + // Roll back the start points. + patch.Start1 -= len(prefix) + patch.Start2 -= len(prefix) + // Extend the lengths. + patch.Length1 += len(prefix) + len(suffix) + patch.Length2 += len(prefix) + len(suffix) + + return patch +} + +// PatchMake computes a list of patches. +func (dmp *DiffMatchPatch) PatchMake(opt ...interface{}) []Patch { + if len(opt) == 1 { + diffs, _ := opt[0].([]Diff) + text1 := dmp.DiffText1(diffs) + return dmp.PatchMake(text1, diffs) + } else if len(opt) == 2 { + text1 := opt[0].(string) + switch t := opt[1].(type) { + case string: + diffs := dmp.DiffMain(text1, t, true) + if len(diffs) > 2 { + diffs = dmp.DiffCleanupSemantic(diffs) + diffs = dmp.DiffCleanupEfficiency(diffs) + } + return dmp.PatchMake(text1, diffs) + case []Diff: + return dmp.patchMake2(text1, t) + } + } else if len(opt) == 3 { + return dmp.PatchMake(opt[0], opt[2]) + } + return []Patch{} +} + +// patchMake2 computes a list of patches to turn text1 into text2. +// text2 is not provided, diffs are the delta between text1 and text2. +func (dmp *DiffMatchPatch) patchMake2(text1 string, diffs []Diff) []Patch { + // Check for null inputs not needed since null can't be passed in C#. + patches := []Patch{} + if len(diffs) == 0 { + return patches // Get rid of the null case. + } + + patch := Patch{} + charCount1 := 0 // Number of characters into the text1 string. + charCount2 := 0 // Number of characters into the text2 string. + // Start with text1 (prepatchText) and apply the diffs until we arrive at text2 (postpatchText). We recreate the patches one by one to determine context info. + prepatchText := text1 + postpatchText := text1 + + for i, aDiff := range diffs { + if len(patch.diffs) == 0 && aDiff.Type != DiffEqual { + // A new patch starts here. + patch.Start1 = charCount1 + patch.Start2 = charCount2 + } + + switch aDiff.Type { + case DiffInsert: + patch.diffs = append(patch.diffs, aDiff) + patch.Length2 += len(aDiff.Text) + postpatchText = postpatchText[:charCount2] + + aDiff.Text + postpatchText[charCount2:] + case DiffDelete: + patch.Length1 += len(aDiff.Text) + patch.diffs = append(patch.diffs, aDiff) + postpatchText = postpatchText[:charCount2] + postpatchText[charCount2+len(aDiff.Text):] + case DiffEqual: + if len(aDiff.Text) <= 2*dmp.PatchMargin && + len(patch.diffs) != 0 && i != len(diffs)-1 { + // Small equality inside a patch. + patch.diffs = append(patch.diffs, aDiff) + patch.Length1 += len(aDiff.Text) + patch.Length2 += len(aDiff.Text) + } + if len(aDiff.Text) >= 2*dmp.PatchMargin { + // Time for a new patch. + if len(patch.diffs) != 0 { + patch = dmp.PatchAddContext(patch, prepatchText) + patches = append(patches, patch) + patch = Patch{} + // Unlike Unidiff, our patch lists have a rolling context. http://code.google.com/p/google-diff-match-patch/wiki/Unidiff Update prepatch text & pos to reflect the application of the just completed patch. + prepatchText = postpatchText + charCount1 = charCount2 + } + } + } + + // Update the current character count. + if aDiff.Type != DiffInsert { + charCount1 += len(aDiff.Text) + } + if aDiff.Type != DiffDelete { + charCount2 += len(aDiff.Text) + } + } + + // Pick up the leftover patch if not empty. + if len(patch.diffs) != 0 { + patch = dmp.PatchAddContext(patch, prepatchText) + patches = append(patches, patch) + } + + return patches +} + +// PatchDeepCopy returns an array that is identical to a given an array of patches. +func (dmp *DiffMatchPatch) PatchDeepCopy(patches []Patch) []Patch { + patchesCopy := []Patch{} + for _, aPatch := range patches { + patchCopy := Patch{} + for _, aDiff := range aPatch.diffs { + patchCopy.diffs = append(patchCopy.diffs, Diff{ + aDiff.Type, + aDiff.Text, + }) + } + patchCopy.Start1 = aPatch.Start1 + patchCopy.Start2 = aPatch.Start2 + patchCopy.Length1 = aPatch.Length1 + patchCopy.Length2 = aPatch.Length2 + patchesCopy = append(patchesCopy, patchCopy) + } + return patchesCopy +} + +// PatchApply merges a set of patches onto the text. Returns a patched text, as well as an array of true/false values indicating which patches were applied. +func (dmp *DiffMatchPatch) PatchApply(patches []Patch, text string) (string, []bool) { + if len(patches) == 0 { + return text, []bool{} + } + + // Deep copy the patches so that no changes are made to originals. + patches = dmp.PatchDeepCopy(patches) + + nullPadding := dmp.PatchAddPadding(patches) + text = nullPadding + text + nullPadding + patches = dmp.PatchSplitMax(patches) + + x := 0 + // delta keeps track of the offset between the expected and actual location of the previous patch. If there are patches expected at positions 10 and 20, but the first patch was found at 12, delta is 2 and the second patch has an effective expected position of 22. + delta := 0 + results := make([]bool, len(patches)) + for _, aPatch := range patches { + expectedLoc := aPatch.Start2 + delta + text1 := dmp.DiffText1(aPatch.diffs) + var startLoc int + endLoc := -1 + if len(text1) > dmp.MatchMaxBits { + // PatchSplitMax will only provide an oversized pattern in the case of a monster delete. + startLoc = dmp.MatchMain(text, text1[:dmp.MatchMaxBits], expectedLoc) + if startLoc != -1 { + endLoc = dmp.MatchMain(text, + text1[len(text1)-dmp.MatchMaxBits:], expectedLoc+len(text1)-dmp.MatchMaxBits) + if endLoc == -1 || startLoc >= endLoc { + // Can't find valid trailing context. Drop this patch. + startLoc = -1 + } + } + } else { + startLoc = dmp.MatchMain(text, text1, expectedLoc) + } + if startLoc == -1 { + // No match found. :( + results[x] = false + // Subtract the delta for this failed patch from subsequent patches. + delta -= aPatch.Length2 - aPatch.Length1 + } else { + // Found a match. :) + results[x] = true + delta = startLoc - expectedLoc + var text2 string + if endLoc == -1 { + text2 = text[startLoc:int(math.Min(float64(startLoc+len(text1)), float64(len(text))))] + } else { + text2 = text[startLoc:int(math.Min(float64(endLoc+dmp.MatchMaxBits), float64(len(text))))] + } + if text1 == text2 { + // Perfect match, just shove the Replacement text in. + text = text[:startLoc] + dmp.DiffText2(aPatch.diffs) + text[startLoc+len(text1):] + } else { + // Imperfect match. Run a diff to get a framework of equivalent indices. + diffs := dmp.DiffMain(text1, text2, false) + if len(text1) > dmp.MatchMaxBits && float64(dmp.DiffLevenshtein(diffs))/float64(len(text1)) > dmp.PatchDeleteThreshold { + // The end points match, but the content is unacceptably bad. + results[x] = false + } else { + diffs = dmp.DiffCleanupSemanticLossless(diffs) + index1 := 0 + for _, aDiff := range aPatch.diffs { + if aDiff.Type != DiffEqual { + index2 := dmp.DiffXIndex(diffs, index1) + if aDiff.Type == DiffInsert { + // Insertion + text = text[:startLoc+index2] + aDiff.Text + text[startLoc+index2:] + } else if aDiff.Type == DiffDelete { + // Deletion + startIndex := startLoc + index2 + text = text[:startIndex] + + text[startIndex+dmp.DiffXIndex(diffs, index1+len(aDiff.Text))-index2:] + } + } + if aDiff.Type != DiffDelete { + index1 += len(aDiff.Text) + } + } + } + } + } + x++ + } + // Strip the padding off. + text = text[len(nullPadding) : len(nullPadding)+(len(text)-2*len(nullPadding))] + return text, results +} + +// PatchAddPadding adds some padding on text start and end so that edges can match something. +// Intended to be called only from within patchApply. +func (dmp *DiffMatchPatch) PatchAddPadding(patches []Patch) string { + paddingLength := dmp.PatchMargin + nullPadding := "" + for x := 1; x <= paddingLength; x++ { + nullPadding += string(x) + } + + // Bump all the patches forward. + for i := range patches { + patches[i].Start1 += paddingLength + patches[i].Start2 += paddingLength + } + + // Add some padding on start of first diff. + if len(patches[0].diffs) == 0 || patches[0].diffs[0].Type != DiffEqual { + // Add nullPadding equality. + patches[0].diffs = append([]Diff{Diff{DiffEqual, nullPadding}}, patches[0].diffs...) + patches[0].Start1 -= paddingLength // Should be 0. + patches[0].Start2 -= paddingLength // Should be 0. + patches[0].Length1 += paddingLength + patches[0].Length2 += paddingLength + } else if paddingLength > len(patches[0].diffs[0].Text) { + // Grow first equality. + extraLength := paddingLength - len(patches[0].diffs[0].Text) + patches[0].diffs[0].Text = nullPadding[len(patches[0].diffs[0].Text):] + patches[0].diffs[0].Text + patches[0].Start1 -= extraLength + patches[0].Start2 -= extraLength + patches[0].Length1 += extraLength + patches[0].Length2 += extraLength + } + + // Add some padding on end of last diff. + last := len(patches) - 1 + if len(patches[last].diffs) == 0 || patches[last].diffs[len(patches[last].diffs)-1].Type != DiffEqual { + // Add nullPadding equality. + patches[last].diffs = append(patches[last].diffs, Diff{DiffEqual, nullPadding}) + patches[last].Length1 += paddingLength + patches[last].Length2 += paddingLength + } else if paddingLength > len(patches[last].diffs[len(patches[last].diffs)-1].Text) { + // Grow last equality. + lastDiff := patches[last].diffs[len(patches[last].diffs)-1] + extraLength := paddingLength - len(lastDiff.Text) + patches[last].diffs[len(patches[last].diffs)-1].Text += nullPadding[:extraLength] + patches[last].Length1 += extraLength + patches[last].Length2 += extraLength + } + + return nullPadding +} + +// PatchSplitMax looks through the patches and breaks up any which are longer than the maximum limit of the match algorithm. +// Intended to be called only from within patchApply. +func (dmp *DiffMatchPatch) PatchSplitMax(patches []Patch) []Patch { + patchSize := dmp.MatchMaxBits + for x := 0; x < len(patches); x++ { + if patches[x].Length1 <= patchSize { + continue + } + bigpatch := patches[x] + // Remove the big old patch. + patches = append(patches[:x], patches[x+1:]...) + x-- + + Start1 := bigpatch.Start1 + Start2 := bigpatch.Start2 + precontext := "" + for len(bigpatch.diffs) != 0 { + // Create one of several smaller patches. + patch := Patch{} + empty := true + patch.Start1 = Start1 - len(precontext) + patch.Start2 = Start2 - len(precontext) + if len(precontext) != 0 { + patch.Length1 = len(precontext) + patch.Length2 = len(precontext) + patch.diffs = append(patch.diffs, Diff{DiffEqual, precontext}) + } + for len(bigpatch.diffs) != 0 && patch.Length1 < patchSize-dmp.PatchMargin { + diffType := bigpatch.diffs[0].Type + diffText := bigpatch.diffs[0].Text + if diffType == DiffInsert { + // Insertions are harmless. + patch.Length2 += len(diffText) + Start2 += len(diffText) + patch.diffs = append(patch.diffs, bigpatch.diffs[0]) + bigpatch.diffs = bigpatch.diffs[1:] + empty = false + } else if diffType == DiffDelete && len(patch.diffs) == 1 && patch.diffs[0].Type == DiffEqual && len(diffText) > 2*patchSize { + // This is a large deletion. Let it pass in one chunk. + patch.Length1 += len(diffText) + Start1 += len(diffText) + empty = false + patch.diffs = append(patch.diffs, Diff{diffType, diffText}) + bigpatch.diffs = bigpatch.diffs[1:] + } else { + // Deletion or equality. Only take as much as we can stomach. + diffText = diffText[:min(len(diffText), patchSize-patch.Length1-dmp.PatchMargin)] + + patch.Length1 += len(diffText) + Start1 += len(diffText) + if diffType == DiffEqual { + patch.Length2 += len(diffText) + Start2 += len(diffText) + } else { + empty = false + } + patch.diffs = append(patch.diffs, Diff{diffType, diffText}) + if diffText == bigpatch.diffs[0].Text { + bigpatch.diffs = bigpatch.diffs[1:] + } else { + bigpatch.diffs[0].Text = + bigpatch.diffs[0].Text[len(diffText):] + } + } + } + // Compute the head context for the next patch. + precontext = dmp.DiffText2(patch.diffs) + precontext = precontext[max(0, len(precontext)-dmp.PatchMargin):] + + postcontext := "" + // Append the end context for this patch. + if len(dmp.DiffText1(bigpatch.diffs)) > dmp.PatchMargin { + postcontext = dmp.DiffText1(bigpatch.diffs)[:dmp.PatchMargin] + } else { + postcontext = dmp.DiffText1(bigpatch.diffs) + } + + if len(postcontext) != 0 { + patch.Length1 += len(postcontext) + patch.Length2 += len(postcontext) + if len(patch.diffs) != 0 && patch.diffs[len(patch.diffs)-1].Type == DiffEqual { + patch.diffs[len(patch.diffs)-1].Text += postcontext + } else { + patch.diffs = append(patch.diffs, Diff{DiffEqual, postcontext}) + } + } + if !empty { + x++ + patches = append(patches[:x], append([]Patch{patch}, patches[x:]...)...) + } + } + } + return patches +} + +// PatchToText takes a list of patches and returns a textual representation. +func (dmp *DiffMatchPatch) PatchToText(patches []Patch) string { + var text bytes.Buffer + for _, aPatch := range patches { + _, _ = text.WriteString(aPatch.String()) + } + return text.String() +} + +// PatchFromText parses a textual representation of patches and returns a List of Patch objects. +func (dmp *DiffMatchPatch) PatchFromText(textline string) ([]Patch, error) { + patches := []Patch{} + if len(textline) == 0 { + return patches, nil + } + text := strings.Split(textline, "\n") + textPointer := 0 + patchHeader := regexp.MustCompile("^@@ -(\\d+),?(\\d*) \\+(\\d+),?(\\d*) @@$") + + var patch Patch + var sign uint8 + var line string + for textPointer < len(text) { + + if !patchHeader.MatchString(text[textPointer]) { + return patches, errors.New("Invalid patch string: " + text[textPointer]) + } + + patch = Patch{} + m := patchHeader.FindStringSubmatch(text[textPointer]) + + patch.Start1, _ = strconv.Atoi(m[1]) + if len(m[2]) == 0 { + patch.Start1-- + patch.Length1 = 1 + } else if m[2] == "0" { + patch.Length1 = 0 + } else { + patch.Start1-- + patch.Length1, _ = strconv.Atoi(m[2]) + } + + patch.Start2, _ = strconv.Atoi(m[3]) + + if len(m[4]) == 0 { + patch.Start2-- + patch.Length2 = 1 + } else if m[4] == "0" { + patch.Length2 = 0 + } else { + patch.Start2-- + patch.Length2, _ = strconv.Atoi(m[4]) + } + textPointer++ + + for textPointer < len(text) { + if len(text[textPointer]) > 0 { + sign = text[textPointer][0] + } else { + textPointer++ + continue + } + + line = text[textPointer][1:] + line = strings.Replace(line, "+", "%2b", -1) + line, _ = url.QueryUnescape(line) + if sign == '-' { + // Deletion. + patch.diffs = append(patch.diffs, Diff{DiffDelete, line}) + } else if sign == '+' { + // Insertion. + patch.diffs = append(patch.diffs, Diff{DiffInsert, line}) + } else if sign == ' ' { + // Minor equality. + patch.diffs = append(patch.diffs, Diff{DiffEqual, line}) + } else if sign == '@' { + // Start of next patch. + break + } else { + // WTF? + return patches, errors.New("Invalid patch mode '" + string(sign) + "' in: " + string(line)) + } + textPointer++ + } + + patches = append(patches, patch) + } + return patches, nil +} diff --git a/vendor/github.com/sergi/go-diff/diffmatchpatch/stringutil.go b/vendor/github.com/sergi/go-diff/diffmatchpatch/stringutil.go new file mode 100644 index 00000000000..265f29cc7e5 --- /dev/null +++ b/vendor/github.com/sergi/go-diff/diffmatchpatch/stringutil.go @@ -0,0 +1,88 @@ +// Copyright (c) 2012-2016 The go-diff authors. All rights reserved. +// https://github.com/sergi/go-diff +// See the included LICENSE file for license details. +// +// go-diff is a Go implementation of Google's Diff, Match, and Patch library +// Original library is Copyright (c) 2006 Google Inc. +// http://code.google.com/p/google-diff-match-patch/ + +package diffmatchpatch + +import ( + "strings" + "unicode/utf8" +) + +// unescaper unescapes selected chars for compatibility with JavaScript's encodeURI. +// In speed critical applications this could be dropped since the receiving application will certainly decode these fine. Note that this function is case-sensitive. Thus "%3F" would not be unescaped. But this is ok because it is only called with the output of HttpUtility.UrlEncode which returns lowercase hex. Example: "%3f" -> "?", "%24" -> "$", etc. +var unescaper = strings.NewReplacer( + "%21", "!", "%7E", "~", "%27", "'", + "%28", "(", "%29", ")", "%3B", ";", + "%2F", "/", "%3F", "?", "%3A", ":", + "%40", "@", "%26", "&", "%3D", "=", + "%2B", "+", "%24", "$", "%2C", ",", "%23", "#", "%2A", "*") + +// indexOf returns the first index of pattern in str, starting at str[i]. +func indexOf(str string, pattern string, i int) int { + if i > len(str)-1 { + return -1 + } + if i <= 0 { + return strings.Index(str, pattern) + } + ind := strings.Index(str[i:], pattern) + if ind == -1 { + return -1 + } + return ind + i +} + +// lastIndexOf returns the last index of pattern in str, starting at str[i]. +func lastIndexOf(str string, pattern string, i int) int { + if i < 0 { + return -1 + } + if i >= len(str) { + return strings.LastIndex(str, pattern) + } + _, size := utf8.DecodeRuneInString(str[i:]) + return strings.LastIndex(str[:i+size], pattern) +} + +// runesIndexOf returns the index of pattern in target, starting at target[i]. +func runesIndexOf(target, pattern []rune, i int) int { + if i > len(target)-1 { + return -1 + } + if i <= 0 { + return runesIndex(target, pattern) + } + ind := runesIndex(target[i:], pattern) + if ind == -1 { + return -1 + } + return ind + i +} + +func runesEqual(r1, r2 []rune) bool { + if len(r1) != len(r2) { + return false + } + for i, c := range r1 { + if c != r2[i] { + return false + } + } + return true +} + +// runesIndex is the equivalent of strings.Index for rune slices. +func runesIndex(r1, r2 []rune) int { + last := len(r1) - len(r2) + for i := 0; i <= last; i++ { + if runesEqual(r1[i:i+len(r2)], r2) { + return i + } + } + return -1 +} From 8f0150b3f089d74ffaeb717a017c54ac6f45e172 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Mon, 11 Feb 2019 18:33:47 +0100 Subject: [PATCH 09/26] Fix bugs in tests --- commands/root/root.go | 3 --- configs/navigate.go | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/commands/root/root.go b/commands/root/root.go index d7a574975b0..dd91667ad93 100644 --- a/commands/root/root.go +++ b/commands/root/root.go @@ -18,7 +18,6 @@ package root import ( - "fmt" "io/ioutil" "os" "path/filepath" @@ -134,8 +133,6 @@ func initConfigs() { commands.Config.Navigate("/", pwd) - fmt.Println(yamlConfigFile) - if yamlConfigFile != "" { commands.Config.ConfigFile = paths.New(yamlConfigFile) if err := commands.Config.LoadFromYAML(commands.Config.ConfigFile); err != nil { diff --git a/configs/navigate.go b/configs/navigate.go index 734c7c1f8f6..416a1b6035e 100644 --- a/configs/navigate.go +++ b/configs/navigate.go @@ -10,7 +10,7 @@ import ( func (c *Configuration) Navigate(root, pwd string) { relativePath, err := filepath.Rel(root, pwd) if err != nil { - panic(err) + return } // From the root to the current folder, search for arduino-cli.yaml files From 5f9cff0734d5ab9f8d579794b17f810fb03cd0c8 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Wed, 13 Feb 2019 14:00:27 +0100 Subject: [PATCH 10/26] Read default config file (in .arduino15) --- commands/root/root.go | 1 + 1 file changed, 1 insertion(+) diff --git a/commands/root/root.go b/commands/root/root.go index dd91667ad93..176b27e30ea 100644 --- a/commands/root/root.go +++ b/commands/root/root.go @@ -132,6 +132,7 @@ func initConfigs() { } commands.Config.Navigate("/", pwd) + commands.Config.LoadFromYAML(commands.Config.ConfigFile) if yamlConfigFile != "" { commands.Config.ConfigFile = paths.New(yamlConfigFile) From 8659dd2683e6e7681c5ef1526db3fc979fc6a90a Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Wed, 13 Feb 2019 14:03:18 +0100 Subject: [PATCH 11/26] Rename configuration file --- .gitignore | 2 +- README.md | 4 ++-- commands/config/init.go | 2 +- configs/directories.go | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index f73247cad87..0dd0591bbc0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,5 @@ /main /.vscode/settings.json /cmd/formatter/debug.test -/.cli-config.yml +/arduino-cli.yaml /wiki diff --git a/README.md b/README.md index ebfa59f0e5b..191c36b944e 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ Great! Now we have the Board FQBN (Fully Qualified Board Name) `arduino:samd:mkr and the Board Name look good, we are ready to compile and upload the sketch #### Adding 3rd party cores -To add 3rd party core packages add a link of the additional package to the file `.cli-config.yml` +To add 3rd party core packages add a link of the additional package to the file `arduino-cli.yaml` If you want to add the ESP8266 core, for example: @@ -295,7 +295,7 @@ Flags: -h, --help help for core Global Flags: - --config-file string The custom config file (if not specified the default one will be used). (example "/home/megabug/.config/arduino/arduino-cli/.cli-config.yml") + --config-file string The custom config file (if not specified the default one will be used). (example "/home/megabug/.config/arduino/arduino-cli/arduino-cli.yaml") --debug Enables debug output (super verbose, used to debug the CLI). --format string The output format, can be [text|json]. (default "text") diff --git a/commands/config/init.go b/commands/config/init.go index 914ee57b081..de7819b7c53 100644 --- a/commands/config/init.go +++ b/commands/config/init.go @@ -42,7 +42,7 @@ func initInitCommand() *cobra.Command { initCommand.Flags().BoolVar(&initFlags._default, "default", false, "If omitted, ask questions to the user about setting configuration properties, otherwise use default configuration.") initCommand.Flags().StringVar(&initFlags.location, "save-as", "", - "Sets where to save the configuration file [default is ./.cli-config.yml].") + "Sets where to save the configuration file [default is ./arduino-cli.yaml].") return initCommand } diff --git a/configs/directories.go b/configs/directories.go index 43e295e17da..8284a786e68 100644 --- a/configs/directories.go +++ b/configs/directories.go @@ -27,7 +27,7 @@ import ( "github.com/shibukawa/configdir" ) -// getDefaultConfigFilePath returns the default path for .cli-config.yml. It searches the following directories for an existing .cli-config.yml file: +// getDefaultConfigFilePath returns the default path for arduino-cli.yaml. It searches the following directories for an existing arduino-cli.yaml file: // - User level configuration folder(e.g. $HOME/.config///setting.json in Linux) // - System level configuration folder(e.g. /etc/xdg///setting.json in Linux) // If it doesn't find one, it defaults to the user level configuration folder @@ -35,13 +35,13 @@ func getDefaultConfigFilePath() *paths.Path { configDirs := configdir.New("arduino", "arduino-cli") // Search for a suitable configuration file - path := configDirs.QueryFolderContainsFile(".cli-config.yml") + path := configDirs.QueryFolderContainsFile("arduino-cli.yaml") if path != nil { - return paths.New(path.Path, ".cli-config.yml") + return paths.New(path.Path, "arduino-cli.yaml") } // Default to the global configuration locals := configDirs.QueryFolders(configdir.Global) - return paths.New(locals[0].Path, ".cli-config.yml") + return paths.New(locals[0].Path, "arduino-cli.yaml") return nil } From f9bf3a8dd52c5fff37481b428635cefbe0765fe4 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Wed, 13 Feb 2019 14:30:34 +0100 Subject: [PATCH 12/26] Remove configdir --- Gopkg.lock | 9 - configs/directories.go | 34 ++-- vendor/github.com/shibukawa/configdir/LICENSE | 21 --- .../github.com/shibukawa/configdir/README.rst | 111 ------------ .../github.com/shibukawa/configdir/config.go | 160 ------------------ .../shibukawa/configdir/config_darwin.go | 8 - .../shibukawa/configdir/config_windows.go | 8 - .../shibukawa/configdir/config_xdg.go | 34 ---- 8 files changed, 20 insertions(+), 365 deletions(-) delete mode 100644 vendor/github.com/shibukawa/configdir/LICENSE delete mode 100644 vendor/github.com/shibukawa/configdir/README.rst delete mode 100644 vendor/github.com/shibukawa/configdir/config.go delete mode 100644 vendor/github.com/shibukawa/configdir/config_darwin.go delete mode 100644 vendor/github.com/shibukawa/configdir/config_windows.go delete mode 100644 vendor/github.com/shibukawa/configdir/config_xdg.go diff --git a/Gopkg.lock b/Gopkg.lock index 6cf71918bbc..c313b219953 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -274,14 +274,6 @@ revision = "1744e2970ca51c86172c8190fadad617561ed6e7" version = "v1.0.0" -[[projects]] - branch = "master" - digest = "1:8209ed8bf2336848aa760c11e809f93ba96fffa64d3c2948e970a362a46e534b" - name = "github.com/shibukawa/configdir" - packages = ["."] - pruneopts = "UT" - revision = "e180dbdc8da04c4fa04272e875ce64949f38bd3e" - [[projects]] digest = "1:9e9193aa51197513b3abcb108970d831fbcf40ef96aa845c4f03276e1fa316d2" name = "github.com/sirupsen/logrus" @@ -458,7 +450,6 @@ "github.com/pkg/errors", "github.com/pmylund/sortutil", "github.com/sergi/go-diff/diffmatchpatch", - "github.com/shibukawa/configdir", "github.com/sirupsen/logrus", "github.com/spf13/cobra", "github.com/spf13/cobra/doc", diff --git a/configs/directories.go b/configs/directories.go index 8284a786e68..1e04dbfde2b 100644 --- a/configs/directories.go +++ b/configs/directories.go @@ -24,26 +24,32 @@ import ( "github.com/arduino/go-paths-helper" "github.com/arduino/go-win32-utils" - "github.com/shibukawa/configdir" ) -// getDefaultConfigFilePath returns the default path for arduino-cli.yaml. It searches the following directories for an existing arduino-cli.yaml file: -// - User level configuration folder(e.g. $HOME/.config///setting.json in Linux) -// - System level configuration folder(e.g. /etc/xdg///setting.json in Linux) -// If it doesn't find one, it defaults to the user level configuration folder +// getDefaultConfigFilePath returns the default path for arduino-cli.yaml func getDefaultConfigFilePath() *paths.Path { - configDirs := configdir.New("arduino", "arduino-cli") + usr, err := user.Current() + if err != nil { + panic(fmt.Errorf("retrieving user home dir: %s", err)) + } + arduinoDataDir := paths.New(usr.HomeDir) - // Search for a suitable configuration file - path := configDirs.QueryFolderContainsFile("arduino-cli.yaml") - if path != nil { - return paths.New(path.Path, "arduino-cli.yaml") + switch runtime.GOOS { + case "linux": + arduinoDataDir = arduinoDataDir.Join(".arduino15") + case "darwin": + arduinoDataDir = arduinoDataDir.Join("Library", "arduino15") + case "windows": + localAppDataPath, err := win32.GetLocalAppDataFolder() + if err != nil { + panic(err) + } + arduinoDataDir = paths.New(localAppDataPath).Join("Arduino15") + default: + panic(fmt.Errorf("unsupported OS: %s", runtime.GOOS)) } - // Default to the global configuration - locals := configDirs.QueryFolders(configdir.Global) - return paths.New(locals[0].Path, "arduino-cli.yaml") - return nil + return arduinoDataDir } func getDefaultArduinoDataDir() (*paths.Path, error) { diff --git a/vendor/github.com/shibukawa/configdir/LICENSE b/vendor/github.com/shibukawa/configdir/LICENSE deleted file mode 100644 index b20af456a19..00000000000 --- a/vendor/github.com/shibukawa/configdir/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2016 shibukawa - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/github.com/shibukawa/configdir/README.rst b/vendor/github.com/shibukawa/configdir/README.rst deleted file mode 100644 index 99906697da3..00000000000 --- a/vendor/github.com/shibukawa/configdir/README.rst +++ /dev/null @@ -1,111 +0,0 @@ -configdir for Golang -===================== - -Multi platform library of configuration directory for Golang. - -This library helps to get regular directories for configuration files or cache files that matches target operationg system's convention. - -It assumes the following folders are standard paths of each environment: - -.. list-table:: - :header-rows: 1 - - - * - * Windows: - * Linux/BSDs: - * MacOSX: - - * System level configuration folder - * ``%PROGRAMDATA%`` (``C:\\ProgramData``) - * ``${XDG_CONFIG_DIRS}`` (``/etc/xdg``) - * ``/Library/Application Support`` - - * User level configuration folder - * ``%APPDATA%`` (``C:\\Users\\\\AppData\\Roaming``) - * ``${XDG_CONFIG_HOME}`` (``${HOME}/.config``) - * ``${HOME}/Library/Application Support`` - - * User wide cache folder - * ``%LOCALAPPDATA%`` ``(C:\\Users\\\\AppData\\Local)`` - * ``${XDG_CACHE_HOME}`` (``${HOME}/.cache``) - * ``${HOME}/Library/Caches`` - -Examples ------------- - -Getting Configuration -~~~~~~~~~~~~~~~~~~~~~~~~ - -``configdir.ConfigDir.QueryFolderContainsFile()`` searches files in the following order: - -* Local path (if you add the path via LocalPath parameter) -* User level configuration folder(e.g. ``$HOME/.config///setting.json`` in Linux) -* System level configuration folder(e.g. ``/etc/xdg///setting.json`` in Linux) - -``configdir.Config`` provides some convenient methods(``ReadFile``, ``WriteFile`` and so on). - -.. code-block:: go - - var config Config - - configDirs := configdir.New("vendor-name", "application-name") - // optional: local path has the highest priority - configDirs.LocalPath, _ = filepath.Abs(".") - folder := configDirs.QueryFolderContainsFile("setting.json") - if folder != nil { - data, _ := folder.ReadFile("setting.json") - json.Unmarshal(data, &config) - } else { - config = DefaultConfig - } - -Write Configuration -~~~~~~~~~~~~~~~~~~~~~~ - -When storing configuration, get configuration folder by using ``configdir.ConfigDir.QueryFolders()`` method. - -.. code-block:: go - - configDirs := configdir.New("vendor-name", "application-name") - - var config Config - data, _ := json.Marshal(&config) - - // Stores to local folder - folders := configDirs.QueryFolders(configdir.Local) - folders[0].WriteFile("setting.json", data) - - // Stores to user folder - folders = configDirs.QueryFolders(configdir.Global) - folders[0].WriteFile("setting.json", data) - - // Stores to system folder - folders = configDirs.QueryFolders(configdir.System) - folders[0].WriteFile("setting.json", data) - -Getting Cache Folder -~~~~~~~~~~~~~~~~~~~~~~ - -It is similar to the above example, but returns cache folder. - -.. code-block:: go - - configDirs := configdir.New("vendor-name", "application-name") - cache := configDirs.QueryCacheFolder() - - resp, err := http.Get("http://examples.com/sdk.zip") - if err != nil { - log.Fatal(err) - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - - cache.WriteFile("sdk.zip", body) - -Document ------------- - -https://godoc.org/github.com/shibukawa/configdir - -License ------------- - -MIT - diff --git a/vendor/github.com/shibukawa/configdir/config.go b/vendor/github.com/shibukawa/configdir/config.go deleted file mode 100644 index 8a20e54b59a..00000000000 --- a/vendor/github.com/shibukawa/configdir/config.go +++ /dev/null @@ -1,160 +0,0 @@ -// configdir provides access to configuration folder in each platforms. -// -// System wide configuration folders: -// -// - Windows: %PROGRAMDATA% (C:\ProgramData) -// - Linux/BSDs: ${XDG_CONFIG_DIRS} (/etc/xdg) -// - MacOSX: "/Library/Application Support" -// -// User wide configuration folders: -// -// - Windows: %APPDATA% (C:\Users\\AppData\Roaming) -// - Linux/BSDs: ${XDG_CONFIG_HOME} (${HOME}/.config) -// - MacOSX: "${HOME}/Library/Application Support" -// -// User wide cache folders: -// -// - Windows: %LOCALAPPDATA% (C:\Users\\AppData\Local) -// - Linux/BSDs: ${XDG_CACHE_HOME} (${HOME}/.cache) -// - MacOSX: "${HOME}/Library/Caches" -// -// configdir returns paths inside the above folders. - -package configdir - -import ( - "io/ioutil" - "os" - "path/filepath" -) - -type ConfigType int - -const ( - System ConfigType = iota - Global - All - Existing - Local - Cache -) - -// Config represents each folder -type Config struct { - Path string - Type ConfigType -} - -func (c Config) Open(fileName string) (*os.File, error) { - return os.Open(filepath.Join(c.Path, fileName)) -} - -func (c Config) Create(fileName string) (*os.File, error) { - err := c.CreateParentDir(fileName) - if err != nil { - return nil, err - } - return os.Create(filepath.Join(c.Path, fileName)) -} - -func (c Config) ReadFile(fileName string) ([]byte, error) { - return ioutil.ReadFile(filepath.Join(c.Path, fileName)) -} - -// CreateParentDir creates the parent directory of fileName inside c. fileName -// is a relative path inside c, containing zero or more path separators. -func (c Config) CreateParentDir(fileName string) error { - return os.MkdirAll(filepath.Dir(filepath.Join(c.Path, fileName)), 0755) -} - -func (c Config) WriteFile(fileName string, data []byte) error { - err := c.CreateParentDir(fileName) - if err != nil { - return err - } - return ioutil.WriteFile(filepath.Join(c.Path, fileName), data, 0644) -} - -func (c Config) MkdirAll() error { - return os.MkdirAll(c.Path, 0755) -} - -func (c Config) Exists(fileName string) bool { - _, err := os.Stat(filepath.Join(c.Path, fileName)) - return !os.IsNotExist(err) -} - -// ConfigDir keeps setting for querying folders. -type ConfigDir struct { - VendorName string - ApplicationName string - LocalPath string -} - -func New(vendorName, applicationName string) ConfigDir { - return ConfigDir{ - VendorName: vendorName, - ApplicationName: applicationName, - } -} - -func (c ConfigDir) joinPath(root string) string { - if c.VendorName != "" && hasVendorName { - return filepath.Join(root, c.VendorName, c.ApplicationName) - } - return filepath.Join(root, c.ApplicationName) -} - -func (c ConfigDir) QueryFolders(configType ConfigType) []*Config { - if configType == Cache { - return []*Config{c.QueryCacheFolder()} - } - var result []*Config - if c.LocalPath != "" && configType != System && configType != Global { - result = append(result, &Config{ - Path: c.LocalPath, - Type: Local, - }) - } - if configType != System && configType != Local { - result = append(result, &Config{ - Path: c.joinPath(globalSettingFolder), - Type: Global, - }) - } - if configType != Global && configType != Local { - for _, root := range systemSettingFolders { - result = append(result, &Config{ - Path: c.joinPath(root), - Type: System, - }) - } - } - if configType != Existing { - return result - } - var existing []*Config - for _, entry := range result { - if _, err := os.Stat(entry.Path); !os.IsNotExist(err) { - existing = append(existing, entry) - } - } - return existing -} - -func (c ConfigDir) QueryFolderContainsFile(fileName string) *Config { - configs := c.QueryFolders(Existing) - for _, config := range configs { - if _, err := os.Stat(filepath.Join(config.Path, fileName)); !os.IsNotExist(err) { - return config - } - } - return nil -} - -func (c ConfigDir) QueryCacheFolder() *Config { - return &Config{ - Path: c.joinPath(cacheFolder), - Type: Cache, - } -} diff --git a/vendor/github.com/shibukawa/configdir/config_darwin.go b/vendor/github.com/shibukawa/configdir/config_darwin.go deleted file mode 100644 index d668507a7e3..00000000000 --- a/vendor/github.com/shibukawa/configdir/config_darwin.go +++ /dev/null @@ -1,8 +0,0 @@ -package configdir - -import "os" - -var hasVendorName = true -var systemSettingFolders = []string{"/Library/Application Support"} -var globalSettingFolder = os.Getenv("HOME") + "/Library/Application Support" -var cacheFolder = os.Getenv("HOME") + "/Library/Caches" diff --git a/vendor/github.com/shibukawa/configdir/config_windows.go b/vendor/github.com/shibukawa/configdir/config_windows.go deleted file mode 100644 index 0984821778d..00000000000 --- a/vendor/github.com/shibukawa/configdir/config_windows.go +++ /dev/null @@ -1,8 +0,0 @@ -package configdir - -import "os" - -var hasVendorName = true -var systemSettingFolders = []string{os.Getenv("PROGRAMDATA")} -var globalSettingFolder = os.Getenv("APPDATA") -var cacheFolder = os.Getenv("LOCALAPPDATA") diff --git a/vendor/github.com/shibukawa/configdir/config_xdg.go b/vendor/github.com/shibukawa/configdir/config_xdg.go deleted file mode 100644 index 026ca68a0bb..00000000000 --- a/vendor/github.com/shibukawa/configdir/config_xdg.go +++ /dev/null @@ -1,34 +0,0 @@ -// +build !windows,!darwin - -package configdir - -import ( - "os" - "path/filepath" - "strings" -) - -// https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html - -var hasVendorName = true -var systemSettingFolders []string -var globalSettingFolder string -var cacheFolder string - -func init() { - if os.Getenv("XDG_CONFIG_HOME") != "" { - globalSettingFolder = os.Getenv("XDG_CONFIG_HOME") - } else { - globalSettingFolder = filepath.Join(os.Getenv("HOME"), ".config") - } - if os.Getenv("XDG_CONFIG_DIRS") != "" { - systemSettingFolders = strings.Split(os.Getenv("XDG_CONFIG_DIRS"), ":") - } else { - systemSettingFolders = []string{"/etc/xdg"} - } - if os.Getenv("XDG_CACHE_HOME") != "" { - cacheFolder = os.Getenv("XDG_CACHE_HOME") - } else { - cacheFolder = filepath.Join(os.Getenv("HOME"), ".cache") - } -} From 392f7cfc7c5f2d23e60099105662714d90f17a86 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Wed, 13 Feb 2019 14:37:00 +0100 Subject: [PATCH 13/26] Consistent usage of paths helper --- commands/config/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/config/init.go b/commands/config/init.go index de7819b7c53..bf7ae667193 100644 --- a/commands/config/init.go +++ b/commands/config/init.go @@ -66,7 +66,7 @@ func runInitCommand(cmd *cobra.Command, args []string) { filepath = commands.Config.ConfigFile.String() } - err := os.MkdirAll(commands.Config.ConfigFile.Parent().String(), 0766) + err := commands.Config.ConfigFile.Parent().MkdirAll() if err != nil { formatter.PrintError(err, "Cannot create config file.") os.Exit(commands.ErrGeneric) From b2e6f47f022f0b799b8ee1a4b24ee55fccc0a433 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Wed, 13 Feb 2019 14:38:36 +0100 Subject: [PATCH 14/26] Add license where missing --- configs/navigate.go | 16 ++++++++++++++++ configs/navigate_test.go | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/configs/navigate.go b/configs/navigate.go index 416a1b6035e..f685f41fef5 100644 --- a/configs/navigate.go +++ b/configs/navigate.go @@ -1,3 +1,19 @@ +/* + * This file is part of arduino-cli. + * + * Copyright 2018 ARDUINO SA (http://www.arduino.cc/) + * + * This software is released under the GNU General Public License version 3, + * which covers the main part of arduino-cli. + * The terms of this license can be found at: + * https://www.gnu.org/licenses/gpl-3.0.en.html + * + * You can be released from the requirements of the above licenses by purchasing + * a commercial license. Buying such a license is mandatory if you want to modify or + * otherwise use the software for commercial activities involving the Arduino + * software without disclosing the source code of your own applications. To purchase + * a commercial license, send an email to license@arduino.cc. + */ package configs import ( diff --git a/configs/navigate_test.go b/configs/navigate_test.go index e819307d8ef..b7eccdd5f35 100644 --- a/configs/navigate_test.go +++ b/configs/navigate_test.go @@ -1,3 +1,19 @@ +/* + * This file is part of arduino-cli. + * + * Copyright 2018 ARDUINO SA (http://www.arduino.cc/) + * + * This software is released under the GNU General Public License version 3, + * which covers the main part of arduino-cli. + * The terms of this license can be found at: + * https://www.gnu.org/licenses/gpl-3.0.en.html + * + * You can be released from the requirements of the above licenses by purchasing + * a commercial license. Buying such a license is mandatory if you want to modify or + * otherwise use the software for commercial activities involving the Arduino + * software without disclosing the source code of your own applications. To purchase + * a commercial license, send an email to license@arduino.cc. + */ package configs_test import ( From 11be6bd4a1b9efffbbec3b52d978718b9f1ad1a8 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Wed, 13 Feb 2019 14:46:13 +0100 Subject: [PATCH 15/26] Don't show the default package index --- configs/yaml_serializer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/yaml_serializer.go b/configs/yaml_serializer.go index c7ff2b9060f..4125eb7e9ae 100644 --- a/configs/yaml_serializer.go +++ b/configs/yaml_serializer.go @@ -109,9 +109,9 @@ func (config *Configuration) SerializeToYAML() ([]byte, error) { Password: config.ProxyPassword, } } - if len(config.BoardManagerAdditionalUrls) > 0 { - c.BoardsManager = &yamlBoardsManagerConfig{AdditionalURLS: []string{}} - for _, URL := range config.BoardManagerAdditionalUrls { + c.BoardsManager = &yamlBoardsManagerConfig{AdditionalURLS: []string{}} + if len(config.BoardManagerAdditionalUrls) > 1 { + for _, URL := range config.BoardManagerAdditionalUrls[1:] { c.BoardsManager.AdditionalURLS = appendIfMissing(c.BoardsManager.AdditionalURLS, URL.String()) } } From 930b89ad98c1a6b1a4c96e0ee1371b43e1a699c0 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Wed, 13 Feb 2019 14:49:42 +0100 Subject: [PATCH 16/26] Make linter happy --- configs/navigate.go | 1 + configs/navigate_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/configs/navigate.go b/configs/navigate.go index f685f41fef5..ca77c675e36 100644 --- a/configs/navigate.go +++ b/configs/navigate.go @@ -14,6 +14,7 @@ * software without disclosing the source code of your own applications. To purchase * a commercial license, send an email to license@arduino.cc. */ + package configs import ( diff --git a/configs/navigate_test.go b/configs/navigate_test.go index b7eccdd5f35..a32e1f78415 100644 --- a/configs/navigate_test.go +++ b/configs/navigate_test.go @@ -14,6 +14,7 @@ * software without disclosing the source code of your own applications. To purchase * a commercial license, send an email to license@arduino.cc. */ + package configs_test import ( From 2453397793afe81fd6c6253a79b661f04b275c04 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Wed, 13 Feb 2019 14:56:14 +0100 Subject: [PATCH 17/26] Fix navigate tests --- configs/testdata/navigate/inheritance/golden.yaml | 1 - configs/testdata/navigate/local/golden.yaml | 1 - configs/testdata/navigate/noconfig/golden.yaml | 4 +--- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/configs/testdata/navigate/inheritance/golden.yaml b/configs/testdata/navigate/inheritance/golden.yaml index 6aede490ac9..d960a3ea97f 100644 --- a/configs/testdata/navigate/inheritance/golden.yaml +++ b/configs/testdata/navigate/inheritance/golden.yaml @@ -3,5 +3,4 @@ sketchbook_path: /tmp arduino_data: $HOME/.arduino15 board_manager: additional_urls: - - https://downloads.arduino.cc/packages/package_index.json - https://downloads.arduino.cc/package_index_mraa.json \ No newline at end of file diff --git a/configs/testdata/navigate/local/golden.yaml b/configs/testdata/navigate/local/golden.yaml index 0fc132bfd79..8f29908b0a8 100644 --- a/configs/testdata/navigate/local/golden.yaml +++ b/configs/testdata/navigate/local/golden.yaml @@ -3,5 +3,4 @@ sketchbook_path: $HOME/Arduino arduino_data: $HOME/.arduino15 board_manager: additional_urls: - - https://downloads.arduino.cc/packages/package_index.json - https://downloads.arduino.cc/package_index_mraa.json \ No newline at end of file diff --git a/configs/testdata/navigate/noconfig/golden.yaml b/configs/testdata/navigate/noconfig/golden.yaml index 49d1c3ae825..cbcb7e96dcb 100644 --- a/configs/testdata/navigate/noconfig/golden.yaml +++ b/configs/testdata/navigate/noconfig/golden.yaml @@ -1,6 +1,4 @@ proxy_type: auto sketchbook_path: $HOME/Arduino arduino_data: $HOME/.arduino15 -board_manager: - additional_urls: - - https://downloads.arduino.cc/packages/package_index.json \ No newline at end of file +board_manager: {} \ No newline at end of file From 017316052d472007e2a211a436f26a060ef67363 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Wed, 13 Feb 2019 15:57:26 +0100 Subject: [PATCH 18/26] Return an error message when there's a old configuration file --- commands/root/root.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/commands/root/root.go b/commands/root/root.go index 176b27e30ea..ee543e31219 100644 --- a/commands/root/root.go +++ b/commands/root/root.go @@ -18,6 +18,7 @@ package root import ( + "fmt" "io/ioutil" "os" "path/filepath" @@ -116,6 +117,12 @@ func preRun(cmd *cobra.Command, args []string) { // initConfigs initializes the configuration from the specified file. func initConfigs() { + // Return error if an old configuration file is found + if paths.New(".cli-config.yml").Exist() { + logrus.Error("Old configuration file detected. Ensure you are using the new `arduino-cli.yaml` configuration") + formatter.PrintError(fmt.Errorf("old configuration file detected"), "Ensure you are using the new `arduino-cli.yaml` configuration") + os.Exit(commands.ErrGeneric) + } // Start with default configuration if conf, err := configs.NewConfiguration(); err != nil { logrus.WithError(err).Error("Error creating default configuration") From 470559419cab4f8d77292c99fe6ec9a0b509f74f Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Wed, 13 Feb 2019 16:18:25 +0100 Subject: [PATCH 19/26] Fix path of default config --- configs/directories.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/directories.go b/configs/directories.go index 1e04dbfde2b..4595069656a 100644 --- a/configs/directories.go +++ b/configs/directories.go @@ -49,7 +49,7 @@ func getDefaultConfigFilePath() *paths.Path { panic(fmt.Errorf("unsupported OS: %s", runtime.GOOS)) } - return arduinoDataDir + return arduinoDataDir.Join("arduino-cli.yaml") } func getDefaultArduinoDataDir() (*paths.Path, error) { From 6145d7486465c4ecfac9f2739bdf705032b52687 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 14 Feb 2019 12:23:23 +0100 Subject: [PATCH 20/26] Correct order of preferences loading. Added more logging. --- commands/root/root.go | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/commands/root/root.go b/commands/root/root.go index ee543e31219..23a3102db31 100644 --- a/commands/root/root.go +++ b/commands/root/root.go @@ -27,9 +27,9 @@ import ( "golang.org/x/crypto/ssh/terminal" - "github.com/mattn/go-colorable" + colorable "github.com/mattn/go-colorable" - "github.com/arduino/go-paths-helper" + paths "github.com/arduino/go-paths-helper" "github.com/arduino/arduino-cli/commands" "github.com/arduino/arduino-cli/commands/board" @@ -118,11 +118,15 @@ func preRun(cmd *cobra.Command, args []string) { // initConfigs initializes the configuration from the specified file. func initConfigs() { // Return error if an old configuration file is found - if paths.New(".cli-config.yml").Exist() { - logrus.Error("Old configuration file detected. Ensure you are using the new `arduino-cli.yaml` configuration") - formatter.PrintError(fmt.Errorf("old configuration file detected"), "Ensure you are using the new `arduino-cli.yaml` configuration") + if old := paths.New(".cli-config.yml"); old.Exist() { + logrus.Errorf("Old configuration file detected: %s.", old) + logrus.Info("The name of this file has been changed to `arduino-cli.yaml`, please rename the file fix it.") + formatter.PrintError( + fmt.Errorf("old configuration file detected: %s", old), + "The name of this file has been changed to `arduino-cli.yaml`, please rename the file fix it.") os.Exit(commands.ErrGeneric) } + // Start with default configuration if conf, err := configs.NewConfiguration(); err != nil { logrus.WithError(err).Error("Error creating default configuration") @@ -138,18 +142,14 @@ func initConfigs() { logrus.WithError(err).Warn("Did not manage to find current path") } - commands.Config.Navigate("/", pwd) - commands.Config.LoadFromYAML(commands.Config.ConfigFile) - - if yamlConfigFile != "" { - commands.Config.ConfigFile = paths.New(yamlConfigFile) + // Read configuration from global config file + if commands.Config.ConfigFile.Exist() { + logrus.Infof("Reading configuration from %s", commands.Config.ConfigFile) if err := commands.Config.LoadFromYAML(commands.Config.ConfigFile); err != nil { - logrus.WithError(err).Warn("Did not manage to get config file, using default configuration") + logrus.WithError(err).Warnf("Could not read configuration from %s", commands.Config.ConfigFile) } } - logrus.Info("Initiating configuration") - if commands.Config.IsBundledInDesktopIDE() { logrus.Info("CLI is bundled into the IDE") err := commands.Config.LoadFromDesktopIDEPreferences() @@ -159,6 +159,21 @@ func initConfigs() { } else { logrus.Info("CLI is not bundled into the IDE") } + + // Read configuration from parent folders (project config) + commands.Config.Navigate("/", pwd) + + // Read configuration from environment vars commands.Config.LoadFromEnv() + + // Read configuration from user specified file + if yamlConfigFile != "" { + commands.Config.ConfigFile = paths.New(yamlConfigFile) + logrus.Infof("Reading configuration from %s", commands.Config.ConfigFile) + if err := commands.Config.LoadFromYAML(commands.Config.ConfigFile); err != nil { + logrus.WithError(err).Warnf("Could not read configuration from %s", commands.Config.ConfigFile) + } + } + logrus.Info("Configuration set") } From 3d25175f74ea80ae59714fd503d63c50acbe5af2 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 14 Feb 2019 15:58:39 +0100 Subject: [PATCH 21/26] Updated go-paths-helper --- Gopkg.lock | 4 ++-- .../arduino/go-paths-helper/paths.go | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index c313b219953..98a3c3a4bce 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -30,11 +30,11 @@ [[projects]] branch = "master" - digest = "1:fd2ebfc02b6ad10599b226d2c0265f160e95e7c80e23f01dcf34a8aff0de98c9" + digest = "1:045d5ae3596598b9b9591042561fbcbf2d484d2a744fd007151cab3862a6b8f6" name = "github.com/arduino/go-paths-helper" packages = ["."] pruneopts = "UT" - revision = "751652ddd9f0a98650e681673c2c73937002e889" + revision = "c3c98d1bf2e1069f60ab84bff3a2eb3c5422f3b0" [[projects]] branch = "master" diff --git a/vendor/github.com/arduino/go-paths-helper/paths.go b/vendor/github.com/arduino/go-paths-helper/paths.go index 4a4af393595..a5d11229c8f 100644 --- a/vendor/github.com/arduino/go-paths-helper/paths.go +++ b/vendor/github.com/arduino/go-paths-helper/paths.go @@ -444,6 +444,25 @@ func (p *Path) EquivalentTo(other *Path) bool { return p.Clean().path == other.Clean().path } +// Parents returns all the parents directories of the current path. If the path is absolute +// it starts from the current path to the root, if the path is relative is starts from the +// current path to the current directory. +// The path should be clean for this method to work properly (no .. or . or other shortcuts). +// This function does not performs any check on the returned paths. +func (p *Path) Parents() []*Path { + res := []*Path{} + dir := p + for { + res = append(res, dir) + parent := dir.Parent() + if parent.EquivalentTo(dir) { + break + } + dir = parent + } + return res +} + func (p *Path) String() string { return p.path } From 5c48c847ea7c9d6a8f9b4a1bb49c147a46229d52 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 14 Feb 2019 16:01:26 +0100 Subject: [PATCH 22/26] Correctly handle error when looking for configuration on parent folders --- commands/root/root.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/commands/root/root.go b/commands/root/root.go index 23a3102db31..10ba1d94cd8 100644 --- a/commands/root/root.go +++ b/commands/root/root.go @@ -21,7 +21,6 @@ import ( "fmt" "io/ioutil" "os" - "path/filepath" "github.com/arduino/arduino-cli/output" @@ -136,12 +135,6 @@ func initConfigs() { commands.Config = conf } - // Navigate through folders - pwd, err := filepath.Abs(".") - if err != nil { - logrus.WithError(err).Warn("Did not manage to find current path") - } - // Read configuration from global config file if commands.Config.ConfigFile.Exist() { logrus.Infof("Reading configuration from %s", commands.Config.ConfigFile) @@ -161,7 +154,11 @@ func initConfigs() { } // Read configuration from parent folders (project config) - commands.Config.Navigate("/", pwd) + if pwd, err := paths.Getwd(); err != nil { + logrus.WithError(err).Warn("Did not manage to find current path") + } else { + commands.Config.Navigate("/", pwd.String()) + } // Read configuration from environment vars commands.Config.LoadFromEnv() From 66ce6f84e146a5f8232a9bbe5e9190cc6f03492c Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 14 Feb 2019 23:25:24 +0100 Subject: [PATCH 23/26] configs: Correctly override board manager additional urls --- commands/commands_test.go | 11 ++++++++++- configs/yaml_serializer.go | 3 +++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/commands/commands_test.go b/commands/commands_test.go index 45904b49c46..017548488a0 100644 --- a/commands/commands_test.go +++ b/commands/commands_test.go @@ -74,6 +74,15 @@ func executeWithArgs(t *testing.T, args ...string) (int, []byte) { // This closure is here because we won't that the defer are executed after the end of the "executeWithArgs" method func() { + // Create an empty config for the CLI test + conf := paths.New("arduino-cli.yaml") + require.False(t, conf.Exist()) + err := conf.WriteFile([]byte("board_manager:\n additional_urls:\n")) + require.NoError(t, err) + defer func() { + require.NoError(t, conf.Remove()) + }() + redirect := &stdOutRedirect{} redirect.Open(t) defer func() { @@ -544,7 +553,7 @@ func TestInvalidCoreURL(t *testing.T) { require.NoError(t, err, "making temporary dir") defer tmp.RemoveAll() - configFile := tmp.Join("cli-config.yml") + configFile := tmp.Join("arduino-cli.yaml") configFile.WriteFile([]byte(` board_manager: additional_urls: diff --git a/configs/yaml_serializer.go b/configs/yaml_serializer.go index 4125eb7e9ae..ae3dc3c5b8b 100644 --- a/configs/yaml_serializer.go +++ b/configs/yaml_serializer.go @@ -77,6 +77,9 @@ func (config *Configuration) LoadFromYAML(path *paths.Path) error { } } if ret.BoardsManager != nil { + if len(config.BoardManagerAdditionalUrls) > 1 { + config.BoardManagerAdditionalUrls = config.BoardManagerAdditionalUrls[:1] + } for _, rawurl := range ret.BoardsManager.AdditionalURLS { url, err := url.Parse(rawurl) if err != nil { From de5ac2352c6e0c43562e4846e2ee393dc78df87a Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 14 Feb 2019 23:26:51 +0100 Subject: [PATCH 24/26] UploadsTests: go back to previous working directory after test --- commands/commands_test.go | 48 ++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/commands/commands_test.go b/commands/commands_test.go index 017548488a0..21e90cd420c 100644 --- a/commands/commands_test.go +++ b/commands/commands_test.go @@ -521,27 +521,33 @@ func TestCompileCommands(t *testing.T) { require.True(t, paths.New(test1).Join("Test1.arduino.avr.nano.hex").Exist()) // Build sketch with --output path - require.NoError(t, os.Chdir(tmp)) - exitCode, d = executeWithArgs(t, "compile", "-b", "arduino:avr:nano", "-o", "test", test1) - require.Zero(t, exitCode, "exit code") - require.Contains(t, string(d), "Sketch uses") - require.True(t, paths.New("test.hex").Exist()) - - exitCode, d = executeWithArgs(t, "compile", "-b", "arduino:avr:nano", "-o", "test2.hex", test1) - require.Zero(t, exitCode, "exit code") - require.Contains(t, string(d), "Sketch uses") - require.True(t, paths.New("test2.hex").Exist()) - require.NoError(t, paths.New(tmp, "anothertest").MkdirAll()) - - exitCode, d = executeWithArgs(t, "compile", "-b", "arduino:avr:nano", "-o", "anothertest/test", test1) - require.Zero(t, exitCode, "exit code") - require.Contains(t, string(d), "Sketch uses") - require.True(t, paths.New("anothertest", "test.hex").Exist()) - - exitCode, d = executeWithArgs(t, "compile", "-b", "arduino:avr:nano", "-o", tmp+"/anothertest/test2", test1) - require.Zero(t, exitCode, "exit code") - require.Contains(t, string(d), "Sketch uses") - require.True(t, paths.New("anothertest", "test2.hex").Exist()) + { + pwd, err := os.Getwd() + require.NoError(t, err) + defer func() { require.NoError(t, os.Chdir(pwd)) }() + require.NoError(t, os.Chdir(tmp)) + + exitCode, d = executeWithArgs(t, "compile", "-b", "arduino:avr:nano", "-o", "test", test1) + require.Zero(t, exitCode, "exit code") + require.Contains(t, string(d), "Sketch uses") + require.True(t, paths.New("test.hex").Exist()) + + exitCode, d = executeWithArgs(t, "compile", "-b", "arduino:avr:nano", "-o", "test2.hex", test1) + require.Zero(t, exitCode, "exit code") + require.Contains(t, string(d), "Sketch uses") + require.True(t, paths.New("test2.hex").Exist()) + require.NoError(t, paths.New(tmp, "anothertest").MkdirAll()) + + exitCode, d = executeWithArgs(t, "compile", "-b", "arduino:avr:nano", "-o", "anothertest/test", test1) + require.Zero(t, exitCode, "exit code") + require.Contains(t, string(d), "Sketch uses") + require.True(t, paths.New("anothertest", "test.hex").Exist()) + + exitCode, d = executeWithArgs(t, "compile", "-b", "arduino:avr:nano", "-o", tmp+"/anothertest/test2", test1) + require.Zero(t, exitCode, "exit code") + require.Contains(t, string(d), "Sketch uses") + require.True(t, paths.New("anothertest", "test2.hex").Exist()) + } } func TestInvalidCoreURL(t *testing.T) { From ebb0de9d3925039fd872515c73bb84937c1ef670 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 14 Feb 2019 23:27:35 +0100 Subject: [PATCH 25/26] Refactored duplicated code --- commands/root/root.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/commands/root/root.go b/commands/root/root.go index 10ba1d94cd8..24f758775ec 100644 --- a/commands/root/root.go +++ b/commands/root/root.go @@ -137,10 +137,7 @@ func initConfigs() { // Read configuration from global config file if commands.Config.ConfigFile.Exist() { - logrus.Infof("Reading configuration from %s", commands.Config.ConfigFile) - if err := commands.Config.LoadFromYAML(commands.Config.ConfigFile); err != nil { - logrus.WithError(err).Warnf("Could not read configuration from %s", commands.Config.ConfigFile) - } + readConfigFrom(commands.Config.ConfigFile) } if commands.Config.IsBundledInDesktopIDE() { @@ -166,11 +163,15 @@ func initConfigs() { // Read configuration from user specified file if yamlConfigFile != "" { commands.Config.ConfigFile = paths.New(yamlConfigFile) - logrus.Infof("Reading configuration from %s", commands.Config.ConfigFile) - if err := commands.Config.LoadFromYAML(commands.Config.ConfigFile); err != nil { - logrus.WithError(err).Warnf("Could not read configuration from %s", commands.Config.ConfigFile) - } + readConfigFrom(commands.Config.ConfigFile) } logrus.Info("Configuration set") } + +func readConfigFrom(path *paths.Path) { + logrus.Infof("Reading configuration from %s", path) + if err := commands.Config.LoadFromYAML(path); err != nil { + logrus.WithError(err).Warnf("Could not read configuration from %s", path) + } +} From fc648888c9f0d07cb577ee554b78aa211ee6a52d Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 14 Feb 2019 23:28:51 +0100 Subject: [PATCH 26/26] configs: if cwd could not be determined try to open 'arduino-cli.yaml' as last resource --- commands/root/root.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/commands/root/root.go b/commands/root/root.go index 24f758775ec..471d65bc579 100644 --- a/commands/root/root.go +++ b/commands/root/root.go @@ -153,6 +153,9 @@ func initConfigs() { // Read configuration from parent folders (project config) if pwd, err := paths.Getwd(); err != nil { logrus.WithError(err).Warn("Did not manage to find current path") + if path := paths.New("arduino-cli.yaml"); path.Exist() { + readConfigFrom(path) + } } else { commands.Config.Navigate("/", pwd.String()) }