From 411f10da7a11d4607b76cb57f353a20f7e475b9c Mon Sep 17 00:00:00 2001 From: giulcioffi Date: Thu, 19 Nov 2020 16:55:36 +0100 Subject: [PATCH 1/7] Initial commit --- commands/daemon/testdata/arduino-cli.yml | 1 + commands/instances.go | 36 +++++++++++++++++++++++- configuration/configuration.go | 1 + configuration/configuration_test.go | 1 + configuration/defaults.go | 1 + docs/configuration.md | 7 +++++ 6 files changed, 46 insertions(+), 1 deletion(-) diff --git a/commands/daemon/testdata/arduino-cli.yml b/commands/daemon/testdata/arduino-cli.yml index 61fb5f655eb..29e8e6d4344 100644 --- a/commands/daemon/testdata/arduino-cli.yml +++ b/commands/daemon/testdata/arduino-cli.yml @@ -2,6 +2,7 @@ board_manager: additional_urls: - http://foobar.com - http://example.com + additional_paths: [] daemon: port: "50051" diff --git a/commands/instances.go b/commands/instances.go index 501b2c14fcb..33f0e78828a 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -200,9 +200,30 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexReq, downloadCB Downlo } indexpath := paths.New(configuration.Settings.GetString("directories.Data")) + json_paths := []string{} + json_paths = append(json_paths, configuration.Settings.GetStringSlice("board_manager.additional_paths")...) + for _, x := range json_paths { + //path_to_json, err := paths.New(x).Abs() + path_to_json := indexpath.Join(x) + + if _, err := packageindex.LoadIndex(path_to_json); err != nil { + return nil, fmt.Errorf("invalid package index in %s: %s", path_to_json, err) + } + + if err := indexpath.MkdirAll(); err != nil { + return nil, fmt.Errorf("can't create data directory %s: %s", indexpath, err) + } + + //may not be necessary + //coreIndexPath := indexpath.Join(path.Base(path_to_json.Path)) + //if err := tmp.CopyTo(coreIndexPath); err != nil { + // return nil, fmt.Errorf("saving downloaded index %s: %s", path_to_json, err) + //} + } urls := []string{globals.DefaultIndexURL} urls = append(urls, configuration.Settings.GetStringSlice("board_manager.additional_urls")...) for _, u := range urls { + logrus.Info("URL: %s", u) URL, err := url.Parse(u) if err != nil { logrus.Warnf("unable to parse additional URL: %s", u) @@ -238,7 +259,7 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexReq, downloadCB Downlo // Check for signature var tmpSig *paths.Path var coreIndexSigPath *paths.Path - if URL.Hostname() == "downloads.arduino.cc" { + if URL.Hostname() == "downloads.WRONG.net" { URLSig, err := url.Parse(URL.String()) if err != nil { return nil, fmt.Errorf("parsing url for index signature check: %s", err) @@ -649,6 +670,19 @@ func createInstance(ctx context.Context, getLibOnly bool) (*createInstanceResult } } + indexpath := paths.New(configuration.Settings.GetString("directories.Data")) + json_paths := []string{} + json_paths = append(json_paths, configuration.Settings.GetStringSlice("board_manager.additional_paths")...) + for _, x := range json_paths { + //path_to_json, err := paths.New(x).Abs() + path_to_json := indexpath.Join(x) + + _, err := res.Pm.LoadPackageIndexFromFile(path_to_json) + if err != nil { + res.PlatformIndexErrors = append(res.PlatformIndexErrors, err.Error()) + } + } + if err := res.Pm.LoadHardware(); err != nil { return res, fmt.Errorf("error loading hardware packages: %s", err) } diff --git a/configuration/configuration.go b/configuration/configuration.go index 01f424c586d..841bd42ea76 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -76,6 +76,7 @@ func BindFlags(cmd *cobra.Command, settings *viper.Viper) { settings.BindPFlag("logging.file", cmd.Flag("log-file")) settings.BindPFlag("logging.format", cmd.Flag("log-format")) settings.BindPFlag("board_manager.additional_urls", cmd.Flag("additional-urls")) + settings.BindPFlag("board_manager.additional_paths", cmd.Flag("additional-paths")) } // getDefaultArduinoDataDir returns the full path to the default arduino folder diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 45ee9261535..8bd86ab7479 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -89,6 +89,7 @@ func TestInit(t *testing.T) { require.Equal(t, "text", settings.GetString("logging.format")) require.Empty(t, settings.GetStringSlice("board_manager.additional_urls")) + require.Empty(t, settings.GetStringSlice("board_manager.additional_paths")) require.NotEmpty(t, settings.GetString("directories.Data")) require.NotEmpty(t, settings.GetString("directories.Downloads")) diff --git a/configuration/defaults.go b/configuration/defaults.go index 58eae4921a7..d07e9e2169d 100644 --- a/configuration/defaults.go +++ b/configuration/defaults.go @@ -33,6 +33,7 @@ func SetDefaults(settings *viper.Viper) { // Boards Manager settings.SetDefault("board_manager.additional_urls", []string{}) + settings.SetDefault("board_manager.additional_paths", []string{}) // arduino directories settings.SetDefault("directories.Data", getDefaultArduinoDataDir()) diff --git a/docs/configuration.md b/docs/configuration.md index 2b1e67c6a81..0590753001c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2,6 +2,7 @@ - `board_manager` - `additional_urls` - the URLs to any additional Boards Manager package index files needed for your boards platforms. + - `additional_paths` - the file paths to any additional Boards Manager package index files needed for your boards platforms. - `daemon` - options related to running Arduino CLI as a [gRPC] server. - `port` - TCP port used for gRPC client connections. - `directories` - directories used by Arduino CLI. @@ -66,6 +67,12 @@ Setting an additional Boards Manager URL using the `ARDUINO_BOARD_MANAGER_ADDITI $ export ARDUINO_BOARD_MANAGER_ADDITIONAL_URLS=https://downloads.arduino.cc/packages/package_staging_index.json ``` +Setting an additional Boards Manager URL using the `ARDUINO_BOARD_MANAGER_ADDITIONAL_PATHS` environment variable: + +```sh +$ export ARDUINO_BOARD_MANAGER_ADDITIONAL_PATHS=packages/package_staging_index.json +``` + ### Configuration file [`arduino-cli config init`][arduino-cli config init] creates or updates a configuration file with the current From 474bdf9ee6f6a719e212f4f3b5a55cd4aaf4db4c Mon Sep 17 00:00:00 2001 From: giulcioffi Date: Thu, 19 Nov 2020 18:14:47 +0100 Subject: [PATCH 2/7] Bind new flag --additional-paths and add new function LoadIndexNoSign --- arduino/cores/packageindex/index.go | 17 +++++++++++++++++ cli/cli.go | 1 + commands/instances.go | 16 ++++------------ 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/arduino/cores/packageindex/index.go b/arduino/cores/packageindex/index.go index 11ceb05bafd..5c69afa0913 100644 --- a/arduino/cores/packageindex/index.go +++ b/arduino/cores/packageindex/index.go @@ -312,3 +312,20 @@ func LoadIndex(jsonIndexFile *paths.Path) (*Index, error) { } return &index, nil } + +// LoadIndex reads a package_index.json from a file and returns the corresponding Index structure. +func LoadIndexNoSign(jsonIndexFile *paths.Path, valid_signature bool) (*Index, error) { + buff, err := jsonIndexFile.ReadFile() + if err != nil { + return nil, err + } + var index Index + err = json.Unmarshal(buff, &index) + if err != nil { + return nil, err + } + + index.IsTrusted = true + + return &index, nil +} diff --git a/cli/cli.go b/cli/cli.go index ca711c159fc..617173f8324 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -104,6 +104,7 @@ func createCliCommandTree(cmd *cobra.Command) { cmd.PersistentFlags().StringVar(&outputFormat, "format", "text", "The output format, can be {text|json}.") cmd.PersistentFlags().StringVar(&configFile, "config-file", "", "The custom config file (if not specified the default will be used).") cmd.PersistentFlags().StringSlice("additional-urls", []string{}, "Comma-separated list of additional URLs for the Boards Manager.") + cmd.PersistentFlags().StringSlice("additional-paths", []string{}, "Comma-separated list of additional file paths for the Boards Manager.") configuration.BindFlags(cmd, configuration.Settings) } diff --git a/commands/instances.go b/commands/instances.go index 33f0e78828a..c4e8661f89b 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -203,27 +203,19 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexReq, downloadCB Downlo json_paths := []string{} json_paths = append(json_paths, configuration.Settings.GetStringSlice("board_manager.additional_paths")...) for _, x := range json_paths { + logrus.Info("JSON_PATH: ", x) //path_to_json, err := paths.New(x).Abs() path_to_json := indexpath.Join(x) - if _, err := packageindex.LoadIndex(path_to_json); err != nil { + if _, err := packageindex.LoadIndexNoSign(path_to_json, false); err != nil { return nil, fmt.Errorf("invalid package index in %s: %s", path_to_json, err) } - if err := indexpath.MkdirAll(); err != nil { - return nil, fmt.Errorf("can't create data directory %s: %s", indexpath, err) - } - - //may not be necessary - //coreIndexPath := indexpath.Join(path.Base(path_to_json.Path)) - //if err := tmp.CopyTo(coreIndexPath); err != nil { - // return nil, fmt.Errorf("saving downloaded index %s: %s", path_to_json, err) - //} } urls := []string{globals.DefaultIndexURL} urls = append(urls, configuration.Settings.GetStringSlice("board_manager.additional_urls")...) for _, u := range urls { - logrus.Info("URL: %s", u) + logrus.Info("URL: ", u) URL, err := url.Parse(u) if err != nil { logrus.Warnf("unable to parse additional URL: %s", u) @@ -259,7 +251,7 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexReq, downloadCB Downlo // Check for signature var tmpSig *paths.Path var coreIndexSigPath *paths.Path - if URL.Hostname() == "downloads.WRONG.net" { + if URL.Hostname() == "downloads.arduino.cc" { URLSig, err := url.Parse(URL.String()) if err != nil { return nil, fmt.Errorf("parsing url for index signature check: %s", err) From 372288d126c4a49194c7960777c4823a51125535 Mon Sep 17 00:00:00 2001 From: giulcioffi Date: Mon, 23 Nov 2020 16:53:25 +0100 Subject: [PATCH 3/7] Use absolute file path to package_index.json --- arduino/cores/packageindex/index.go | 2 +- commands/instances.go | 20 +++++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/arduino/cores/packageindex/index.go b/arduino/cores/packageindex/index.go index 5c69afa0913..5c98b1ec08c 100644 --- a/arduino/cores/packageindex/index.go +++ b/arduino/cores/packageindex/index.go @@ -314,7 +314,7 @@ func LoadIndex(jsonIndexFile *paths.Path) (*Index, error) { } // LoadIndex reads a package_index.json from a file and returns the corresponding Index structure. -func LoadIndexNoSign(jsonIndexFile *paths.Path, valid_signature bool) (*Index, error) { +func LoadIndexNoSign(jsonIndexFile *paths.Path) (*Index, error) { buff, err := jsonIndexFile.ReadFile() if err != nil { return nil, err diff --git a/commands/instances.go b/commands/instances.go index c4e8661f89b..3239193a70d 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -21,6 +21,7 @@ import ( "fmt" "io/ioutil" "net/url" + "os" "path" "github.com/arduino/arduino-cli/arduino/builder" @@ -202,16 +203,25 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexReq, downloadCB Downlo indexpath := paths.New(configuration.Settings.GetString("directories.Data")) json_paths := []string{} json_paths = append(json_paths, configuration.Settings.GetStringSlice("board_manager.additional_paths")...) + for _, x := range json_paths { logrus.Info("JSON_PATH: ", x) - //path_to_json, err := paths.New(x).Abs() - path_to_json := indexpath.Join(x) - if _, err := packageindex.LoadIndexNoSign(path_to_json, false); err != nil { + path_to_json, _ := paths.New(x).Abs() + + if _, err := packageindex.LoadIndexNoSign(path_to_json); err != nil { return nil, fmt.Errorf("invalid package index in %s: %s", path_to_json, err) + } else { + fi, _ := os.Stat(x) + downloadCB(&rpc.DownloadProgress{ + File: "Updating index: " + path_to_json.Base(), + TotalSize: fi.Size(), + }) + downloadCB(&rpc.DownloadProgress{Completed: true}) } } + urls := []string{globals.DefaultIndexURL} urls = append(urls, configuration.Settings.GetStringSlice("board_manager.additional_urls")...) for _, u := range urls { @@ -662,12 +672,12 @@ func createInstance(ctx context.Context, getLibOnly bool) (*createInstanceResult } } - indexpath := paths.New(configuration.Settings.GetString("directories.Data")) + //indexpath := paths.New(configuration.Settings.GetString("directories.Data")) json_paths := []string{} json_paths = append(json_paths, configuration.Settings.GetStringSlice("board_manager.additional_paths")...) for _, x := range json_paths { //path_to_json, err := paths.New(x).Abs() - path_to_json := indexpath.Join(x) + path_to_json, _ := paths.New(x).Abs() _, err := res.Pm.LoadPackageIndexFromFile(path_to_json) if err != nil { From cc400b90b6c610f5f1659371b56debf3db770a5b Mon Sep 17 00:00:00 2001 From: giulcioffi Date: Tue, 1 Dec 2020 12:39:57 +0100 Subject: [PATCH 4/7] Add documentation for additional-paths feature --- docs/configuration.md | 22 +++++++++++++++++++--- docs/getting-started.md | 26 +++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0590753001c..ab47f3147de 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2,7 +2,8 @@ - `board_manager` - `additional_urls` - the URLs to any additional Boards Manager package index files needed for your boards platforms. - - `additional_paths` - the file paths to any additional Boards Manager package index files needed for your boards platforms. + - `additional_paths` - the absolute file paths to any additional Boards Manager package index files needed for your + boards platforms. - `daemon` - options related to running Arduino CLI as a [gRPC] server. - `port` - TCP port used for gRPC client connections. - `directories` - directories used by Arduino CLI. @@ -67,10 +68,10 @@ Setting an additional Boards Manager URL using the `ARDUINO_BOARD_MANAGER_ADDITI $ export ARDUINO_BOARD_MANAGER_ADDITIONAL_URLS=https://downloads.arduino.cc/packages/package_staging_index.json ``` -Setting an additional Boards Manager URL using the `ARDUINO_BOARD_MANAGER_ADDITIONAL_PATHS` environment variable: +Setting an additional Boards Manager file using the `ARDUINO_BOARD_MANAGER_ADDITIONAL_PATHS` environment variable: ```sh -$ export ARDUINO_BOARD_MANAGER_ADDITIONAL_PATHS=packages/package_staging_index.json +$ export ARDUINO_BOARD_MANAGER_ADDITIONAL_PATHS=/path/to/your/package_staging_index.json ``` ### Configuration file @@ -135,6 +136,21 @@ Doing the same using a TOML format file: additional_urls = [ "https://downloads.arduino.cc/packages/package_staging_index.json" ] ``` +Setting an additional Boards Manager File using a YAML format configuration file: + +```yaml +board_manager: + additional_paths: + - /path/to/your/package_staging_index.json +``` + +Doing the same using a TOML format file: + +```toml +[board_manager] +additional_paths = [ "/path/to/your/package_staging_index.json" ] +``` + [grpc]: https://grpc.io [sketchbook directory]: sketch-specification.md#sketchbook [arduino cli lib install]: commands/arduino-cli_lib_install.md diff --git a/docs/getting-started.md b/docs/getting-started.md index ca517e3d598..5d0f6bd4947 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -177,12 +177,24 @@ board_manager: - https://arduino.esp8266.com/stable/package_esp8266com_index.json ``` -From now on, commands supporting custom cores will automatically use the additional URL from the configuration file: +If you have your package indexes locally installed, you can list their file path in the Arduino CLI configuration file. + +For example, to add the NRF52832 core, edit the configuration file and change the `board_manager` settings as follows: + +```yaml +board_manager: + additional_paths: + - /absolute/path/to/your/package_nrf52832_index.json +``` + +From now on, commands supporting custom cores will automatically use the additional URL and additional paths from the +configuration file: ```sh $ arduino-cli core update-index Updating index: package_index.json downloaded Updating index: package_esp8266com_index.json downloaded +Updating index: package_nrf52832_index.json Updating index: package_index.json downloaded $ arduino-cli core search esp8266 @@ -202,6 +214,18 @@ ID Version Name esp8266:esp8266 2.5.2 esp8266 ``` +The same applies to the additional package index file provided by file paths. Use the `--additional-paths` option, that +has to be specified every time and for every command that operates on a 3rd party platform core, for example: + +```sh +$ arduino-cli core update-index --additional-paths /absolute/path/to/your/package_esp8266com_index.json +Updating index: package_esp8266com_index.json downloaded + +$ arduino-cli core search esp8266 --additional-paths /absolute/path/to/your/package_esp8266com_index.json +ID Version Name +esp8266:esp8266 2.5.2 esp8266 +``` + ## Compile and upload the sketch To compile the sketch you run the `compile` command, passing the proper FQBN string: From a7e099ebb5acb473fed1b9db60283a6fb7eeffef Mon Sep 17 00:00:00 2001 From: giulcioffi Date: Tue, 1 Dec 2020 14:19:52 +0100 Subject: [PATCH 5/7] Update comment for LoadIndexNoSign --- arduino/cores/packageindex/index.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arduino/cores/packageindex/index.go b/arduino/cores/packageindex/index.go index 5c98b1ec08c..2dabff7f9e1 100644 --- a/arduino/cores/packageindex/index.go +++ b/arduino/cores/packageindex/index.go @@ -313,7 +313,7 @@ func LoadIndex(jsonIndexFile *paths.Path) (*Index, error) { return &index, nil } -// LoadIndex reads a package_index.json from a file and returns the corresponding Index structure. +// LoadIndexNoSign reads a package_index.json from a file and returns the corresponding Index structure. func LoadIndexNoSign(jsonIndexFile *paths.Path) (*Index, error) { buff, err := jsonIndexFile.ReadFile() if err != nil { From 7fc92e4025138d2e0b17909f8272b63892cbb21f Mon Sep 17 00:00:00 2001 From: giulcioffi Date: Tue, 1 Dec 2020 14:20:54 +0100 Subject: [PATCH 6/7] Update variable names to pass checks + simplify code --- commands/instances.go | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/commands/instances.go b/commands/instances.go index 3239193a70d..91af60b523b 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -201,25 +201,23 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexReq, downloadCB Downlo } indexpath := paths.New(configuration.Settings.GetString("directories.Data")) - json_paths := []string{} - json_paths = append(json_paths, configuration.Settings.GetStringSlice("board_manager.additional_paths")...) - for _, x := range json_paths { - logrus.Info("JSON_PATH: ", x) + for _, x := range configuration.Settings.GetStringSlice("board_manager.additional_paths") { + logrus.Info("JSON PATH: ", x) - path_to_json, _ := paths.New(x).Abs() + pathJSON, _ := paths.New(x).Abs() - if _, err := packageindex.LoadIndexNoSign(path_to_json); err != nil { - return nil, fmt.Errorf("invalid package index in %s: %s", path_to_json, err) - } else { - fi, _ := os.Stat(x) - downloadCB(&rpc.DownloadProgress{ - File: "Updating index: " + path_to_json.Base(), - TotalSize: fi.Size(), - }) - downloadCB(&rpc.DownloadProgress{Completed: true}) + if _, err := packageindex.LoadIndexNoSign(pathJSON); err != nil { + return nil, fmt.Errorf("invalid package index in %s: %s", pathJSON, err) } + fi, _ := os.Stat(x) + downloadCB(&rpc.DownloadProgress{ + File: "Updating index: " + pathJSON.Base(), + TotalSize: fi.Size(), + }) + downloadCB(&rpc.DownloadProgress{Completed: true}) + } urls := []string{globals.DefaultIndexURL} @@ -672,14 +670,10 @@ func createInstance(ctx context.Context, getLibOnly bool) (*createInstanceResult } } - //indexpath := paths.New(configuration.Settings.GetString("directories.Data")) - json_paths := []string{} - json_paths = append(json_paths, configuration.Settings.GetStringSlice("board_manager.additional_paths")...) - for _, x := range json_paths { - //path_to_json, err := paths.New(x).Abs() - path_to_json, _ := paths.New(x).Abs() + for _, x := range configuration.Settings.GetStringSlice("board_manager.additional_paths") { + pathJSON, _ := paths.New(x).Abs() - _, err := res.Pm.LoadPackageIndexFromFile(path_to_json) + _, err := res.Pm.LoadPackageIndexFromFile(pathJSON) if err != nil { res.PlatformIndexErrors = append(res.PlatformIndexErrors, err.Error()) } From 2e1043eb6f1a1c79da3b12209139a3bc5fd172bf Mon Sep 17 00:00:00 2001 From: giulcioffi Date: Tue, 1 Dec 2020 16:10:31 +0100 Subject: [PATCH 7/7] Add new config for additional_paths --- cli/config/validate.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/cli/config/validate.go b/cli/config/validate.go index d448ed32955..b1a02b7267d 100644 --- a/cli/config/validate.go +++ b/cli/config/validate.go @@ -21,18 +21,19 @@ import ( ) var validMap = map[string]reflect.Kind{ - "board_manager.additional_urls": reflect.Slice, - "daemon.port": reflect.String, - "directories.data": reflect.String, - "directories.downloads": reflect.String, - "directories.user": reflect.String, - "library.enable_unsafe_install": reflect.Bool, - "logging.file": reflect.String, - "logging.format": reflect.String, - "logging.level": reflect.String, - "sketch.always_export_binaries": reflect.Bool, - "telemetry.addr": reflect.String, - "telemetry.enabled": reflect.Bool, + "board_manager.additional_urls": reflect.Slice, + "board_manager.additional_paths": reflect.Slice, + "daemon.port": reflect.String, + "directories.data": reflect.String, + "directories.downloads": reflect.String, + "directories.user": reflect.String, + "library.enable_unsafe_install": reflect.Bool, + "logging.file": reflect.String, + "logging.format": reflect.String, + "logging.level": reflect.String, + "sketch.always_export_binaries": reflect.Bool, + "telemetry.addr": reflect.String, + "telemetry.enabled": reflect.Bool, } func typeOf(key string) (reflect.Kind, error) {