diff --git a/cli/cli.go b/cli/cli.go index 72556e2d88a..733e712444e 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -17,7 +17,6 @@ package cli import ( "fmt" - "io/ioutil" "os" "strings" @@ -37,7 +36,6 @@ import ( "github.com/arduino/arduino-cli/cli/lib" "github.com/arduino/arduino-cli/cli/monitor" "github.com/arduino/arduino-cli/cli/outdated" - "github.com/arduino/arduino-cli/cli/output" "github.com/arduino/arduino-cli/cli/sketch" "github.com/arduino/arduino-cli/cli/update" "github.com/arduino/arduino-cli/cli/updater" @@ -47,17 +45,14 @@ import ( "github.com/arduino/arduino-cli/configuration" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/inventory" - "github.com/fatih/color" - "github.com/mattn/go-colorable" - "github.com/rifflock/lfshook" + "github.com/arduino/arduino-cli/logging" + "github.com/arduino/arduino-cli/output" "github.com/sirupsen/logrus" "github.com/spf13/cobra" semver "go.bug.st/relaxed-semver" ) var ( - verbose bool - outputFormat string configFile string updaterMessageChan chan *semver.Version = make(chan *semver.Version) ) @@ -104,57 +99,34 @@ func createCliCommandTree(cmd *cobra.Command) { cmd.AddCommand(burnbootloader.NewCommand()) cmd.AddCommand(version.NewCommand()) - cmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, tr("Print the logs on the standard output.")) validLogLevels := []string{"trace", "debug", "info", "warn", "error", "fatal", "panic"} - cmd.PersistentFlags().String("log-level", "", tr("Messages with this level and above will be logged. Valid levels are: %s", strings.Join(validLogLevels, ", "))) + validLogFormats := []string{"text", "json"} + cmd.PersistentFlags().String("log-level", "info", tr("Messages with this level and above will be logged. Valid levels are: %s", strings.Join(validLogLevels, ", "))) + cmd.PersistentFlags().String("log-file", "", tr("Path to the file where logs will be written.")) + cmd.PersistentFlags().String("log-format", "text", tr("The output format for the logs, can be: %s", strings.Join(validLogFormats, ", "))) cmd.RegisterFlagCompletionFunc("log-level", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return validLogLevels, cobra.ShellCompDirectiveDefault }) - cmd.PersistentFlags().String("log-file", "", tr("Path to the file where logs will be written.")) - validLogFormats := []string{"text", "json"} - cmd.PersistentFlags().String("log-format", "", tr("The output format for the logs, can be: %s", strings.Join(validLogFormats, ", "))) cmd.RegisterFlagCompletionFunc("log-format", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return validLogFormats, cobra.ShellCompDirectiveDefault }) validOutputFormats := []string{"text", "json", "jsonmini", "yaml"} - cmd.PersistentFlags().StringVar(&outputFormat, "format", "text", tr("The output format for the logs, can be: %s", strings.Join(validOutputFormats, ", "))) + cmd.PersistentFlags().BoolP("verbose", "v", false, tr("Print the logs on the standard output.")) + cmd.PersistentFlags().String("format", "text", tr("The output format for the logs, can be: %s", strings.Join(validOutputFormats, ", "))) + cmd.PersistentFlags().Bool("no-color", false, "Disable colored output.") cmd.RegisterFlagCompletionFunc("format", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return validOutputFormats, cobra.ShellCompDirectiveDefault }) cmd.PersistentFlags().StringVar(&configFile, "config-file", "", tr("The custom config file (if not specified the default will be used).")) cmd.PersistentFlags().StringSlice("additional-urls", []string{}, tr("Comma-separated list of additional URLs for the Boards Manager.")) - cmd.PersistentFlags().Bool("no-color", false, "Disable colored output.") configuration.BindFlags(cmd, configuration.Settings) } -// convert the string passed to the `--log-level` option to the corresponding -// logrus formal level. -func toLogLevel(s string) (t logrus.Level, found bool) { - t, found = map[string]logrus.Level{ - "trace": logrus.TraceLevel, - "debug": logrus.DebugLevel, - "info": logrus.InfoLevel, - "warn": logrus.WarnLevel, - "error": logrus.ErrorLevel, - "fatal": logrus.FatalLevel, - "panic": logrus.PanicLevel, - }[s] - - return -} - -func parseFormatString(arg string) (feedback.OutputFormat, bool) { - f, found := map[string]feedback.OutputFormat{ - "json": feedback.JSON, - "jsonmini": feedback.JSONMini, - "text": feedback.Text, - "yaml": feedback.YAML, - }[strings.ToLower(arg)] - - return f, found -} - func preRun(cmd *cobra.Command, args []string) { + if cmd.Name() == "daemon" { + return + } + configFile := configuration.Settings.ConfigFileUsed() // initialize inventory @@ -164,12 +136,13 @@ func preRun(cmd *cobra.Command, args []string) { os.Exit(errorcodes.ErrBadArgument) } - // https://no-color.org/ - color.NoColor = configuration.Settings.GetBool("output.no_color") || os.Getenv("NO_COLOR") != "" - - // Set default feedback output to colorable - feedback.SetOut(colorable.NewColorableStdout()) - feedback.SetErr(colorable.NewColorableStderr()) + outputFormat, err := cmd.Flags().GetString("format") + if err != nil { + feedback.Errorf(tr("Error getting flag value: %s", err)) + os.Exit(errorcodes.ErrBadCall) + } + noColor := configuration.Settings.GetBool("output.no_color") || os.Getenv("NO_COLOR") != "" + output.Setup(outputFormat, noColor) updaterMessageChan = make(chan *semver.Version) go func() { @@ -185,70 +158,19 @@ func preRun(cmd *cobra.Command, args []string) { updaterMessageChan <- updater.CheckForUpdate(currentVersion) }() - // - // Prepare logging - // - - // decide whether we should log to stdout - if verbose { - // if we print on stdout, do it in full colors - logrus.SetOutput(colorable.NewColorableStdout()) - logrus.SetFormatter(&logrus.TextFormatter{ - ForceColors: true, - DisableColors: color.NoColor, - }) - } else { - logrus.SetOutput(ioutil.Discard) - } - - // set the Logger format - logFormat := strings.ToLower(configuration.Settings.GetString("logging.format")) - if logFormat == "json" { - logrus.SetFormatter(&logrus.JSONFormatter{}) - } - - // should we log to file? - logFile := configuration.Settings.GetString("logging.file") - if logFile != "" { - file, err := os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - fmt.Println(tr("Unable to open file for logging: %s", logFile)) - os.Exit(errorcodes.ErrBadCall) - } - - // we use a hook so we don't get color codes in the log file - if logFormat == "json" { - logrus.AddHook(lfshook.NewHook(file, &logrus.JSONFormatter{})) - } else { - logrus.AddHook(lfshook.NewHook(file, &logrus.TextFormatter{})) - } - } - - // configure logging filter - if lvl, found := toLogLevel(configuration.Settings.GetString("logging.level")); !found { - feedback.Errorf(tr("Invalid option for --log-level: %s"), configuration.Settings.GetString("logging.level")) - os.Exit(errorcodes.ErrBadArgument) - } else { - logrus.SetLevel(lvl) - } - - // - // Prepare the Feedback system - // - - // normalize the format strings - outputFormat = strings.ToLower(outputFormat) - // configure the output package - output.OutputFormat = outputFormat - // check the right output format was passed - format, found := parseFormatString(outputFormat) - if !found { - feedback.Errorf(tr("Invalid output format: %s"), outputFormat) + // Setups logging if necessary + verbose, err := cmd.Flags().GetBool("verbose") + if err != nil { + feedback.Errorf(tr("Error getting flag value: %s", err)) os.Exit(errorcodes.ErrBadCall) } - - // use the output format to configure the Feedback - feedback.SetFormat(format) + logging.Setup( + verbose, + noColor, + configuration.Settings.GetString("logging.level"), + configuration.Settings.GetString("logging.file"), + configuration.Settings.GetString("logging.format"), + ) // // Print some status info and check command is consistent diff --git a/cli/config/validate.go b/cli/config/validate.go index 38df07145c3..9e2a2b65aa8 100644 --- a/cli/config/validate.go +++ b/cli/config/validate.go @@ -26,7 +26,6 @@ import ( var validMap = map[string]reflect.Kind{ "board_manager.additional_urls": reflect.Slice, - "daemon.port": reflect.String, "directories.data": reflect.String, "directories.downloads": reflect.String, "directories.user": reflect.String, diff --git a/cli/daemon/daemon.go b/cli/daemon/daemon.go index 0be6a0289bf..5c33e7fda64 100644 --- a/cli/daemon/daemon.go +++ b/cli/daemon/daemon.go @@ -31,7 +31,8 @@ import ( "github.com/arduino/arduino-cli/commands/daemon" "github.com/arduino/arduino-cli/configuration" "github.com/arduino/arduino-cli/i18n" - "github.com/arduino/arduino-cli/metrics" + "github.com/arduino/arduino-cli/logging" + "github.com/arduino/arduino-cli/output" srv_commands "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" srv_debug "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/debug/v1" srv_monitor "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/monitor/v1" @@ -44,64 +45,89 @@ import ( var ( tr = i18n.Tr - daemonize bool - debug bool debugFilters []string + + daemonConfigFile string ) // NewCommand created a new `daemon` command func NewCommand() *cobra.Command { daemonCommand := &cobra.Command{ Use: "daemon", - Short: tr("Run as a daemon on port: %s", configuration.Settings.GetString("daemon.port")), - Long: tr("Running as a daemon the initialization of cores and libraries is done only once."), + Short: tr("Run as a daemon on specified IP and port"), + Long: tr("Running as a daemon multiple different client can use the same Arduino CLI process with different settings."), Example: " " + os.Args[0] + " daemon", Args: cobra.NoArgs, Run: runDaemonCommand, } - daemonCommand.PersistentFlags().String("port", "", tr("The TCP port the daemon will listen to")) - configuration.Settings.BindPFlag("daemon.port", daemonCommand.PersistentFlags().Lookup("port")) - daemonCommand.Flags().BoolVar(&daemonize, "daemonize", false, tr("Do not terminate daemon process if the parent process dies")) - daemonCommand.Flags().BoolVar(&debug, "debug", false, tr("Enable debug logging of gRPC calls")) - daemonCommand.Flags().StringSliceVar(&debugFilters, "debug-filter", []string{}, tr("Display only the provided gRPC calls")) + daemonCommand.Flags().String("ip", "127.0.0.1", tr("The IP the daemon will listen to")) + daemonCommand.Flags().String("port", "50051", tr("The TCP port the daemon will listen to")) + daemonCommand.Flags().Bool("daemonize", false, tr("Run daemon process in background")) + daemonCommand.Flags().Bool("debug", false, tr("Enable debug logging of gRPC calls")) + daemonCommand.Flags().StringSlice("debug-filter", []string{}, tr("Display only the provided gRPC calls when debug is enabled")) + daemonCommand.Flags().Bool("metrics-enabled", false, tr("Enable local metrics collection")) + daemonCommand.Flags().String("metrics-address", ":9090", tr("Metrics local address")) + // Metrics for the time being are ignored and unused, might as well hide this setting + // from the user since they would do nothing. + daemonCommand.Flags().MarkHidden("metrics-enabled") + daemonCommand.Flags().MarkHidden("metrics-address") + + daemonCommand.Flags().StringVar(&daemonConfigFile, "config-file", "", tr("The daemon config file (if not specified default values will be used).")) return daemonCommand } func runDaemonCommand(cmd *cobra.Command, args []string) { - logrus.Info("Executing `arduino-cli daemon`") + s, err := load(cmd, daemonConfigFile) + if err != nil { + feedback.Errorf(tr("Error reading daemon config file: %v"), err) + os.Exit(errorcodes.ErrGeneric) + } - if configuration.Settings.GetBool("metrics.enabled") { - metrics.Activate("daemon") - stats.Incr("daemon", stats.T("success", "true")) - defer stats.Flush() + noColor := s.NoColor || os.Getenv("NO_COLOR") != "" + output.Setup(s.OutputFormat, noColor) + + if daemonConfigFile != "" { + // Tell the user which config file we're using only after output setup + feedback.Printf(tr("Using daemon config file %s", daemonConfigFile)) } - port := configuration.Settings.GetString("daemon.port") + + logging.Setup( + s.Verbose, + noColor, + s.LogLevel, + s.LogFile, + s.LogFormat, + ) + + logrus.Info("Executing `arduino-cli daemon`") + gRPCOptions := []grpc.ServerOption{} - if debug { + if s.Debug { + debugFilters = s.DebugFilter gRPCOptions = append(gRPCOptions, grpc.UnaryInterceptor(unaryLoggerInterceptor), grpc.StreamInterceptor(streamLoggerInterceptor), ) } - s := grpc.NewServer(gRPCOptions...) + server := grpc.NewServer(gRPCOptions...) // Set specific user-agent for the daemon configuration.Settings.Set("network.user_agent_ext", "daemon") // register the commands service - srv_commands.RegisterArduinoCoreServiceServer(s, &daemon.ArduinoCoreServerImpl{ + srv_commands.RegisterArduinoCoreServiceServer(server, &daemon.ArduinoCoreServerImpl{ VersionString: globals.VersionInfo.VersionString, }) // Register the monitors service - srv_monitor.RegisterMonitorServiceServer(s, &daemon.MonitorService{}) + srv_monitor.RegisterMonitorServiceServer(server, &daemon.MonitorService{}) // Register the settings service - srv_settings.RegisterSettingsServiceServer(s, &daemon.SettingsService{}) + srv_settings.RegisterSettingsServiceServer(server, &daemon.SettingsService{}) // Register the debug session service - srv_debug.RegisterDebugServiceServer(s, &daemon.DebugService{}) + srv_debug.RegisterDebugServiceServer(server, &daemon.DebugService{}) - if !daemonize { + if !s.Daemonize { // When parent process ends terminate also the daemon go func() { // Stdin is closed when the controlling parent process ends @@ -112,35 +138,34 @@ func runDaemonCommand(cmd *cobra.Command, args []string) { }() } - ip := "127.0.0.1" - lis, err := net.Listen("tcp", fmt.Sprintf("%s:%s", ip, port)) + lis, err := net.Listen("tcp", fmt.Sprintf("%s:%s", s.IP, s.Port)) if err != nil { // Invalid port, such as "Foo" var dnsError *net.DNSError if errors.As(err, &dnsError) { - feedback.Errorf(tr("Failed to listen on TCP port: %[1]s. %[2]s is unknown name."), port, dnsError.Name) + feedback.Errorf(tr("Failed to listen on TCP port: %[1]s. %[2]s is unknown name."), s.Port, dnsError.Name) os.Exit(errorcodes.ErrCoreConfig) } // Invalid port number, such as -1 var addrError *net.AddrError if errors.As(err, &addrError) { - feedback.Errorf(tr("Failed to listen on TCP port: %[1]s. %[2]s is an invalid port."), port, addrError.Addr) + feedback.Errorf(tr("Failed to listen on TCP port: %[1]s. %[2]s is an invalid port."), s.Port, addrError.Addr) os.Exit(errorcodes.ErrCoreConfig) } // Port is already in use var syscallErr *os.SyscallError if errors.As(err, &syscallErr) && errors.Is(syscallErr.Err, syscall.EADDRINUSE) { - feedback.Errorf(tr("Failed to listen on TCP port: %s. Address already in use."), port) + feedback.Errorf(tr("Failed to listen on TCP port: %s. Address already in use."), s.Port) os.Exit(errorcodes.ErrNetwork) } - feedback.Errorf(tr("Failed to listen on TCP port: %[1]s. Unexpected error: %[2]v"), port, err) + feedback.Errorf(tr("Failed to listen on TCP port: %[1]s. Unexpected error: %[2]v"), s.Port, err) os.Exit(errorcodes.ErrGeneric) } // We need to parse the port used only if the user let // us choose it randomly, in all other cases we already // know which is used. - if port == "0" { + if s.Port == "0" { address := lis.Addr() split := strings.Split(address.String(), ":") @@ -148,15 +173,15 @@ func runDaemonCommand(cmd *cobra.Command, args []string) { feedback.Error(tr("Failed choosing port, address: %s", address)) } - port = split[len(split)-1] + s.Port = split[len(split)-1] } feedback.PrintResult(daemonResult{ - IP: ip, - Port: port, + IP: s.IP, + Port: s.Port, }) - if err := s.Serve(lis); err != nil { + if err := server.Serve(lis); err != nil { logrus.Fatalf("Failed to serve: %v", err) } } diff --git a/cli/daemon/settings.go b/cli/daemon/settings.go new file mode 100644 index 00000000000..0927601dd64 --- /dev/null +++ b/cli/daemon/settings.go @@ -0,0 +1,76 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package daemon + +import ( + "fmt" + + "github.com/arduino/go-paths-helper" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// settings defines all the configuration that can be set +// for the daemon command. +// The mapstructure tag must be identical to the name +// of the flag as defined in the cobra.Command used to load +// the settings. +type settings struct { + IP string `mapstructure:"ip"` + Port string `mapstructure:"port"` + Daemonize bool `mapstructure:"daemonize"` + Debug bool `mapstructure:"debug"` + DebugFilter []string `mapstructure:"debug-filter"` + Verbose bool `mapstructure:"verbose"` + OutputFormat string `mapstructure:"format"` + NoColor bool `mapstructure:"no-color"` + MetricsEnabled bool `mapstructure:"metrics-enabled"` + MetricsAddress string `mapstructure:"metrics-address"` + LogLevel string `mapstructure:"log-level"` + LogFile string `mapstructure:"log-file"` + LogFormat string `mapstructure:"log-format"` +} + +// load returns a settings struct populated with all the configurations +// read from configFile if it exists, flags override file values. +// If the config file doesn't exist only uses flag values. +// Returns error if it fails to read the file. +func load(cmd *cobra.Command, configFile string) (s *settings, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %s", r) + } + }() + + v := viper.New() + v.SetConfigFile(configFile) + v.BindPFlags(cmd.Flags()) + + // Try to read config file only if it exists. + // We fallback to flags default + if configFile != "" && paths.New(configFile).Exist() { + if err := v.ReadInConfig(); err != nil { + return nil, err + } + } + + s = &settings{} + if err := v.Unmarshal(s); err != nil { + return nil, err + } + + return s, nil +} diff --git a/cli/daemon/settings_test.go b/cli/daemon/settings_test.go new file mode 100644 index 00000000000..c2f97ffc96d --- /dev/null +++ b/cli/daemon/settings_test.go @@ -0,0 +1,83 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package daemon + +import ( + "testing" + + "github.com/arduino/go-paths-helper" + "github.com/stretchr/testify/require" +) + +func TestLoadWithoutConfigs(t *testing.T) { + cmd := NewCommand() + configFile := "unexisting-config.yml" + + s, err := load(cmd, configFile) + require.NoError(t, err) + require.NotNil(t, s) + + // Verify default settings + require.Equal(t, "127.0.0.1", s.IP) + require.Equal(t, "50051", s.Port) + require.Equal(t, false, s.Daemonize) + require.Equal(t, false, s.Debug) + require.Equal(t, []string{}, s.DebugFilter) + require.Equal(t, false, s.MetricsEnabled) + require.Equal(t, ":9090", s.MetricsAddress) +} + +func TestLoadWithConfigs(t *testing.T) { + cmd := NewCommand() + configFile := paths.New("testdata", "daemon-config.yaml") + require.NoError(t, configFile.ToAbs()) + + s, err := load(cmd, configFile.String()) + require.NoError(t, err) + require.NotNil(t, s) + + // Verify settings ar correctly read from config file + require.Equal(t, "127.0.0.1", s.IP) + require.Equal(t, "0", s.Port) + require.Equal(t, false, s.Daemonize) + require.Equal(t, false, s.Debug) + require.Equal(t, []string{}, s.DebugFilter) + require.Equal(t, false, s.MetricsEnabled) + require.Equal(t, ":9090", s.MetricsAddress) +} + +func TestLoadWithConfigsAndFlags(t *testing.T) { + cmd := NewCommand() + configFile := paths.New("testdata", "daemon-config.yaml") + require.NoError(t, configFile.ToAbs()) + + cmd.Flags().Set("ip", "0.0.0.0") + cmd.Flags().Set("port", "12345") + + s, err := load(cmd, configFile.String()) + require.NoError(t, err) + require.NotNil(t, s) + + // Verify settings read from config file are override by those + // set via flags + require.Equal(t, "0.0.0.0", s.IP) + require.Equal(t, "12345", s.Port) + require.Equal(t, false, s.Daemonize) + require.Equal(t, false, s.Debug) + require.Equal(t, []string{}, s.DebugFilter) + require.Equal(t, false, s.MetricsEnabled) + require.Equal(t, ":9090", s.MetricsAddress) +} diff --git a/cli/daemon/testdata/daemon-config.yaml b/cli/daemon/testdata/daemon-config.yaml new file mode 100644 index 00000000000..f4d38b24143 --- /dev/null +++ b/cli/daemon/testdata/daemon-config.yaml @@ -0,0 +1,13 @@ +ip: "127.0.0.1" +port: "0" +daemonize: false +debug: false +debug-filter: [] +verbose: false +format: "json" +no-color: false +metrics-enabled: false +metrics-address: ":9090" +log-level: "info" +log-file: "" +log-format: "text" diff --git a/cli/globals/globals.go b/cli/globals/globals.go index fc6811e38e4..8a73ba08745 100644 --- a/cli/globals/globals.go +++ b/cli/globals/globals.go @@ -19,7 +19,6 @@ import ( "os" "path/filepath" - "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/version" ) @@ -28,5 +27,4 @@ var ( VersionInfo = version.NewInfo(filepath.Base(os.Args[0])) // DefaultIndexURL is the default index url DefaultIndexURL = "https://downloads.arduino.cc/packages/package_index.json" - tr = i18n.Tr ) diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 0282f55b641..06640963ac6 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -94,8 +94,6 @@ func TestInit(t *testing.T) { require.NotEmpty(t, settings.GetString("directories.Downloads")) require.NotEmpty(t, settings.GetString("directories.User")) - require.Equal(t, "50051", settings.GetString("daemon.port")) - require.Equal(t, true, settings.GetBool("metrics.enabled")) require.Equal(t, ":9090", settings.GetString("metrics.addr")) } diff --git a/configuration/defaults.go b/configuration/defaults.go index 323eb65203b..0b944643d76 100644 --- a/configuration/defaults.go +++ b/configuration/defaults.go @@ -42,9 +42,6 @@ func SetDefaults(settings *viper.Viper) { // Sketch compilation settings.SetDefault("sketch.always_export_binaries", false) - // daemon settings - settings.SetDefault("daemon.port", "50051") - // metrics settings settings.SetDefault("metrics.enabled", true) settings.SetDefault("metrics.addr", ":9090") diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index 0515eeb3365..64744d8450c 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -4,6 +4,32 @@ Here you can find a list of migration guides to handle breaking changes between ## Unreleased +### Split `daemon` mode configs from core configs + +The `daemon.*` settings have been removed from the config read by the `arduino-cli` when running in command line mode. + +The `arduino-cli daemon` now doesn't read the same config file as the other commands, the `--config-file` flag is still +present but reads a file with the format described [here](configuration.md#daemon-configuration-keys). + +All the settings in the config file can be override with the following flags: + +- `--ip` +- `--port` +- `--daemonize` +- `--debug` +- `--debug-filter` +- `--verbose`, `-v` +- `--format` +- `--no-color` +- `--log-level` +- `--log-file` +- `--log-format` + +None of those settings will be read from the default `arduino-cli.yaml` file stored in the `.arduino15` or `Arduino15` +folder anymore when running the `arduino-cli daemon` command. + +Those that start the `daemon` process will be tasked to manage the config file used by it. + ### `commands.Compile` function change A new argument `progressCB` has been added to `commands.Compile`, the new function signature is: diff --git a/docs/configuration.md b/docs/configuration.md index 9072035dde9..306a99df84f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2,8 +2,6 @@ - `board_manager` - `additional_urls` - the URLs to any additional Boards Manager package index files needed for your boards platforms. -- `daemon` - options related to running Arduino CLI as a [gRPC] server. - - `port` - TCP port used for gRPC client connections. - `directories` - directories used by Arduino CLI. - `data` - directory used to store Boards/Library Manager index files and Boards Manager platform installations. - `downloads` - directory used to stage downloaded archives during Boards/Library Manager installations. @@ -133,6 +131,103 @@ Doing the same using a TOML format file: additional_urls = [ "https://downloads.arduino.cc/packages/package_staging_index.json" ] ``` +## Daemon + +The `daemon` mode may be configured in two ways: + +1. Command line flags +1. Configuration file + +If a configuration option is configured by multiple methods, the value set by the method highest on the above list +overwrites the ones below it. + +If a configuration option is not set, Arduino CLI uses a default value. + +### Command line flags + +Arduino CLI's `daemon` mode flags are documented in the [Arduino CLI `daemon` command reference][arduino-cli daemon]. + +#### Example + +Starting the `daemon` mode with a different port using the [`--port`][arduino-cli daemon flags] flag + +```shell +$ arduino-cli daemon --port 12345 +``` + + + +### Configuration file + +The `arduino-cli daemon` mode uses a different configuration file from the other commands and it must be managed by the +user that starts the `daemon` process. The Arduino CLI doesn't offer any way of managing those configurations and won't +search for a configuration file if not explicitly specified by the user. + +It can be easily set like this: + +```sh +arduino-cli daemon --config-file /path/to/settings/daemon-settings.yaml +``` + +#### Keys + +The `arduino-cli daemon` mode is completely separated from the command line settings and it uses a different +configuration from the other commands: + +- `ip`: IP used to listen for gRPC connections +- `port`: Port used listen for gRPC connections +- `daemonize`: True to run daemon process in background +- `debug`: True to enable debug logging of gRPC calls +- `debug-filter`: List of gRPC calls to log when in debug mode +- `verbose`: True to print logs in stdout +- `format`: Stdout output format +- `no-color`: True to disable color output to stdout and stderr +- `log-level`: Messages with this level and above will be logged +- `log-file`: Path to the file where logs will be written +- `log-format`: Output format for the logs + +#### File name + +The configuration file name can be anything but the file extension must be appropriate for the file's format. + +#### Supported formats + +The `daemon` mode `--config-file` flag supports a variety of common formats much like when running the Arduino CLI as a +command line tool: + +- [JSON] +- [TOML] +- [YAML] +- [Java properties file] +- [HCL] +- envfile +- [INI] + +#### Example + +Setting a custom port with debug enabled and custom filters using a YAML format configuration file: + +```yaml +port: "12345" +debug: true +debug-filter: + - "Create" + - "PlatformInstall" + - "UpdateIndex" +``` + +Doing the same using a TOML format file: + +```toml +port = "12345" +debug = true +debug-filter = [ + "Create", + "PlatformInstall", + "UpdateIndex" +] +``` + [grpc]: https://grpc.io [sketchbook directory]: sketch-specification.md#sketchbook [arduino cli lib install]: commands/arduino-cli_lib_install.md @@ -151,3 +246,5 @@ additional_urls = [ "https://downloads.arduino.cc/packages/package_staging_index [java properties file]: https://en.wikipedia.org/wiki/.properties [hcl]: https://github.com/hashicorp/hcl [ini]: https://en.wikipedia.org/wiki/INI_file +[arduino-cli daemon]: commands/arduino-cli_daemon.md +[arduino-cli daemon flags]: commands/arduino-cli_daemon.md#options diff --git a/docs/getting-started.md b/docs/getting-started.md index 19628282f91..481edef7297 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -328,27 +328,11 @@ Arduino CLI can be launched as a gRPC server via the `daemon` command. The [client_example] folder contains a sample client code that shows how to interact with the gRPC server. Available services and messages are detailed in the [gRPC reference] pages. -To provide observability for the gRPC server activities besides logs, the `daemon` mode activates and exposes by default -a [Prometheus](https://prometheus.io/) endpoint (http://localhost:9090/metrics) that can be fetched for metrics data -like: - -```text -# TYPE daemon_compile counter -daemon_compile{buildProperties="",exportFile="",fqbn="arduino:samd:mkr1000",installationID="ed6f1f22-1fbe-4b1f-84be-84d035b6369c",jobs="0",libraries="",preprocess="false",quiet="false",showProperties="false",sketchPath="5ff767c6fa5a91230f5cb4e267c889aa61489ab2c4f70f35f921f934c1462cb6",success="true",verbose="true",vidPid="",warnings=""} 1 1580385724726 - -# TYPE daemon_board_list counter -daemon_board_list{installationID="ed6f1f22-1fbe-4b1f-84be-84d035b6369c",success="true"} 1 1580385724833 -``` - -The metrics settings are exposed via the `metrics` section in the CLI configuration: - -```yaml -metrics: - enabled: true - addr: :9090 -``` +The `daemon` mode can be configured via [a configuration file][daemon configuration] or [command line +flags][arduino-cli daemon options]. [configuration documentation]: configuration.md [client_example]: https://github.com/arduino/arduino-cli/blob/master/client_example [grpc reference]: rpc/commands.md -[prometheus]: https://prometheus.io/ +[daemon configuration]: configuration.md#daemon-configuration-file +[arduino-cli daemon options]: commands/arduino-cli_daemon.md#options diff --git a/logging/logging.go b/logging/logging.go new file mode 100644 index 00000000000..fc03ce1effc --- /dev/null +++ b/logging/logging.go @@ -0,0 +1,92 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package logging + +import ( + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/arduino/arduino-cli/cli/errorcodes" + "github.com/arduino/arduino-cli/cli/feedback" + "github.com/arduino/arduino-cli/i18n" + "github.com/mattn/go-colorable" + "github.com/rifflock/lfshook" + "github.com/sirupsen/logrus" +) + +var tr = i18n.Tr + +// Setup logrus using the provided arguments +func Setup(verbose, noColor bool, level, file, format string) { + // Decide whether we should log to stdout + if verbose { + // If we print on stdout, do it in full colors + logrus.SetOutput(colorable.NewColorableStdout()) + logrus.SetFormatter(&logrus.TextFormatter{ + ForceColors: true, + DisableColors: noColor, + }) + } else { + logrus.SetOutput(ioutil.Discard) + } + + // Set the format + format = strings.ToLower(format) + if format == "json" { + logrus.SetFormatter(&logrus.JSONFormatter{}) + } + + // Should we log to file? + if file != "" { + file, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + fmt.Println(tr("Unable to open file for logging: %s", file)) + os.Exit(errorcodes.ErrBadCall) + } + + // We use a hook so we don't get color codes in the log file + if format == "json" { + logrus.AddHook(lfshook.NewHook(file, &logrus.JSONFormatter{})) + } else { + logrus.AddHook(lfshook.NewHook(file, &logrus.TextFormatter{})) + } + } + + // configure logging filter + if lvl, found := toLogLevel(level); !found { + feedback.Errorf(tr("Invalid option for --log-level: %s"), level) + os.Exit(errorcodes.ErrBadArgument) + } else { + logrus.SetLevel(lvl) + } +} + +// convert the string passed to the `--log-level` option to the corresponding +// logrus formal level. +func toLogLevel(s string) (t logrus.Level, found bool) { + t, found = map[string]logrus.Level{ + "trace": logrus.TraceLevel, + "debug": logrus.DebugLevel, + "info": logrus.InfoLevel, + "warn": logrus.WarnLevel, + "error": logrus.ErrorLevel, + "fatal": logrus.FatalLevel, + "panic": logrus.PanicLevel, + }[s] + return +} diff --git a/output/output.go b/output/output.go new file mode 100644 index 00000000000..fa12f3ee127 --- /dev/null +++ b/output/output.go @@ -0,0 +1,62 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package output + +import ( + "os" + "strings" + + "github.com/arduino/arduino-cli/cli/errorcodes" + "github.com/arduino/arduino-cli/cli/feedback" + "github.com/arduino/arduino-cli/cli/output" + "github.com/arduino/arduino-cli/i18n" + "github.com/fatih/color" + "github.com/mattn/go-colorable" +) + +var tr = i18n.Tr + +// Setup the feedback system +func Setup(outputFormat string, noColor bool) { + // Set feedback format output + outputFormat = strings.ToLower(outputFormat) + output.OutputFormat = outputFormat + format, found := parseFormatString(outputFormat) + if !found { + feedback.Errorf(tr("Invalid output format: %s"), outputFormat) + os.Exit(errorcodes.ErrBadCall) + } + feedback.SetFormat(format) + + // Set default feedback output to colorable + feedback.SetOut(colorable.NewColorableStdout()) + feedback.SetErr(colorable.NewColorableStderr()) + + // https://no-color.org/ + color.NoColor = noColor + +} + +func parseFormatString(arg string) (feedback.OutputFormat, bool) { + f, found := map[string]feedback.OutputFormat{ + "json": feedback.JSON, + "jsonmini": feedback.JSONMini, + "text": feedback.Text, + "yaml": feedback.YAML, + }[strings.ToLower(arg)] + + return f, found +} diff --git a/test/test_config.py b/test/test_config.py index ab49ac5caeb..44fd0a06a87 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -33,7 +33,6 @@ def test_init_with_existing_custom_config(run_command, data_dir, working_dir, do configs = yaml.load(config_file.read(), Loader=yaml.FullLoader) config_file.close() assert ["https://example.com"] == configs["board_manager"]["additional_urls"] - assert "50051" == configs["daemon"]["port"] assert data_dir == configs["directories"]["data"] assert downloads_dir == configs["directories"]["downloads"] assert data_dir == configs["directories"]["user"] @@ -53,7 +52,6 @@ def test_init_with_existing_custom_config(run_command, data_dir, working_dir, do configs = yaml.load(config_file.read(), Loader=yaml.FullLoader) config_file.close() assert [] == configs["board_manager"]["additional_urls"] - assert "50051" == configs["daemon"]["port"] assert data_dir == configs["directories"]["data"] assert downloads_dir == configs["directories"]["downloads"] assert data_dir == configs["directories"]["user"] @@ -73,7 +71,6 @@ def test_init_overwrite_existing_custom_file(run_command, data_dir, working_dir, configs = yaml.load(config_file.read(), Loader=yaml.FullLoader) config_file.close() assert ["https://example.com"] == configs["board_manager"]["additional_urls"] - assert "50051" == configs["daemon"]["port"] assert data_dir == configs["directories"]["data"] assert downloads_dir == configs["directories"]["downloads"] assert data_dir == configs["directories"]["user"] @@ -91,7 +88,6 @@ def test_init_overwrite_existing_custom_file(run_command, data_dir, working_dir, configs = yaml.load(config_file.read(), Loader=yaml.FullLoader) config_file.close() assert [] == configs["board_manager"]["additional_urls"] - assert "50051" == configs["daemon"]["port"] assert data_dir == configs["directories"]["data"] assert downloads_dir == configs["directories"]["downloads"] assert data_dir == configs["directories"]["user"] @@ -312,18 +308,18 @@ def test_add_on_unsupported_key(run_command): result = run_command(["config", "dump", "--format", "json"]) assert result.ok settings_json = json.loads(result.stdout) - assert "50051" == settings_json["daemon"]["port"] + assert "text" == settings_json["logging"]["format"] # Tries and fails to add a new item - result = run_command(["config", "add", "daemon.port", "50000"]) + result = run_command(["config", "add", "logging.format", "json"]) assert result.failed - assert "The key 'daemon.port' is not a list of items, can't add to it.\nMaybe use 'config set'?" in result.stderr + assert "The key 'logging.format' is not a list of items, can't add to it.\nMaybe use 'config set'?" in result.stderr # Verifies value is not changed result = run_command(["config", "dump", "--format", "json"]) assert result.ok settings_json = json.loads(result.stdout) - assert "50051" == settings_json["daemon"]["port"] + assert "text" == settings_json["logging"]["format"] def test_remove_single_argument(run_command): @@ -394,13 +390,13 @@ def test_remove_on_unsupported_key(run_command): result = run_command(["config", "dump", "--format", "json"]) assert result.ok settings_json = json.loads(result.stdout) - assert "50051" == settings_json["daemon"]["port"] + assert "text" == settings_json["logging"]["format"] - # Tries and fails to add a new item - result = run_command(["config", "remove", "daemon.port", "50051"]) + # Tries and fails to remove an item + result = run_command(["config", "remove", "logging.format", "text"]) assert result.failed assert ( - "The key 'daemon.port' is not a list of items, can't remove from it.\nMaybe use 'config delete'?" + "The key 'logging.format' is not a list of items, can't remove from it.\nMaybe use 'config delete'?" in result.stderr ) @@ -408,7 +404,7 @@ def test_remove_on_unsupported_key(run_command): result = run_command(["config", "dump", "--format", "json"]) assert result.ok settings_json = json.loads(result.stdout) - assert "50051" == settings_json["daemon"]["port"] + assert "text" == settings_json["logging"]["format"] def test_set_slice_with_single_argument(run_command): diff --git a/test/test_daemon.py b/test/test_daemon.py deleted file mode 100644 index 86c5a65a12e..00000000000 --- a/test/test_daemon.py +++ /dev/null @@ -1,45 +0,0 @@ -# This file is part of arduino-cli. -# -# Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -# -# This software is released under the GNU General Public License version 3, -# which covers the main part of arduino-cli. -# The terms of this license can be found at: -# https://www.gnu.org/licenses/gpl-3.0.en.html -# -# You can be released from the requirements of the above licenses by purchasing -# a commercial license. Buying such a license is mandatory if you want to modify or -# otherwise use the software for commercial activities involving the Arduino -# software without disclosing the source code of your own applications. To purchase -# a commercial license, send an email to license@arduino.cc. - -import os -import time - -import pytest -import requests -import yaml -from prometheus_client.parser import text_string_to_metric_families -from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry - - -@pytest.mark.timeout(60) -def test_metrics_prometheus_endpoint(daemon_runner, data_dir): - # Wait for the inventory file to be created and then parse it - # in order to check the generated ids - inventory_file = os.path.join(data_dir, "inventory.yaml") - while not os.path.exists(inventory_file): - time.sleep(1) - with open(inventory_file, "r") as stream: - inventory = yaml.safe_load(stream) - - # Check if :9090/metrics endpoint is alive, - # metrics is enabled by default in daemon mode - s = requests.Session() - retries = Retry(total=3, backoff_factor=1, status_forcelist=[500, 502, 503, 504]) - s.mount("http://", HTTPAdapter(max_retries=retries)) - metrics = s.get("http://localhost:9090/metrics").text - family = next(text_string_to_metric_families(metrics)) - sample = family.samples[0] - assert inventory["installation"]["id"] == sample.labels["installationID"]