add cli httpclient with support for proxy configuration

cli/globals/globals.go

@@ -16,11 +16,8 @@
 package globals
 
 import (
-	"fmt"
-	"net/http"
 	"os"
 	"path/filepath"
-	"runtime"
 
 	"github.com/arduino/arduino-cli/version"
 )
@@ -31,18 +28,3 @@ var (
 	// DefaultIndexURL is the default index url
 	DefaultIndexURL = "https://downloads.arduino.cc/packages/package_index.json"
 )
-
-// NewHTTPClientHeader returns the http.Header object that must be used by the clients inside the downloaders
-// and adds the subComponent if specified
-func NewHTTPClientHeader(subComponent string) http.Header {
-	if subComponent != "" {
-		subComponent = " " + subComponent
-	}
-	userAgentValue := fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s",
-		VersionInfo.Application,
-		VersionInfo.VersionString,
-		subComponent,
-		runtime.GOARCH, runtime.GOOS, runtime.Version(),
-		VersionInfo.Commit)
-	return http.Header{"User-Agent": []string{userAgentValue}}
-}

commands/board/list.go

@@ -23,8 +23,8 @@ import (
 	"regexp"
 	"sync"
 
-	"github.com/arduino/arduino-cli/cli/globals"
 	"github.com/arduino/arduino-cli/commands"
+	"github.com/arduino/arduino-cli/httpclient"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
 	"github.com/pkg/errors"
 	"github.com/segmentio/stats/v4"
@@ -51,12 +51,17 @@ func apiByVidPid(vid, pid string) ([]*rpc.BoardListItem, error) {
 	url := fmt.Sprintf("%s/%s/%s", vidPidURL, vid, pid)
 	retVal := []*rpc.BoardListItem{}
 	req, _ := http.NewRequest("GET", url, nil)
-	req.Header = globals.NewHTTPClientHeader("")
 	req.Header.Set("Content-Type", "application/json")
 
 	// TODO: use proxy if set
 
-	if res, err := http.DefaultClient.Do(req); err == nil {
+	httpClient, err := httpclient.New()
+
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to initialize http client")
+	}
+
+	if res, err := httpClient.Do(req); err == nil {
 		if res.StatusCode >= 400 {
 			if res.StatusCode == 404 {
 				return nil, ErrNotFound

commands/download.go

@@ -16,10 +16,11 @@
 package commands
 
 import (
+	"net/http"
 	"net/url"
 	"time"
 
-	"github.com/arduino/arduino-cli/cli/globals"
+	"github.com/arduino/arduino-cli/httpclient"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -31,7 +32,9 @@ import (
 // current settings.
 func GetDownloaderConfig() (*downloader.Config, error) {
 	res := &downloader.Config{
-		RequestHeaders: globals.NewHTTPClientHeader(viper.GetString("network.user_agent_ext")),
+		RequestHeaders: http.Header{
+			"User-Agent": []string{httpclient.UserAgent()},
+		},
 	}
 	if viper.IsSet("network.proxy") {
 		proxy := viper.GetString("network.proxy")

httpclient/httpclient.go

@@ -0,0 +1,40 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package httpclient
+
+import (
+	"net/http"
+)
+
+// New returns a default http client for use in the cli API calls
+func New() (*http.Client, error) {
+	config, err := DefaultConfig()
+
+	if err != nil {
+		return nil, err
+	}
+
+	return NewWithConfig(config), nil
+}
+
+// NewWithConfig creates a http client for use in the cli API calls with a given configuration
+func NewWithConfig(config *Config) *http.Client {
+	transport := newHTTPClientTransport(config)
+
+	return &http.Client{
+		Transport: transport,
+	}
+}

httpclient/httpclient_config.go

@@ -0,0 +1,64 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package httpclient
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"runtime"
+
+	"github.com/arduino/arduino-cli/cli/globals"
+	"github.com/spf13/viper"
+)
+
+// Config is the configuration of the http client
+type Config struct {
+	UserAgent string
+	Proxy     *url.URL
+}
+
+// DefaultConfig returns the default http client config
+func DefaultConfig() (*Config, error) {
+	var proxy *url.URL
+	var err error
+	if viper.IsSet("network.proxy") {
+		proxyConfig := viper.GetString("network.proxy")
+		if proxy, err = url.Parse(proxyConfig); err != nil {
+			return nil, errors.New("Invalid network.proxy '" + proxyConfig + "': " + err.Error())
+		}
+	}
+
+	return &Config{
+		UserAgent: UserAgent(),
+		Proxy:     proxy,
+	}, nil
+}
+
+// UserAgent returns the user agent for the cli http client
+func UserAgent() string {
+	subComponent := viper.GetString("network.user_agent_ext")
+	if subComponent != "" {
+		subComponent = " " + subComponent
+	}
+
+	return fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s",
+		globals.VersionInfo.Application,
+		globals.VersionInfo.VersionString,
+		subComponent,
+		runtime.GOARCH, runtime.GOOS, runtime.Version(),
+		globals.VersionInfo.Commit)
+}

httpclient/httpclient_test.go

@@ -0,0 +1,70 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package httpclient
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUserAgentHeader(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, r.Header.Get("User-Agent"))
+	}))
+	defer ts.Close()
+
+	client := NewWithConfig(&Config{
+		UserAgent: "test-user-agent",
+	})
+
+	request, err := http.NewRequest("GET", ts.URL, nil)
+	require.NoError(t, err)
+
+	response, err := client.Do(request)
+	require.NoError(t, err)
+
+	b, err := ioutil.ReadAll(response.Body)
+	require.NoError(t, err)
+
+	require.Equal(t, "test-user-agent", string(b))
+}
+
+func TestProxy(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer ts.Close()
+
+	proxyURL, err := url.Parse(ts.URL)
+	require.NoError(t, err)
+
+	client := NewWithConfig(&Config{
+		Proxy: proxyURL,
+	})
+
+	request, err := http.NewRequest("GET", "http://arduino.cc", nil)
+	require.NoError(t, err)
+
+	response, err := client.Do(request)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusNoContent, response.StatusCode)
+}

httpclient/httpclient_transport.go

@@ -0,0 +1,41 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package httpclient
+
+import "net/http"
+
+type httpClientRoundTripper struct {
+	transport http.RoundTripper
+	config    *Config
+}
+
+func newHTTPClientTransport(config *Config) http.RoundTripper {
+	proxy := http.ProxyURL(config.Proxy)
+
+	transport := &http.Transport{
+		Proxy: proxy,
+	}
+
+	return &httpClientRoundTripper{
+		transport: transport,
+		config:    config,
+	}
+}
+
+func (h *httpClientRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	req.Header.Add("User-Agent", h.config.UserAgent)
+	return h.transport.RoundTrip(req)
+}