v1.4.0

.github/workflows/test.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches: [master]
   pull_request:
-    branches: [master]
+    branches: [master, staging]
 
 jobs:
   test:

config/config.go

@@ -21,8 +21,17 @@ import (
 
 // State represents a desired system state
 type State struct {
-	Targets task.Targets      `json:"targets"`
-	Env     map[string]string `json:"env"`
+	Targets     task.Targets      `json:"targets"`
+	AuthMethods []AuthMethod      `json:"auths"`
+	Env         map[string]string `json:"env"`
+}
+
+// AuthMethod represents a method of authentication for a target
+type AuthMethod struct {
+	Name    string `json:"name"`     // name of the auth method
+	Path    string `json:"path"`     // path within the secret store
+	UserKey string `json:"user_key"` // key for username
+	PassKey string `json:"pass_key"` // key for password
 }
 
 // ConfigFromDirectory searches a directory for configuration files and
@@ -72,6 +81,7 @@ func (cb *configBuilder) construct(hostname string) (err error) {
 	cb.vm.Run(`'use strict';
 var STATE = {
 	targets: [],
+	auths: [],
 	env: {}
 };
 
@@ -90,6 +100,15 @@ function T(t) {
 function E(k, v) {
 	STATE.env[k] = v
 }
+
+function A(a) {
+	if(a.name === undefined) { throw "auth name undefined"; }
+	if(a.path === undefined) { throw "auth path undefined"; }
+	if(a.user_key === undefined) { throw "auth user_key undefined"; }
+	if(a.pass_key === undefined) { throw "auth pass_key undefined"; }
+
+	STATE.auths.push(a);
+}
 `)
 
 	cb.vm.Set("HOSTNAME", hostname) //nolint:errcheck

executor/cmd.go

@@ -12,13 +12,24 @@ var _ Executor = &CommandExecutor{}
 
 // CommandExecutor handles command invocation targets
 type CommandExecutor struct {
-	secrets secret.Store
+	secrets            secret.Store
+	passEnvironment    bool   // pass the Pico process environment to children
+	configSecretPath   string // path to global secrets to pass to children
+	configSecretPrefix string // only pass secrets with this prefix, usually GLOBAL_
 }
 
 // NewCommandExecutor creates a new CommandExecutor
-func NewCommandExecutor(secrets secret.Store) CommandExecutor {
+func NewCommandExecutor(
+	secrets secret.Store,
+	passEnvironment bool,
+	configSecretPath string,
+	configSecretPrefix string,
+) CommandExecutor {
 	return CommandExecutor{
-		secrets: secrets,
+		secrets:            secrets,
+		passEnvironment:    passEnvironment,
+		configSecretPath:   configSecretPath,
+		configSecretPrefix: configSecretPrefix,
 	}
 }
 
@@ -34,34 +45,66 @@ func (e *CommandExecutor) Subscribe(bus chan task.ExecutionTask) {
 	}
 }
 
-func (e *CommandExecutor) execute(
-	target task.Target,
+type exec struct {
+	path            string
+	env             map[string]string
+	shutdown        bool
+	passEnvironment bool
+}
+
+func (e *CommandExecutor) prepare(
+	name string,
 	path string,
 	shutdown bool,
 	execEnv map[string]string,
-) (err error) {
-	secrets, err := e.secrets.GetSecretsForTarget(target.Name)
+) (exec, error) {
+	// get global secrets from the Pico config path in the secret store.
+	// only secrets with the prefix are retrieved.
+	global, err := secret.GetPrefixedSecrets(e.secrets, e.configSecretPath, e.configSecretPrefix)
 	if err != nil {
-		return errors.Wrap(err, "failed to get secrets for target")
+		return exec{}, errors.Wrap(err, "failed to get global secrets for target")
+	}
+
+	secrets, err := e.secrets.GetSecretsForTarget(name)
+	if err != nil {
+		return exec{}, errors.Wrap(err, "failed to get secrets for target")
 	}
 
 	env := make(map[string]string)
 
-	// merge execution environment with secrets
+	// merge execution environment with secrets in the following order:
+	// globals first, then execution environment, then per-target secrets
+	for k, v := range global {
+		env[k] = v
+	}
 	for k, v := range execEnv {
 		env[k] = v
 	}
 	for k, v := range secrets {
 		env[k] = v
 	}
 
+	return exec{path, env, shutdown, e.passEnvironment}, nil
+}
+
+func (e *CommandExecutor) execute(
+	target task.Target,
+	path string,
+	shutdown bool,
+	execEnv map[string]string,
+) (err error) {
+	ex, err := e.prepare(target.Name, path, shutdown, execEnv)
+	if err != nil {
+		return err
+	}
+
 	zap.L().Debug("executing with secrets",
 		zap.String("target", target.Name),
 		zap.Strings("cmd", target.Up),
 		zap.String("url", target.RepoURL),
 		zap.String("dir", path),
-		zap.Int("env", len(env)),
-		zap.Int("secrets", len(secrets)))
+		zap.Any("env", ex.env),
+		zap.Bool("passthrough", e.passEnvironment))
 
-	return target.Execute(path, env, shutdown)
+	return target.Execute(ex.path, ex.env, ex.shutdown, ex.passEnvironment)
 }

executor/cmd_test.go

@@ -1,4 +1,4 @@
-package executor_test
+package executor
 
 import (
 	"os"
@@ -7,9 +7,9 @@ import (
 
 	"golang.org/x/sync/errgroup"
 
-	"github.com/picostack/pico/executor"
 	"github.com/picostack/pico/secret/memory"
 	"github.com/picostack/pico/task"
+	"github.com/stretchr/testify/assert"
 
 	_ "github.com/picostack/pico/logger"
 )
@@ -20,11 +20,13 @@ func TestMain(m *testing.M) {
 }
 
 func TestCommandExecutor(t *testing.T) {
-	ce := executor.NewCommandExecutor(&memory.MemorySecrets{
-		Secrets: map[string]string{
-			"SOME_SECRET": "123",
+	ce := NewCommandExecutor(&memory.MemorySecrets{
+		Secrets: map[string]map[string]string{
+			"test": map[string]string{
+				"SOME_SECRET": "123",
+			},
 		},
-	})
+	}, false, "pico", "GLOBAL_")
 	bus := make(chan task.ExecutionTask)
 
 	g := errgroup.Group{}
@@ -55,3 +57,56 @@ func TestCommandExecutor(t *testing.T) {
 
 	os.RemoveAll(".test/.git")
 }
+
+func TestCommandPrepareWithoutPassthrough(t *testing.T) {
+	ce := NewCommandExecutor(&memory.MemorySecrets{
+		Secrets: map[string]map[string]string{
+			"test": map[string]string{
+				"SOME_SECRET": "123",
+			},
+		},
+	}, false, "pico", "GLOBAL_")
+
+	ex, err := ce.prepare("test", "./", false, map[string]string{
+		"DATA_DIR": "/data/shared",
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, exec{
+		path: "./",
+		env: map[string]string{
+			"SOME_SECRET": "123",
+			"DATA_DIR":    "/data/shared",
+		},
+		shutdown:        false,
+		passEnvironment: false,
+	}, ex)
+}
+
+func TestCommandPrepareWithGlobal(t *testing.T) {
+	ce := NewCommandExecutor(&memory.MemorySecrets{
+		Secrets: map[string]map[string]string{
+			"test": map[string]string{
+				"SOME_SECRET": "123",
+			},
+			"pico": map[string]string{
+				"GLOBAL_SECRET": "456",
+				"IGNORE":        "this",
+			},
+		},
+	}, false, "pico", "GLOBAL_")
+
+	ex, err := ce.prepare("test", "./", false, map[string]string{
+		"DATA_DIR": "/data/shared",
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, exec{
+		path: "./",
+		env: map[string]string{
+			"SOME_SECRET": "123",
+			"SECRET":      "456",
+			"DATA_DIR":    "/data/shared",
+		},
+		shutdown:        false,
+		passEnvironment: false,
+	}, ex)
+}

go.mod

@@ -3,7 +3,7 @@ module github.com/picostack/pico
 go 1.13
 
 require (
-	github.com/Southclaws/gitwatch v1.3.2
+	github.com/Southclaws/gitwatch v1.3.3
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/eapache/go-resiliency v1.2.0
 	github.com/frankban/quicktest v1.4.1 // indirect

go.sum

@@ -9,6 +9,8 @@ github.com/Southclaws/gitwatch v1.3.1 h1:4XtiujsnxHKSKze3Tb5sWwTdBxSVW/JLbK54ruJ
 github.com/Southclaws/gitwatch v1.3.1/go.mod h1:xCudUiwWxkDYZ69cEhlTwAKIzbG1OpnA/s/pjPIW6gU=
 github.com/Southclaws/gitwatch v1.3.2 h1:zmt571n8ItXgkRJPyCFtFjcymvsFOGcm7JnHNpFDP+8=
 github.com/Southclaws/gitwatch v1.3.2/go.mod h1:xCudUiwWxkDYZ69cEhlTwAKIzbG1OpnA/s/pjPIW6gU=
+github.com/Southclaws/gitwatch v1.3.3 h1:w5AI9IcMEVqb6cPyDjM9tvOI4r26m4UHAl5BVEvgKac=
+github.com/Southclaws/gitwatch v1.3.3/go.mod h1:xCudUiwWxkDYZ69cEhlTwAKIzbG1OpnA/s/pjPIW6gU=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=

main.go

@@ -13,6 +13,7 @@ import (
 
 	_ "github.com/picostack/pico/logger"
 	"github.com/picostack/pico/service"
+	"github.com/picostack/pico/task"
 )
 
 var version = "master"
@@ -38,14 +39,18 @@ this repository has new commits, Pico will automatically reconfigure.`,
 			Usage:     "argument `target` specifies Git repository for configuration.",
 			ArgsUsage: "target",
 			Flags: []cli.Flag{
+				cli.StringFlag{Name: "git-username", EnvVar: "GIT_USERNAME"},
+				cli.StringFlag{Name: "git-password", EnvVar: "GIT_PASSWORD"},
 				cli.StringFlag{Name: "hostname", EnvVar: "HOSTNAME"},
 				cli.StringFlag{Name: "directory", EnvVar: "DIRECTORY", Value: "./cache/"},
-				cli.BoolFlag{Name: "no-ssh", EnvVar: "NO_SSH"},
+				cli.DurationFlag{Name: "pass-env", EnvVar: "PASS_ENV"},
+				cli.BoolFlag{Name: "ssh", EnvVar: "SSH"},
 				cli.DurationFlag{Name: "check-interval", EnvVar: "CHECK_INTERVAL", Value: time.Second * 10},
 				cli.StringFlag{Name: "vault-addr", EnvVar: "VAULT_ADDR"},
 				cli.StringFlag{Name: "vault-token", EnvVar: "VAULT_TOKEN"},
 				cli.StringFlag{Name: "vault-path", EnvVar: "VAULT_PATH", Value: "/secret"},
 				cli.DurationFlag{Name: "vault-renew-interval", EnvVar: "VAULT_RENEW_INTERVAL", Value: time.Hour * 24},
+				cli.StringFlag{Name: "vault-config-path", EnvVar: "VAULT_CONFIG_PATH", Value: "pico"},
 			},
 			Action: func(c *cli.Context) (err error) {
 				if !c.Args().Present() {
@@ -68,15 +73,21 @@ this repository has new commits, Pico will automatically reconfigure.`,
 				zap.L().Debug("initialising service")
 
 				svc, err := service.Initialise(service.Config{
-					Target:        c.Args().First(),
-					Hostname:      hostname,
-					Directory:     c.String("directory"),
-					NoSSH:         c.Bool("no-ssh"),
-					CheckInterval: c.Duration("check-interval"),
-					VaultAddress:  c.String("vault-addr"),
-					VaultToken:    c.String("vault-token"),
-					VaultPath:     c.String("vault-path"),
-					VaultRenewal:  c.Duration("vault-renew-interval"),
+					Target: task.Repo{
+						URL:  c.Args().First(),
+						User: c.String("git-username"),
+						Pass: c.String("git-password"),
+					},
+					Hostname:        hostname,
+					Directory:       c.String("directory"),
+					PassEnvironment: c.Bool("pass-env"),
+					SSH:             c.Bool("ssh"),
+					CheckInterval:   c.Duration("check-interval"),
+					VaultAddress:    c.String("vault-addr"),
+					VaultToken:      c.String("vault-token"),
+					VaultPath:       c.String("vault-path"),
+					VaultRenewal:    c.Duration("vault-renew-interval"),
+					VaultConfig:     c.String("vault-config-path"),
 				})
 				if err != nil {
 					return errors.Wrap(err, "failed to initialise")

reconfigurer/git.go

@@ -23,7 +23,7 @@ type GitProvider struct {
 	hostname      string
 	configRepo    string
 	checkInterval time.Duration
-	ssh           transport.AuthMethod
+	authMethod    transport.AuthMethod
 
 	configWatcher *gitwatch.Session
 }
@@ -34,14 +34,14 @@ func New(
 	hostname string,
 	configRepo string,
 	checkInterval time.Duration,
-	ssh transport.AuthMethod,
+	authMethod transport.AuthMethod,
 ) *GitProvider {
 	return &GitProvider{
 		directory:     directory,
 		hostname:      hostname,
 		configRepo:    configRepo,
 		checkInterval: checkInterval,
-		ssh:           ssh,
+		authMethod:    authMethod,
 	}
 }
 
@@ -104,7 +104,7 @@ func (p *GitProvider) watchConfig() (err error) {
 		[]gitwatch.Repository{{URL: p.configRepo}},
 		p.checkInterval,
 		p.directory,
-		p.ssh,
+		p.authMethod,
 		false)
 	if err != nil {
 		return errors.Wrap(err, "failed to watch config target")

secret/memory/memory.go

@@ -6,12 +6,16 @@ import (
 
 // MemorySecrets implements a simple in-memory secret.Store for testing
 type MemorySecrets struct {
-	Secrets map[string]string
+	Secrets map[string]map[string]string
 }
 
 var _ secret.Store = &MemorySecrets{}
 
 // GetSecretsForTarget implements secret.Store
 func (v *MemorySecrets) GetSecretsForTarget(name string) (map[string]string, error) {
-	return v.Secrets, nil
+	table, ok := v.Secrets[name]
+	if !ok {
+		return nil, nil
+	}
+	return table, nil
 }