Skip to content

Commit 806fdc3

Browse files
ibuildtheclouddrpebcak
ibuildthecloud
authored and
drpebcak
committed
feat: support inline package.json and requirements.txt
1 parent ae328ac commit 806fdc3

25 files changed

+635
-46
lines changed

‎pkg/engine/cmd.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,7 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T
282282
)
283283

284284
if strings.TrimSpace(rest) != "" {
285-
f, err := os.CreateTemp("", version.ProgramName+requiredFileExtensions[args[0]])
285+
f, err := os.CreateTemp(env.Getenv("GPTSCRIPT_TMPDIR", envvars), version.ProgramName+requiredFileExtensions[args[0]])
286286
if err != nil {
287287
return nil, nil, err
288288
}

‎pkg/env/env.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,15 @@ func ToEnvLike(v string) string {
2626
return strings.ToUpper(v)
2727
}
2828

29+
func Getenv(key string, envs []string) string {
30+
for i := len(envs) - 1; i >= 0; i-- {
31+
if k, v, ok := strings.Cut(envs[i], "="); ok && k == key {
32+
return v
33+
}
34+
}
35+
return ""
36+
}
37+
2938
func Matches(cmd []string, bin string) bool {
3039
switch len(cmd) {
3140
case 0:

‎pkg/parser/parser.go

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import (
1616
var (
1717
sepRegex = regexp.MustCompile(`^\s*---+\s*$`)
1818
strictSepRegex = regexp.MustCompile(`^---\n$`)
19-
skipRegex = regexp.MustCompile(`^![-\w]+\s*$`)
19+
skipRegex = regexp.MustCompile(`^![-.:\w]+\s*$`)
2020
)
2121

2222
func normalize(key string) string {
@@ -308,6 +308,8 @@ func Parse(input io.Reader, opts ...Options) (Document, error) {
308308
}
309309
}
310310

311+
nodes = assignMetadata(nodes)
312+
311313
if !opt.AssignGlobals {
312314
return Document{
313315
Nodes: nodes,
@@ -359,6 +361,42 @@ func Parse(input io.Reader, opts ...Options) (Document, error) {
359361
}, nil
360362
}
361363

364+
func assignMetadata(nodes []Node) (result []Node) {
365+
metadata := map[string]map[string]string{}
366+
result = make([]Node, 0, len(nodes))
367+
for _, node := range nodes {
368+
if node.TextNode != nil {
369+
body, ok := strings.CutPrefix(node.TextNode.Text, "!metadata:")
370+
if ok {
371+
line, rest, ok := strings.Cut(body, "\n")
372+
if ok {
373+
toolName, metaKey, ok := strings.Cut(strings.TrimSpace(line), ":")
374+
if ok {
375+
d, ok := metadata[toolName]
376+
if !ok {
377+
d = map[string]string{}
378+
metadata[toolName] = d
379+
}
380+
d[metaKey] = strings.TrimSpace(rest)
381+
}
382+
}
383+
}
384+
}
385+
}
386+
if len(metadata) == 0 {
387+
return nodes
388+
}
389+
390+
for _, node := range nodes {
391+
if node.ToolNode != nil {
392+
node.ToolNode.Tool.MetaData = metadata[node.ToolNode.Tool.Name]
393+
}
394+
result = append(result, node)
395+
}
396+
397+
return
398+
}
399+
362400
func isGPTScriptHashBang(line string) bool {
363401
if !strings.HasPrefix(line, "#!") {
364402
return false

‎pkg/parser/parser_test.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66

77
"github.com/gptscript-ai/gptscript/pkg/types"
88
"github.com/hexops/autogold/v2"
9+
"github.com/stretchr/testify/assert"
910
"github.com/stretchr/testify/require"
1011
)
1112

@@ -239,3 +240,31 @@ share output filters: shared
239240
}},
240241
}}).Equal(t, out)
241242
}
243+
244+
func TestParseMetaData(t *testing.T) {
245+
input := `
246+
name: first
247+
248+
body
249+
---
250+
!metadata:first:package.json
251+
foo=base
252+
f
253+
254+
---
255+
!metadata:first2:requirements.txt
256+
asdf2
257+
258+
---
259+
!metadata:first:requirements.txt
260+
asdf
261+
`
262+
tools, err := ParseTools(strings.NewReader(input))
263+
require.NoError(t, err)
264+
265+
assert.Len(t, tools, 1)
266+
autogold.Expect(map[string]string{
267+
"package.json": "foo=base\nf",
268+
"requirements.txt": "asdf",
269+
}).Equal(t, tools[0].MetaData)
270+
}

‎pkg/repos/get.go

Lines changed: 9 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ const credentialHelpersRepo = "github.com/gptscript-ai/gptscript-credential-help
2626

2727
type Runtime interface {
2828
ID() string
29-
Supports(cmd []string) bool
30-
Setup(ctx context.Context, dataRoot, toolSource string, env []string) ([]string, error)
29+
Supports(tool types.Tool, cmd []string) bool
30+
Setup(ctx context.Context, tool types.Tool, dataRoot, toolSource string, env []string) ([]string, error)
3131
}
3232

3333
type noopRuntime struct {
@@ -37,11 +37,11 @@ func (n noopRuntime) ID() string {
3737
return "none"
3838
}
3939

40-
func (n noopRuntime) Supports(_ []string) bool {
40+
func (n noopRuntime) Supports(_ types.Tool, _ []string) bool {
4141
return false
4242
}
4343

44-
func (n noopRuntime) Setup(_ context.Context, _, _ string, _ []string) ([]string, error) {
44+
func (n noopRuntime) Setup(_ context.Context, _ types.Tool, _, _ string, _ []string) ([]string, error) {
4545
return nil, nil
4646
}
4747

@@ -52,7 +52,6 @@ type Manager struct {
5252
credHelperDirs credentials.CredentialHelperDirs
5353
runtimes []Runtime
5454
credHelperConfig *credHelperConfig
55-
supportLocal bool
5655
}
5756

5857
type credHelperConfig struct {
@@ -62,10 +61,6 @@ type credHelperConfig struct {
6261
env []string
6362
}
6463

65-
func (m *Manager) SetSupportLocal() {
66-
m.supportLocal = true
67-
}
68-
6964
func New(cacheDir string, runtimes ...Runtime) *Manager {
7065
root := filepath.Join(cacheDir, "repos")
7166
return &Manager{
@@ -216,7 +211,7 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e
216211
}
217212
}
218213

219-
newEnv, err := runtime.Setup(ctx, m.runtimeDir, targetFinal, env)
214+
newEnv, err := runtime.Setup(ctx, tool, m.runtimeDir, targetFinal, env)
220215
if err != nil {
221216
return "", nil, err
222217
}
@@ -240,17 +235,10 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e
240235

241236
func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error) {
242237
var isLocal bool
243-
if !m.supportLocal {
244-
if tool.Source.Repo == nil {
245-
return tool.WorkingDir, env, nil
246-
}
247-
248-
if tool.Source.Repo.VCS != "git" {
249-
return "", nil, fmt.Errorf("only git is supported, found VCS %s for %s", tool.Source.Repo.VCS, tool.ID)
250-
}
251-
} else if tool.Source.Repo == nil {
238+
if tool.Source.Repo == nil {
252239
isLocal = true
253-
id := hash.Digest(tool)[:12]
240+
d, _ := json.Marshal(tool)
241+
id := hash.Digest(d)[:12]
254242
tool.Source.Repo = &types.Repo{
255243
VCS: "<local>",
256244
Root: id,
@@ -261,7 +249,7 @@ func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []st
261249
}
262250

263251
for _, runtime := range m.runtimes {
264-
if runtime.Supports(cmd) {
252+
if runtime.Supports(tool, cmd) {
265253
log.Debugf("Runtime %s supports %v", runtime.ID(), cmd)
266254
return m.setup(ctx, runtime, tool, env)
267255
}

‎pkg/repos/runtimes/busybox/busybox.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env"
1919
"github.com/gptscript-ai/gptscript/pkg/hash"
2020
"github.com/gptscript-ai/gptscript/pkg/repos/download"
21+
"github.com/gptscript-ai/gptscript/pkg/types"
2122
)
2223

2324
//go:embed SHASUMS256.txt
@@ -32,7 +33,7 @@ func (r *Runtime) ID() string {
3233
return "busybox"
3334
}
3435

35-
func (r *Runtime) Supports(cmd []string) bool {
36+
func (r *Runtime) Supports(_ types.Tool, cmd []string) bool {
3637
if runtime.GOOS != "windows" {
3738
return false
3839
}
@@ -44,7 +45,7 @@ func (r *Runtime) Supports(cmd []string) bool {
4445
return false
4546
}
4647

47-
func (r *Runtime) Setup(ctx context.Context, dataRoot, _ string, env []string) ([]string, error) {
48+
func (r *Runtime) Setup(ctx context.Context, _ types.Tool, dataRoot, _ string, env []string) ([]string, error) {
4849
binPath, err := r.getRuntime(ctx, dataRoot)
4950
if err != nil {
5051
return nil, err

‎pkg/repos/runtimes/busybox/busybox_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"testing"
1212

1313
"github.com/adrg/xdg"
14+
"github.com/gptscript-ai/gptscript/pkg/types"
1415
"github.com/samber/lo"
1516
"github.com/stretchr/testify/require"
1617
)
@@ -31,7 +32,7 @@ func TestRuntime(t *testing.T) {
3132

3233
r := Runtime{}
3334

34-
s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ())
35+
s, err := r.Setup(context.Background(), types.Tool{}, testCacheHome, "testdata", os.Environ())
3536
require.NoError(t, err)
3637
_, err = os.Stat(filepath.Join(firstPath(s), "busybox.exe"))
3738
if errors.Is(err, fs.ErrNotExist) {

‎pkg/repos/runtimes/golang/golang.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env"
1919
"github.com/gptscript-ai/gptscript/pkg/hash"
2020
"github.com/gptscript-ai/gptscript/pkg/repos/download"
21+
"github.com/gptscript-ai/gptscript/pkg/types"
2122
)
2223

2324
//go:embed digests.txt
@@ -34,11 +35,12 @@ func (r *Runtime) ID() string {
3435
return "go" + r.Version
3536
}
3637

37-
func (r *Runtime) Supports(cmd []string) bool {
38-
return len(cmd) > 0 && cmd[0] == "${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool"
38+
func (r *Runtime) Supports(tool types.Tool, cmd []string) bool {
39+
return tool.Source.IsGit() &&
40+
len(cmd) > 0 && cmd[0] == "${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool"
3941
}
4042

41-
func (r *Runtime) Setup(ctx context.Context, dataRoot, toolSource string, env []string) ([]string, error) {
43+
func (r *Runtime) Setup(ctx context.Context, _ types.Tool, dataRoot, toolSource string, env []string) ([]string, error) {
4244
binPath, err := r.getRuntime(ctx, dataRoot)
4345
if err != nil {
4446
return nil, err

‎pkg/repos/runtimes/golang/golang_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"testing"
1111

1212
"github.com/adrg/xdg"
13+
"github.com/gptscript-ai/gptscript/pkg/types"
1314
"github.com/samber/lo"
1415
"github.com/stretchr/testify/assert"
1516
"github.com/stretchr/testify/require"
@@ -27,7 +28,7 @@ func TestRuntime(t *testing.T) {
2728
Version: "1.22.1",
2829
}
2930

30-
s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ())
31+
s, err := r.Setup(context.Background(), types.Tool{}, testCacheHome, "testdata", os.Environ())
3132
require.NoError(t, err)
3233
p, v, _ := strings.Cut(s[0], "=")
3334
v, _, _ = strings.Cut(v, string(filepath.ListSeparator))

‎pkg/repos/runtimes/node/node.go

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,16 @@ import (
1717
runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env"
1818
"github.com/gptscript-ai/gptscript/pkg/hash"
1919
"github.com/gptscript-ai/gptscript/pkg/repos/download"
20+
"github.com/gptscript-ai/gptscript/pkg/types"
2021
)
2122

2223
//go:embed SHASUMS256.txt.asc
2324
var releasesData []byte
2425

25-
const downloadURL = "https://nodejs.org/dist/%s/"
26+
const (
27+
downloadURL = "https://nodejs.org/dist/%s/"
28+
packageJSON = "package.json"
29+
)
2630

2731
type Runtime struct {
2832
// version something like "3.12"
@@ -35,7 +39,10 @@ func (r *Runtime) ID() string {
3539
return "node" + r.Version
3640
}
3741

38-
func (r *Runtime) Supports(cmd []string) bool {
42+
func (r *Runtime) Supports(tool types.Tool, cmd []string) bool {
43+
if _, hasPackageJSON := tool.MetaData[packageJSON]; !hasPackageJSON && !tool.Source.IsGit() {
44+
return false
45+
}
3946
for _, testCmd := range []string{"node", "npx", "npm"} {
4047
if r.supports(testCmd, cmd) {
4148
return true
@@ -54,17 +61,21 @@ func (r *Runtime) supports(testCmd string, cmd []string) bool {
5461
return runtimeEnv.Matches(cmd, testCmd)
5562
}
5663

57-
func (r *Runtime) Setup(ctx context.Context, dataRoot, toolSource string, env []string) ([]string, error) {
64+
func (r *Runtime) Setup(ctx context.Context, tool types.Tool, dataRoot, toolSource string, env []string) ([]string, error) {
5865
binPath, err := r.getRuntime(ctx, dataRoot)
5966
if err != nil {
6067
return nil, err
6168
}
6269

6370
newEnv := runtimeEnv.AppendPath(env, binPath)
64-
if err := r.runNPM(ctx, toolSource, binPath, append(env, newEnv...)); err != nil {
71+
if err := r.runNPM(ctx, tool, toolSource, binPath, append(env, newEnv...)); err != nil {
6572
return nil, err
6673
}
6774

75+
if _, ok := tool.MetaData[packageJSON]; ok {
76+
newEnv = append(newEnv, "GPTSCRIPT_TMPDIR="+toolSource)
77+
}
78+
6879
return newEnv, nil
6980
}
7081

@@ -100,11 +111,16 @@ func (r *Runtime) getReleaseAndDigest() (string, string, error) {
100111
return "", "", fmt.Errorf("failed to find %s release for os=%s arch=%s", r.ID(), osName(), arch())
101112
}
102113

103-
func (r *Runtime) runNPM(ctx context.Context, toolSource, binDir string, env []string) error {
114+
func (r *Runtime) runNPM(ctx context.Context, tool types.Tool, toolSource, binDir string, env []string) error {
104115
log.InfofCtx(ctx, "Running npm in %s", toolSource)
105116
cmd := debugcmd.New(ctx, filepath.Join(binDir, "npm"), "install")
106117
cmd.Env = env
107118
cmd.Dir = toolSource
119+
if contents, ok := tool.MetaData[packageJSON]; ok {
120+
if err := os.WriteFile(filepath.Join(toolSource, packageJSON), []byte(contents+"\n"), 0644); err != nil {
121+
return err
122+
}
123+
}
108124
return cmd.Run()
109125
}
110126

‎pkg/repos/runtimes/node/node_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"testing"
1111

1212
"github.com/adrg/xdg"
13+
"github.com/gptscript-ai/gptscript/pkg/types"
1314
"github.com/samber/lo"
1415
"github.com/stretchr/testify/require"
1516
)
@@ -28,7 +29,7 @@ func TestRuntime(t *testing.T) {
2829
Version: "20",
2930
}
3031

31-
s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ())
32+
s, err := r.Setup(context.Background(), types.Tool{}, testCacheHome, "testdata", os.Environ())
3233
require.NoError(t, err)
3334
_, err = os.Stat(filepath.Join(firstPath(s), "node.exe"))
3435
if errors.Is(err, fs.ErrNotExist) {
@@ -42,7 +43,7 @@ func TestRuntime21(t *testing.T) {
4243
Version: "21",
4344
}
4445

45-
s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ())
46+
s, err := r.Setup(context.Background(), types.Tool{}, testCacheHome, "testdata", os.Environ())
4647
require.NoError(t, err)
4748
_, err = os.Stat(filepath.Join(firstPath(s), "node.exe"))
4849
if errors.Is(err, fs.ErrNotExist) {

0 commit comments

Comments
 (0)