diff --git a/pkg/repos/runtimes/node/node.go b/pkg/repos/runtimes/node/node.go index aa57b059..4d73c13b 100644 --- a/pkg/repos/runtimes/node/node.go +++ b/pkg/repos/runtimes/node/node.go @@ -26,6 +26,7 @@ var releasesData []byte const ( downloadURL = "https://nodejs.org/dist/%s/" packageJSON = "package.json" + nodeModules = "node_modules" ) type Runtime struct { @@ -64,8 +65,15 @@ func (r *Runtime) supports(testCmd string, cmd []string) bool { func (r *Runtime) GetHash(tool types.Tool) (string, error) { if !tool.Source.IsGit() && tool.WorkingDir != "" { + var prefix string + // This hashes if the node_modules directory was deleted + if s, err := os.Stat(filepath.Join(tool.WorkingDir, nodeModules)); err == nil { + prefix = hash.Digest(tool.WorkingDir + s.ModTime().String())[:7] + } else if s, err := os.Stat(tool.WorkingDir); err == nil { + prefix = hash.Digest(tool.WorkingDir + s.ModTime().String())[:7] + } if s, err := os.Stat(filepath.Join(tool.WorkingDir, packageJSON)); err == nil { - return hash.Digest(tool.WorkingDir + s.ModTime().String())[:7], nil + return prefix + hash.Digest(tool.WorkingDir + s.ModTime().String())[:7], nil } } return "", nil diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 93e40670..3035a1d1 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -172,11 +172,7 @@ func (r *Runner) Chat(ctx context.Context, prevState ChatState, prg types.Progra return resp, err } - if state == nil || state.StartContinuation { - if state != nil { - state = state.WithResumeInput(&input) - input = state.InputContextContinuationInput - } + if state == nil { state, err = r.start(callCtx, state, monitor, env, input) if err != nil { return resp, err @@ -186,11 +182,9 @@ func (r *Runner) Chat(ctx context.Context, prevState ChatState, prg types.Progra state.ResumeInput = &input } - if !state.StartContinuation { - state, err = r.resume(callCtx, monitor, env, state) - if err != nil { - return resp, err - } + state, err = r.resume(callCtx, monitor, env, state) + if err != nil { + return resp, err } if state.Result != nil { @@ -260,6 +254,10 @@ func getToolRefInput(prg *types.Program, ref types.ToolReference, input string) targetArgs := prg.ToolSet[ref.ToolID].Arguments targetKeys := map[string]string{} + if ref.Arg == "*" { + return input, nil + } + if targetArgs == nil { return "", nil } @@ -331,24 +329,10 @@ func getToolRefInput(prg *types.Program, ref types.ToolReference, input string) return string(output), err } -func (r *Runner) getContext(callCtx engine.Context, state *State, monitor Monitor, env []string, input string) (result []engine.InputContext, _ *State, _ error) { +func (r *Runner) getContext(callCtx engine.Context, state *State, monitor Monitor, env []string, input string) (result []engine.InputContext, _ error) { toolRefs, err := callCtx.Tool.GetContextTools(*callCtx.Program) if err != nil { - return nil, nil, err - } - - var newState *State - if state != nil { - cp := *state - newState = &cp - if newState.InputContextContinuation != nil { - newState.InputContexts = nil - newState.InputContextContinuation = nil - newState.InputContextContinuationInput = "" - newState.ResumeInput = state.InputContextContinuationResumeInput - - input = state.InputContextContinuationInput - } + return nil, err } for i, toolRef := range toolRefs { @@ -359,29 +343,16 @@ func (r *Runner) getContext(callCtx engine.Context, state *State, monitor Monito contextInput, err := getToolRefInput(callCtx.Program, toolRef, input) if err != nil { - return nil, nil, err + return nil, err } var content *State - if state != nil && state.InputContextContinuation != nil { - content, err = r.subCallResume(callCtx.Ctx, callCtx, monitor, env, toolRef.ToolID, "", state.InputContextContinuation.WithResumeInput(state.ResumeInput), engine.ContextToolCategory) - } else { - content, err = r.subCall(callCtx.Ctx, callCtx, monitor, env, toolRef.ToolID, contextInput, "", engine.ContextToolCategory) - } + content, err = r.subCall(callCtx.Ctx, callCtx, monitor, env, toolRef.ToolID, contextInput, "", engine.ContextToolCategory) if err != nil { - return nil, nil, err + return nil, err } if content.Continuation != nil { - if newState == nil { - newState = &State{} - } - newState.InputContexts = result - newState.InputContextContinuation = content - newState.InputContextContinuationInput = input - if state != nil { - newState.InputContextContinuationResumeInput = state.ResumeInput - } - return nil, newState, nil + return nil, fmt.Errorf("invalid state: context tool [%s] can not result in a continuation", toolRef.ToolID) } result = append(result, engine.InputContext{ ToolID: toolRef.ToolID, @@ -389,7 +360,7 @@ func (r *Runner) getContext(callCtx engine.Context, state *State, monitor Monito }) } - return result, newState, nil + return result, nil } func (r *Runner) call(callCtx engine.Context, monitor Monitor, env []string, input string) (*State, error) { @@ -397,9 +368,6 @@ func (r *Runner) call(callCtx engine.Context, monitor Monitor, env []string, inp if err != nil { return nil, err } - if result.StartContinuation { - return result, nil - } return r.resume(callCtx, monitor, env, result) } @@ -431,15 +399,10 @@ func (r *Runner) start(callCtx engine.Context, state *State, monitor Monitor, en } } - var newState *State - callCtx.InputContext, newState, err = r.getContext(callCtx, state, monitor, env, input) + callCtx.InputContext, err = r.getContext(callCtx, state, monitor, env, input) if err != nil { return nil, err } - if newState != nil && newState.InputContextContinuation != nil { - newState.StartContinuation = true - return newState, nil - } e := engine.Engine{ Model: r.c, @@ -489,11 +452,7 @@ type State struct { SubCalls []SubCallResult `json:"subCalls,omitempty"` SubCallID string `json:"subCallID,omitempty"` - InputContexts []engine.InputContext `json:"inputContexts,omitempty"` - InputContextContinuation *State `json:"inputContextContinuation,omitempty"` - InputContextContinuationInput string `json:"inputContextContinuationInput,omitempty"` - InputContextContinuationResumeInput *string `json:"inputContextContinuationResumeInput,omitempty"` - StartContinuation bool `json:"startContinuation,omitempty"` + InputContexts []engine.InputContext `json:"inputContexts,omitempty"` } func (s State) WithResumeInput(input *string) *State { @@ -506,10 +465,6 @@ func (s State) ContinuationContentToolID() (string, error) { return s.ContinuationToolID, nil } - if s.InputContextContinuation != nil { - return s.InputContextContinuation.ContinuationContentToolID() - } - for _, subCall := range s.SubCalls { if s.SubCallID == subCall.CallID { return subCall.State.ContinuationContentToolID() @@ -523,10 +478,6 @@ func (s State) ContinuationContent() (string, error) { return *s.Continuation.Result, nil } - if s.InputContextContinuation != nil { - return s.InputContextContinuation.ContinuationContent() - } - for _, subCall := range s.SubCalls { if s.SubCallID == subCall.CallID { return subCall.State.ContinuationContent() @@ -545,10 +496,6 @@ func (r *Runner) resume(callCtx engine.Context, monitor Monitor, env []string, s retState, retErr = r.handleOutput(callCtx, monitor, env, retState, retErr) }() - if state.StartContinuation { - return nil, fmt.Errorf("invalid state, resume should not have StartContinuation set to true") - } - if state.Continuation == nil { return nil, errors.New("invalid state, resume should have Continuation data") } @@ -653,8 +600,12 @@ func (r *Runner) resume(callCtx engine.Context, monitor Monitor, env []string, s contentInput = state.Continuation.State.Input } - callCtx.InputContext, state, err = r.getContext(callCtx, state, monitor, env, contentInput) - if err != nil || state.InputContextContinuation != nil { + if state.ResumeInput != nil { + contentInput = *state.ResumeInput + } + + callCtx.InputContext, err = r.getContext(callCtx, state, monitor, env, contentInput) + if err != nil { return state, err } @@ -764,10 +715,6 @@ func (r *Runner) subCalls(callCtx engine.Context, monitor Monitor, env []string, callCtx.LastReturn = state.Continuation } - if state.InputContextContinuation != nil { - return state, nil, nil - } - if state.SubCallID != "" { if state.ResumeInput == nil { return nil, nil, fmt.Errorf("invalid state, input must be set for sub call continuation on callID [%s]", state.SubCallID) diff --git a/pkg/tests/runner2_test.go b/pkg/tests/runner2_test.go new file mode 100644 index 00000000..12ac4fa0 --- /dev/null +++ b/pkg/tests/runner2_test.go @@ -0,0 +1,34 @@ +package tests + +import ( + "context" + "testing" + + "github.com/gptscript-ai/gptscript/pkg/loader" + "github.com/gptscript-ai/gptscript/pkg/tests/tester" + "github.com/stretchr/testify/require" +) + +func TestContextWithAsterick(t *testing.T) { + r := tester.NewRunner(t) + prg, err := loader.ProgramFromSource(context.Background(), ` +chat: true +context: foo with * + +Say hi + +--- +name: foo + +#!/bin/bash + +echo This is the input: ${GPTSCRIPT_INPUT} +`, "") + require.NoError(t, err) + + resp, err := r.Chat(context.Background(), nil, prg, nil, "input 1") + r.AssertStep(t, resp, err) + + resp, err = r.Chat(context.Background(), resp.State, prg, nil, "input 2") + r.AssertStep(t, resp, err) +} diff --git a/pkg/tests/runner_test.go b/pkg/tests/runner_test.go index 141e6aff..f21f936e 100644 --- a/pkg/tests/runner_test.go +++ b/pkg/tests/runner_test.go @@ -212,82 +212,8 @@ func TestContextSubChat(t *testing.T) { prg, err := r.Load("") require.NoError(t, err) - resp, err := r.Chat(context.Background(), nil, prg, os.Environ(), "User 1") - require.NoError(t, err) - r.AssertResponded(t) - assert.False(t, resp.Done) - autogold.Expect("Assistant Response 1 - from chatbot1").Equal(t, resp.Content) - autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step1")) - - r.RespondWith(tester.Result{ - Content: []types.ContentPart{ - { - ToolCall: &types.CompletionToolCall{ - ID: "call_2", - Function: types.CompletionFunctionCall{ - Name: types.ToolNormalizer("sys.chat.finish"), - Arguments: "Response from context chatbot", - }, - }, - }, - }, - }, tester.Result{ - Text: "Assistant Response 2 - from context tool", - }, tester.Result{ - Text: "Assistant Response 3 - from main chat tool", - }) - resp, err = r.Chat(context.Background(), resp.State, prg, os.Environ(), "User 2") - require.NoError(t, err) - r.AssertResponded(t) - assert.False(t, resp.Done) - autogold.Expect("Assistant Response 3 - from main chat tool").Equal(t, resp.Content) - autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step2")) - - r.RespondWith(tester.Result{ - Content: []types.ContentPart{ - { - ToolCall: &types.CompletionToolCall{ - ID: "call_3", - Function: types.CompletionFunctionCall{ - Name: "chatbot", - Arguments: "Input to chatbot1 on resume", - }, - }, - }, - }, - }, tester.Result{ - Text: "Assistant Response 4 - from chatbot1", - }) - resp, err = r.Chat(context.Background(), resp.State, prg, os.Environ(), "User 3") - require.NoError(t, err) - r.AssertResponded(t) - assert.False(t, resp.Done) - autogold.Expect("Assistant Response 3 - from main chat tool").Equal(t, resp.Content) - autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step3")) - - r.RespondWith(tester.Result{ - Content: []types.ContentPart{ - { - ToolCall: &types.CompletionToolCall{ - ID: "call_4", - Function: types.CompletionFunctionCall{ - Name: types.ToolNormalizer("sys.chat.finish"), - Arguments: "Response from context chatbot after resume", - }, - }, - }, - }, - }, tester.Result{ - Text: "Assistant Response 5 - from context tool resume", - }, tester.Result{ - Text: "Assistant Response 6 - from main chat tool resume", - }) - resp, err = r.Chat(context.Background(), resp.State, prg, os.Environ(), "User 4") - require.NoError(t, err) - r.AssertResponded(t) - assert.False(t, resp.Done) - autogold.Expect("Assistant Response 6 - from main chat tool resume").Equal(t, resp.Content) - autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step4")) + _, err = r.Chat(context.Background(), nil, prg, os.Environ(), "User 1") + autogold.Expect("invalid state: context tool [testdata/TestContextSubChat/test.gpt:subtool] can not result in a continuation").Equal(t, err.Error()) } func TestSubChat(t *testing.T) { diff --git a/pkg/tests/testdata/TestContextSubChat/call10-resp.golden b/pkg/tests/testdata/TestContextSubChat/call10-resp.golden deleted file mode 100644 index 144ca8d9..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call10-resp.golden +++ /dev/null @@ -1,9 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "text": "Assistant Response 6 - from main chat tool resume" - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call3-resp.golden b/pkg/tests/testdata/TestContextSubChat/call3-resp.golden deleted file mode 100644 index b116d066..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call3-resp.golden +++ /dev/null @@ -1,16 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_2", - "function": { - "name": "chatFinish", - "arguments": "Response from context chatbot" - } - } - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call3.golden b/pkg/tests/testdata/TestContextSubChat/call3.golden deleted file mode 100644 index 55ad402f..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call3.golden +++ /dev/null @@ -1,61 +0,0 @@ -`{ - "model": "gpt-4o", - "internalSystemPrompt": false, - "tools": [ - { - "function": { - "toolID": "sys.chat.finish", - "name": "chatFinish", - "description": "Concludes the conversation. This can not be used to ask a question.", - "parameters": { - "properties": { - "return": { - "description": "The instructed value to return or a summary of the dialog if no value is instructed", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "This is a chatbot" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "Input to chatbot1" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 1 - from chatbot1" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "User 1" - } - ], - "usage": {} - } - ], - "chat": true -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call4.golden b/pkg/tests/testdata/TestContextSubChat/call4.golden deleted file mode 100644 index e1fb91ea..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call4.golden +++ /dev/null @@ -1,64 +0,0 @@ -`{ - "model": "gpt-4o", - "tools": [ - { - "function": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "name": "chatbot", - "parameters": { - "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the assistant. This may be an instruction or question.", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Call chatbot" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_1", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1" - } - } - } - ], - "usage": {} - }, - { - "role": "tool", - "content": [ - { - "text": "Response from context chatbot" - } - ], - "toolCall": { - "index": 0, - "id": "call_1", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1" - } - }, - "usage": {} - } - ] -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call5-resp.golden b/pkg/tests/testdata/TestContextSubChat/call5-resp.golden deleted file mode 100644 index e49a8481..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call5-resp.golden +++ /dev/null @@ -1,9 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "text": "Assistant Response 3 - from main chat tool" - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call6-resp.golden b/pkg/tests/testdata/TestContextSubChat/call6-resp.golden deleted file mode 100644 index 6807fce9..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call6-resp.golden +++ /dev/null @@ -1,16 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_3", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1 on resume" - } - } - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call6.golden b/pkg/tests/testdata/TestContextSubChat/call6.golden deleted file mode 100644 index 225401db..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call6.golden +++ /dev/null @@ -1,31 +0,0 @@ -`{ - "model": "gpt-4o", - "tools": [ - { - "function": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "name": "chatbot", - "parameters": { - "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the assistant. This may be an instruction or question.", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Call chatbot" - } - ], - "usage": {} - } - ] -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call7.golden b/pkg/tests/testdata/TestContextSubChat/call7.golden deleted file mode 100644 index b0ef4e39..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call7.golden +++ /dev/null @@ -1,43 +0,0 @@ -`{ - "model": "gpt-4o", - "internalSystemPrompt": false, - "tools": [ - { - "function": { - "toolID": "sys.chat.finish", - "name": "chatFinish", - "description": "Concludes the conversation. This can not be used to ask a question.", - "parameters": { - "properties": { - "return": { - "description": "The instructed value to return or a summary of the dialog if no value is instructed", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "This is a chatbot" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "Input to chatbot1 on resume" - } - ], - "usage": {} - } - ], - "chat": true -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call8-resp.golden b/pkg/tests/testdata/TestContextSubChat/call8-resp.golden deleted file mode 100644 index 2e608b31..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call8-resp.golden +++ /dev/null @@ -1,16 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_4", - "function": { - "name": "chatFinish", - "arguments": "Response from context chatbot after resume" - } - } - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call8.golden b/pkg/tests/testdata/TestContextSubChat/call8.golden deleted file mode 100644 index 3d0db61b..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call8.golden +++ /dev/null @@ -1,61 +0,0 @@ -`{ - "model": "gpt-4o", - "internalSystemPrompt": false, - "tools": [ - { - "function": { - "toolID": "sys.chat.finish", - "name": "chatFinish", - "description": "Concludes the conversation. This can not be used to ask a question.", - "parameters": { - "properties": { - "return": { - "description": "The instructed value to return or a summary of the dialog if no value is instructed", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "This is a chatbot" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "Input to chatbot1 on resume" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 4 - from chatbot1" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "User 4" - } - ], - "usage": {} - } - ], - "chat": true -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call9-resp.golden b/pkg/tests/testdata/TestContextSubChat/call9-resp.golden deleted file mode 100644 index 4424246d..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call9-resp.golden +++ /dev/null @@ -1,9 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "text": "Assistant Response 5 - from context tool resume" - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call9.golden b/pkg/tests/testdata/TestContextSubChat/call9.golden deleted file mode 100644 index 33768f26..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call9.golden +++ /dev/null @@ -1,64 +0,0 @@ -`{ - "model": "gpt-4o", - "tools": [ - { - "function": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "name": "chatbot", - "parameters": { - "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the assistant. This may be an instruction or question.", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Call chatbot" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_3", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1 on resume" - } - } - } - ], - "usage": {} - }, - { - "role": "tool", - "content": [ - { - "text": "Response from context chatbot after resume" - } - ], - "toolCall": { - "index": 0, - "id": "call_3", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1 on resume" - } - }, - "usage": {} - } - ] -}` diff --git a/pkg/tests/testdata/TestContextSubChat/step1.golden b/pkg/tests/testdata/TestContextSubChat/step1.golden deleted file mode 100644 index 2ffb138e..00000000 --- a/pkg/tests/testdata/TestContextSubChat/step1.golden +++ /dev/null @@ -1,146 +0,0 @@ -`{ - "done": false, - "content": "Assistant Response 1 - from chatbot1", - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "state": { - "inputContextContinuation": { - "continuation": { - "state": { - "completion": { - "model": "gpt-4o", - "tools": [ - { - "function": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "name": "chatbot", - "parameters": { - "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the assistant. This may be an instruction or question.", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Call chatbot" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_1", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1" - } - } - } - ], - "usage": {} - } - ] - }, - "pending": { - "call_1": { - "index": 0, - "id": "call_1", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1" - } - } - } - }, - "calls": { - "call_1": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "input": "Input to chatbot1" - } - } - }, - "subCalls": [ - { - "toolId": "testdata/TestContextSubChat/test.gpt:chatbot", - "callId": "call_1", - "state": { - "continuation": { - "state": { - "input": "Input to chatbot1", - "completion": { - "model": "gpt-4o", - "internalSystemPrompt": false, - "tools": [ - { - "function": { - "toolID": "sys.chat.finish", - "name": "chatFinish", - "description": "Concludes the conversation. This can not be used to ask a question.", - "parameters": { - "properties": { - "return": { - "description": "The instructed value to return or a summary of the dialog if no value is instructed", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "This is a chatbot" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "Input to chatbot1" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 1 - from chatbot1" - } - ], - "usage": {} - } - ], - "chat": true - } - }, - "result": "Assistant Response 1 - from chatbot1" - }, - "continuationToolID": "testdata/TestContextSubChat/test.gpt:chatbot" - } - } - ], - "subCallID": "call_1" - }, - "inputContextContinuationInput": "User 1", - "startContinuation": true - } -}` diff --git a/pkg/tests/testdata/TestContextSubChat/step3.golden b/pkg/tests/testdata/TestContextSubChat/step3.golden deleted file mode 100644 index 0ccb188b..00000000 --- a/pkg/tests/testdata/TestContextSubChat/step3.golden +++ /dev/null @@ -1,188 +0,0 @@ -`{ - "done": false, - "content": "Assistant Response 3 - from main chat tool", - "toolID": "testdata/TestContextSubChat/test.gpt:", - "state": { - "continuation": { - "state": { - "input": "User 1", - "completion": { - "model": "gpt-4o", - "internalSystemPrompt": false, - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Assistant Response 2 - from context tool\nHello" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "User 1" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 3 - from main chat tool" - } - ], - "usage": {} - } - ], - "chat": true - } - }, - "result": "Assistant Response 3 - from main chat tool" - }, - "continuationToolID": "testdata/TestContextSubChat/test.gpt:", - "resumeInput": "User 3", - "inputContextContinuation": { - "continuation": { - "state": { - "completion": { - "model": "gpt-4o", - "tools": [ - { - "function": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "name": "chatbot", - "parameters": { - "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the assistant. This may be an instruction or question.", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Call chatbot" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_3", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1 on resume" - } - } - } - ], - "usage": {} - } - ] - }, - "pending": { - "call_3": { - "index": 0, - "id": "call_3", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1 on resume" - } - } - } - }, - "calls": { - "call_3": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "input": "Input to chatbot1 on resume" - } - } - }, - "subCalls": [ - { - "toolId": "testdata/TestContextSubChat/test.gpt:chatbot", - "callId": "call_3", - "state": { - "continuation": { - "state": { - "input": "Input to chatbot1 on resume", - "completion": { - "model": "gpt-4o", - "internalSystemPrompt": false, - "tools": [ - { - "function": { - "toolID": "sys.chat.finish", - "name": "chatFinish", - "description": "Concludes the conversation. This can not be used to ask a question.", - "parameters": { - "properties": { - "return": { - "description": "The instructed value to return or a summary of the dialog if no value is instructed", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "This is a chatbot" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "Input to chatbot1 on resume" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 4 - from chatbot1" - } - ], - "usage": {} - } - ], - "chat": true - } - }, - "result": "Assistant Response 4 - from chatbot1" - }, - "continuationToolID": "testdata/TestContextSubChat/test.gpt:chatbot" - } - } - ], - "subCallID": "call_3" - }, - "inputContextContinuationInput": "User 1", - "inputContextContinuationResumeInput": "User 3" - } -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call7-resp.golden b/pkg/tests/testdata/TestContextWithAsterick/call1-resp.golden similarity index 58% rename from pkg/tests/testdata/TestContextSubChat/call7-resp.golden rename to pkg/tests/testdata/TestContextWithAsterick/call1-resp.golden index 3e0c5f3c..2861a036 100644 --- a/pkg/tests/testdata/TestContextSubChat/call7-resp.golden +++ b/pkg/tests/testdata/TestContextWithAsterick/call1-resp.golden @@ -2,7 +2,7 @@ "role": "assistant", "content": [ { - "text": "Assistant Response 4 - from chatbot1" + "text": "TEST RESULT CALL: 1" } ], "usage": {} diff --git a/pkg/tests/testdata/TestContextSubChat/call5.golden b/pkg/tests/testdata/TestContextWithAsterick/call1.golden similarity index 75% rename from pkg/tests/testdata/TestContextSubChat/call5.golden rename to pkg/tests/testdata/TestContextWithAsterick/call1.golden index 2b8cf41e..6d9538ce 100644 --- a/pkg/tests/testdata/TestContextSubChat/call5.golden +++ b/pkg/tests/testdata/TestContextWithAsterick/call1.golden @@ -6,7 +6,7 @@ "role": "system", "content": [ { - "text": "Assistant Response 2 - from context tool\nHello" + "text": "This is the input: input 1\n\nSay hi" } ], "usage": {} @@ -15,7 +15,7 @@ "role": "user", "content": [ { - "text": "User 1" + "text": "input 1" } ], "usage": {} diff --git a/pkg/tests/testdata/TestContextSubChat/call4-resp.golden b/pkg/tests/testdata/TestContextWithAsterick/call2-resp.golden similarity index 56% rename from pkg/tests/testdata/TestContextSubChat/call4-resp.golden rename to pkg/tests/testdata/TestContextWithAsterick/call2-resp.golden index a86ae187..997ca1b9 100644 --- a/pkg/tests/testdata/TestContextSubChat/call4-resp.golden +++ b/pkg/tests/testdata/TestContextWithAsterick/call2-resp.golden @@ -2,7 +2,7 @@ "role": "assistant", "content": [ { - "text": "Assistant Response 2 - from context tool" + "text": "TEST RESULT CALL: 2" } ], "usage": {} diff --git a/pkg/tests/testdata/TestContextSubChat/call10.golden b/pkg/tests/testdata/TestContextWithAsterick/call2.golden similarity index 72% rename from pkg/tests/testdata/TestContextSubChat/call10.golden rename to pkg/tests/testdata/TestContextWithAsterick/call2.golden index c8c98651..f159014c 100644 --- a/pkg/tests/testdata/TestContextSubChat/call10.golden +++ b/pkg/tests/testdata/TestContextWithAsterick/call2.golden @@ -6,7 +6,7 @@ "role": "system", "content": [ { - "text": "Assistant Response 5 - from context tool resume\nHello" + "text": "This is the input: input 2\n\nSay hi" } ], "usage": {} @@ -15,7 +15,7 @@ "role": "user", "content": [ { - "text": "User 1" + "text": "input 1" } ], "usage": {} @@ -24,7 +24,7 @@ "role": "assistant", "content": [ { - "text": "Assistant Response 3 - from main chat tool" + "text": "TEST RESULT CALL: 1" } ], "usage": {} @@ -33,7 +33,7 @@ "role": "user", "content": [ { - "text": "User 3" + "text": "input 2" } ], "usage": {} diff --git a/pkg/tests/testdata/TestContextSubChat/step2.golden b/pkg/tests/testdata/TestContextWithAsterick/step1.golden similarity index 62% rename from pkg/tests/testdata/TestContextSubChat/step2.golden rename to pkg/tests/testdata/TestContextWithAsterick/step1.golden index dfcb2b96..cc42e6a6 100644 --- a/pkg/tests/testdata/TestContextSubChat/step2.golden +++ b/pkg/tests/testdata/TestContextWithAsterick/step1.golden @@ -1,11 +1,11 @@ `{ "done": false, - "content": "Assistant Response 3 - from main chat tool", - "toolID": "testdata/TestContextSubChat/test.gpt:", + "content": "TEST RESULT CALL: 1", + "toolID": "inline:", "state": { "continuation": { "state": { - "input": "User 1", + "input": "input 1", "completion": { "model": "gpt-4o", "internalSystemPrompt": false, @@ -14,7 +14,7 @@ "role": "system", "content": [ { - "text": "Assistant Response 2 - from context tool\nHello" + "text": "This is the input: input 1\n\nSay hi" } ], "usage": {} @@ -23,7 +23,7 @@ "role": "user", "content": [ { - "text": "User 1" + "text": "input 1" } ], "usage": {} @@ -32,7 +32,7 @@ "role": "assistant", "content": [ { - "text": "Assistant Response 3 - from main chat tool" + "text": "TEST RESULT CALL: 1" } ], "usage": {} @@ -41,8 +41,8 @@ "chat": true } }, - "result": "Assistant Response 3 - from main chat tool" + "result": "TEST RESULT CALL: 1" }, - "continuationToolID": "testdata/TestContextSubChat/test.gpt:" + "continuationToolID": "inline:" } }` diff --git a/pkg/tests/testdata/TestContextSubChat/step4.golden b/pkg/tests/testdata/TestContextWithAsterick/step2.golden similarity index 65% rename from pkg/tests/testdata/TestContextSubChat/step4.golden rename to pkg/tests/testdata/TestContextWithAsterick/step2.golden index 5e95d626..02bc92fe 100644 --- a/pkg/tests/testdata/TestContextSubChat/step4.golden +++ b/pkg/tests/testdata/TestContextWithAsterick/step2.golden @@ -1,11 +1,11 @@ `{ "done": false, - "content": "Assistant Response 6 - from main chat tool resume", - "toolID": "testdata/TestContextSubChat/test.gpt:", + "content": "TEST RESULT CALL: 2", + "toolID": "inline:", "state": { "continuation": { "state": { - "input": "User 1", + "input": "input 1", "completion": { "model": "gpt-4o", "internalSystemPrompt": false, @@ -14,7 +14,7 @@ "role": "system", "content": [ { - "text": "Assistant Response 5 - from context tool resume\nHello" + "text": "This is the input: input 2\n\nSay hi" } ], "usage": {} @@ -23,7 +23,7 @@ "role": "user", "content": [ { - "text": "User 1" + "text": "input 1" } ], "usage": {} @@ -32,7 +32,7 @@ "role": "assistant", "content": [ { - "text": "Assistant Response 3 - from main chat tool" + "text": "TEST RESULT CALL: 1" } ], "usage": {} @@ -41,7 +41,7 @@ "role": "user", "content": [ { - "text": "User 3" + "text": "input 2" } ], "usage": {} @@ -50,7 +50,7 @@ "role": "assistant", "content": [ { - "text": "Assistant Response 6 - from main chat tool resume" + "text": "TEST RESULT CALL: 2" } ], "usage": {} @@ -59,8 +59,8 @@ "chat": true } }, - "result": "Assistant Response 6 - from main chat tool resume" + "result": "TEST RESULT CALL: 2" }, - "continuationToolID": "testdata/TestContextSubChat/test.gpt:" + "continuationToolID": "inline:" } }` diff --git a/pkg/tests/tester/runner.go b/pkg/tests/tester/runner.go index 66337ff5..b460ce18 100644 --- a/pkg/tests/tester/runner.go +++ b/pkg/tests/tester/runner.go @@ -135,7 +135,8 @@ func (c *Client) Call(_ context.Context, messageRequest types.CompletionRequest, type Runner struct { *runner.Runner - Client *Client + Client *Client + StepAsserted int } func (r *Runner) RunDefault() string { @@ -166,6 +167,21 @@ func (r *Runner) AssertResponded(t *testing.T) { require.Len(t, r.Client.result, 0) } +func toJSONString(t *testing.T, v interface{}) string { + t.Helper() + x, err := json.MarshalIndent(v, "", " ") + require.NoError(t, err) + return string(x) +} + +func (r *Runner) AssertStep(t *testing.T, resp runner.ChatResponse, err error) { + t.Helper() + r.StepAsserted++ + require.NoError(t, err) + r.AssertResponded(t) + autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+fmt.Sprintf("/step%d", r.StepAsserted))) +} + func (r *Runner) RespondWith(result ...Result) { r.Client.result = append(r.Client.result, result...) }