Files
Michael Suchacz 67a781ce1d feat: remove model_intent from the execute, process_output, and advisor tools (#30397)
The built-in `execute`, `process_output`, and `advisor` tools offered an
optional `model_intent` argument whose text replaced the tool row's
default label. Models fill it inconsistently, so most chats mixed intent
labels with plain command labels. On dogfood over three days, Claude
Opus 5.5 sent it on 17% of `execute` calls through Bedrock and 30%
through the Anthropic API. The rate follows reasoning effort: the few
chats at max or xhigh effort sent it on every call, while chats at high,
medium, or default effort sent it on about a third of calls or fewer.

This removes the argument from the three tool schemas and always renders
the command-based labels ("Ran ...", "Started ... in the background",
"Checked ...", "Consulted the advisor"). It also deletes the frontend
helpers, tests, and stories that existed only for the intent labels.

- Old chats keep working. chatd's decoders and its tool input check
ignore undeclared keys, so calls that still carry `model_intent` run as
before and their rows show the default label.
- The per-server `model_intent` option for MCP tools is unchanged. That
field is required in the schema, and Opus 5.5 fills it on 96 to 99.7% of
those calls.
- The MCP workspace `execute` and `process_output` tools now match the
Coder Agents contract apart from their `workspace` argument, so the
golden test drops its Agents-only exception.
- Pixel diffs from this change: the deleted intent stories and the new
default labels in `AllToolIconsTranscript`,
`PolicyBadgeCoversEveryRenderer`, and
`StreamedAdvisorResultOverlaysDurableCall`.

## Testing

- `go test` for `chattool`, `chatadvisor`, `toolschema`, `coderd/mcp`,
and `coderd/x/chatd -run Advisor`; `go vet ./coderd/x/chatd/...
./coderd/mcp/...`.
- Red check: re-adding the field to `ExecuteArgs` fails
`TestWorkspaceToolsGolden`.
- Vitest unit tests for the tool utils, the generic tool row, and chat
parsing (5 files) and the affected Storybook files (4 files); `biome`,
`oxlint`, and `tsc`.
- `make pre-commit` passed on the rebased head. The pre-push hook was
skipped per the maintainer's standing preference; CI covers it.

## Review record

- `frontend-review` (FE1 to FE10): all PASS or N/A.
- `coder-frontend-review` (8 parallel reviewer roles): no findings. Two
optional side notes were declined: a nested ternary for `ExecuteTool`'s
label (SD3 discourages them) and a story for an old call that still
carries `model_intent` (TS7: it shows the same state as the existing
default-label stories).

> Xum opened this PR on behalf of @ibetitsmike.
2026-10-06 22:50:28 +02:00

837 lines
26 KiB
Go

package chattool_test
import (
"context"
"encoding/json"
"testing"
"time"
"charm.land/fantasy"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"golang.org/x/xerrors"
"github.com/coder/coder/v2/coderd/x/chatd/chattool"
"github.com/coder/coder/v2/codersdk/workspacesdk"
"github.com/coder/coder/v2/codersdk/workspacesdk/agentconnmock"
"github.com/coder/coder/v2/testutil"
)
func TestExecuteTool(t *testing.T) {
t.Parallel()
t.Run("SchemaDisclosesShell", func(t *testing.T) {
t.Parallel()
tool := chattool.Execute(chattool.ExecuteOptions{})
info := tool.Info()
assert.Contains(t, info.Description, `Runs under "sh -c" (POSIX)`)
commandParam, ok := info.Parameters["command"].(map[string]any)
require.True(t, ok)
assert.Equal(t, "string", commandParam["type"])
assert.Contains(t, commandParam["description"], `Runs under "sh -c" (POSIX)`)
})
t.Run("EmptyCommand", func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
tool := newExecuteTool(t, mockConn)
resp, err := tool.Run(context.Background(), fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: `{"command":""}`,
})
require.NoError(t, err)
assert.True(t, resp.IsError)
assert.Contains(t, resp.Content, "command is required")
})
t.Run("AmpersandDetection", func(t *testing.T) {
t.Parallel()
tests := []struct {
name string
command string
runInBackground *bool
wantCommand string
wantBackground bool
wantBackgroundResp bool // true if the response should contain a background_process_id
comment string
}{
{
name: "SimpleBackground",
command: "cmd &",
wantCommand: "cmd",
wantBackground: true,
wantBackgroundResp: true,
comment: "Trailing & is correctly detected and stripped.",
},
{
name: "TrailingDoubleAmpersand",
command: "cmd &&",
wantCommand: "cmd &&",
wantBackground: false,
wantBackgroundResp: false,
comment: "Ends with &&, excluded by the && suffix check.",
},
{
name: "NoAmpersand",
command: "cmd",
wantCommand: "cmd",
wantBackground: false,
wantBackgroundResp: false,
},
{
name: "ChainThenBackground",
command: "cmd1 && cmd2 &",
wantCommand: "cmd1 && cmd2",
wantBackground: true,
wantBackgroundResp: true,
comment: "Ends with & but not &&, so it gets promoted " +
"to background and the trailing & is stripped. " +
"The remaining command runs in background mode.",
},
{
// "|&" is bash's pipe-stderr operator, not
// backgrounding. It must not be detected as a
// trailing "&".
name: "BashPipeStderr",
command: "cmd |&",
wantCommand: "cmd |&",
wantBackground: false,
wantBackgroundResp: false,
},
{
name: "AlreadyBackgroundWithTrailingAmpersand",
command: "cmd &",
runInBackground: ptr(true),
wantCommand: "cmd &",
wantBackground: true,
wantBackgroundResp: true,
comment: "When run_in_background is already true, " +
"the stripping logic is skipped, preserving " +
"the original command.",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
var capturedReq workspacesdk.StartProcessRequest
mockConn.EXPECT().
StartProcess(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, req workspacesdk.StartProcessRequest) (workspacesdk.StartProcessResponse, error) {
capturedReq = req
return workspacesdk.StartProcessResponse{ID: "proc-1"}, nil
})
// For foreground cases, ProcessOutput is polled.
exitCode := 0
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
Return(workspacesdk.ProcessOutputResponse{
Running: false,
ExitCode: &exitCode,
}, nil).
AnyTimes()
tool := newExecuteTool(t, mockConn)
input := map[string]any{"command": tc.command}
if tc.runInBackground != nil {
input["run_in_background"] = *tc.runInBackground
}
inputJSON, err := json.Marshal(input)
require.NoError(t, err)
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: string(inputJSON),
})
require.NoError(t, err)
assert.False(t, resp.IsError, "response should not be an error")
assert.Equal(t, tc.wantCommand, capturedReq.Command,
"command passed to StartProcess")
assert.Equal(t, tc.wantBackground, capturedReq.Background,
"background flag passed to StartProcess")
var result chattool.ExecuteResult
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
if tc.wantBackgroundResp {
assert.NotEmpty(t, result.BackgroundProcessID,
"expected background_process_id in response")
} else {
assert.Empty(t, result.BackgroundProcessID,
"expected no background_process_id")
}
})
}
})
t.Run("ForegroundSuccess", func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
var capturedReq workspacesdk.StartProcessRequest
mockConn.EXPECT().
StartProcess(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, req workspacesdk.StartProcessRequest) (workspacesdk.StartProcessResponse, error) {
capturedReq = req
return workspacesdk.StartProcessResponse{ID: "proc-1"}, nil
})
exitCode := 0
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
Return(workspacesdk.ProcessOutputResponse{
Running: false,
ExitCode: &exitCode,
Output: "hello world",
}, nil)
tool := newExecuteTool(t, mockConn)
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: `{"command":"echo hello"}`,
})
require.NoError(t, err)
assert.False(t, resp.IsError)
var result chattool.ExecuteResult
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
assert.True(t, result.Success)
assert.Equal(t, 0, result.ExitCode)
assert.Equal(t, "hello world", result.Output)
assert.Empty(t, result.BackgroundProcessID)
assert.Equal(t, "true", capturedReq.Env["CODER_CHAT_AGENT"])
assert.NotContains(t, capturedReq.Env, "AGENT_BROWSER_SESSION")
})
t.Run("AgentBrowserSessionEnv", func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
var capturedReq workspacesdk.StartProcessRequest
mockConn.EXPECT().
StartProcess(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, req workspacesdk.StartProcessRequest) (workspacesdk.StartProcessResponse, error) {
capturedReq = req
return workspacesdk.StartProcessResponse{ID: "proc-1"}, nil
})
exitCode := 0
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
Return(workspacesdk.ProcessOutputResponse{
Running: false,
ExitCode: &exitCode,
}, nil)
tool := chattool.Execute(chattool.ExecuteOptions{
GetWorkspaceConn: func(_ context.Context) (workspacesdk.AgentConn, error) {
return mockConn, nil
},
AgentBrowserSession: "chat-123",
})
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: `{"command":"echo hello"}`,
})
require.NoError(t, err)
assert.False(t, resp.IsError)
assert.Equal(t, "chat-123", capturedReq.Env["AGENT_BROWSER_SESSION"])
})
t.Run("ForegroundNonZeroExit", func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
mockConn.EXPECT().
StartProcess(gomock.Any(), gomock.Any()).
Return(workspacesdk.StartProcessResponse{ID: "proc-1"}, nil)
exitCode := 42
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
Return(workspacesdk.ProcessOutputResponse{
Running: false,
ExitCode: &exitCode,
Output: "something failed",
}, nil)
tool := newExecuteTool(t, mockConn)
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: `{"command":"exit 42"}`,
})
require.NoError(t, err)
assert.False(t, resp.IsError)
var result chattool.ExecuteResult
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
assert.False(t, result.Success)
assert.Equal(t, 42, result.ExitCode)
assert.Equal(t, "something failed", result.Output)
})
t.Run("BackgroundExecution", func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
mockConn.EXPECT().
StartProcess(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, req workspacesdk.StartProcessRequest) (workspacesdk.StartProcessResponse, error) {
assert.True(t, req.Background)
return workspacesdk.StartProcessResponse{ID: "bg-42"}, nil
})
tool := newExecuteTool(t, mockConn)
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: `{"command":"sleep 999","run_in_background":true}`,
})
require.NoError(t, err)
assert.False(t, resp.IsError)
var result chattool.ExecuteResult
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
assert.True(t, result.Success)
assert.Equal(t, "bg-42", result.BackgroundProcessID)
})
t.Run("Timeout", func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
mockConn.EXPECT().
StartProcess(gomock.Any(), gomock.Any()).
Return(workspacesdk.StartProcessResponse{ID: "proc-1"}, nil)
// First call (blocking wait) returns context error
// because the 50ms timeout expires. An old agent ignores
// timeout_ms.
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
DoAndReturn(func(ctx context.Context, _ string, _ *workspacesdk.ProcessOutputOptions) (workspacesdk.ProcessOutputResponse, error) {
<-ctx.Done()
return workspacesdk.ProcessOutputResponse{}, ctx.Err()
})
// Second call (snapshot fallback) returns partial output.
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
Return(workspacesdk.ProcessOutputResponse{
Running: true,
Output: "partial output",
}, nil)
tool := newExecuteTool(t, mockConn)
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
// 50ms timeout expires during the blocking wait.
Input: `{"command":"sleep 999","timeout":"50ms"}`,
})
require.NoError(t, err)
assert.False(t, resp.IsError)
var result chattool.ExecuteResult
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
assert.False(t, result.Success)
assert.Equal(t, -1, result.ExitCode)
assert.Contains(t, result.Error, "timed out")
assert.Equal(t, "partial output", result.Output)
})
t.Run("StopsAtAgentExecuteTimeout", func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
// On a retry, the agent's execute timeout, counted from the first
// start, passes before chatd's own.
var executeTimeout time.Duration
mockConn.EXPECT().
StartProcess(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, req workspacesdk.StartProcessRequest) (workspacesdk.StartProcessResponse, error) {
executeTimeout = time.Duration(req.TimeoutMs) * time.Millisecond
return workspacesdk.StartProcessResponse{ID: "proc-1"}, nil
})
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
DoAndReturn(func(ctx context.Context, _ string, opts *workspacesdk.ProcessOutputOptions) (workspacesdk.ProcessOutputResponse, error) {
if executeTimeout == 10*time.Minute && opts != nil && opts.TimeoutFromStartProcess {
return workspacesdk.ProcessOutputResponse{Running: true, TimedOut: true, Output: "partial output"}, nil
}
<-ctx.Done()
return workspacesdk.ProcessOutputResponse{}, ctx.Err()
})
tool := newExecuteTool(t, mockConn)
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: `{"command":"make test","timeout":"10m"}`,
})
require.NoError(t, err)
var result chattool.ExecuteResult
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
assert.Equal(t, "command timed out after 10m0s", result.Error)
assert.Equal(t, "partial output", result.Output)
assert.Equal(t, "proc-1", result.BackgroundProcessID)
})
t.Run("StartProcessError", func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
mockConn.EXPECT().
StartProcess(gomock.Any(), gomock.Any()).
Return(workspacesdk.StartProcessResponse{}, xerrors.New("connection lost"))
tool := newExecuteTool(t, mockConn)
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: `{"command":"echo hi"}`,
})
require.NoError(t, err)
// Errors from StartProcess are returned as a JSON body
// with success=false, not as a ToolResponse error.
assert.False(t, resp.IsError)
var result chattool.ExecuteResult
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
assert.False(t, result.Success)
assert.Contains(t, result.Error, "connection lost")
// Unrelated errors must not trigger the missing-shell
// guidance.
assert.NotContains(t, result.Error, "Git Bash")
assert.NotContains(t, result.Error, "coder.com/docs")
})
t.Run("MissingShellError", func(t *testing.T) {
t.Parallel()
// OS rendering differs (%PATH% vs $PATH); the fragment
// omits the suffix to match both.
tests := []struct {
name string
input string
agentErr string
wantPrefix string
}{
{
name: "ForegroundWindows",
input: `{"command":"echo hi"}`,
agentErr: "unexpected status code 500: Failed to start process.\n\tError: start process: exec: \"sh\": executable file not found in %PATH%",
wantPrefix: "start process:",
},
{
name: "BackgroundWindows",
input: `{"command":"echo hi","run_in_background":true}`,
agentErr: "unexpected status code 500: Failed to start process.\n\tError: start process: exec: \"sh\": executable file not found in %PATH%",
wantPrefix: "start background process:",
},
{
name: "ForegroundPOSIX",
input: `{"command":"echo hi"}`,
agentErr: "unexpected status code 500: Failed to start process.\n\tError: start process: exec: \"sh\": executable file not found in $PATH",
wantPrefix: "start process:",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
mockConn.EXPECT().
StartProcess(gomock.Any(), gomock.Any()).
Return(workspacesdk.StartProcessResponse{}, xerrors.New(tt.agentErr))
tool := newExecuteTool(t, mockConn)
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: tt.input,
})
require.NoError(t, err)
assert.False(t, resp.IsError)
var result chattool.ExecuteResult
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
assert.False(t, result.Success)
// The result keeps the original error for debugging.
assert.Contains(t, result.Error, tt.wantPrefix)
assert.Contains(t, result.Error, `exec: "sh": executable file not found`)
assert.Contains(t, result.Error, "Git Bash")
assert.Contains(t, result.Error, "https://coder.com/docs/ai-coder/agents/architecture#windows-workspace-shell-requirement")
})
}
})
t.Run("BackgroundedFlagOnlyOnIntentionalLaunch", func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
mockConn.EXPECT().
StartProcess(gomock.Any(), gomock.Any()).
Return(workspacesdk.StartProcessResponse{ID: "proc-bg"}, nil)
tool := newExecuteTool(t, mockConn)
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: `{"command":"npm start","run_in_background":true}`,
})
require.NoError(t, err)
assert.False(t, resp.IsError)
var result chattool.ExecuteResult
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
assert.True(t, result.Backgrounded)
assert.Equal(t, "proc-bg", result.BackgroundProcessID)
})
t.Run("ProcessOutputStillRunningSetsRunningFlag", func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
Return(workspacesdk.ProcessOutputResponse{
Running: true,
Output: "starting...",
Command: "npm start",
}, nil)
tool := chattool.ProcessOutput(chattool.ProcessToolOptions{
GetWorkspaceConn: func(_ context.Context) (workspacesdk.AgentConn, error) {
return mockConn, nil
},
})
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "process_output",
Input: `{"process_id":"proc-1","wait_timeout":"0s"}`,
})
require.NoError(t, err)
assert.False(t, resp.IsError)
var result chattool.ExecuteResult
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
assert.True(t, result.Success)
assert.True(t, result.Running)
assert.Equal(t, "process is still running", result.Note)
assert.Equal(t, "npm start", result.Command)
})
t.Run("ProcessOutputCommandPropagated", func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
exitCode := 1
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
Return(workspacesdk.ProcessOutputResponse{
Running: false,
ExitCode: &exitCode,
Output: "server exited: EADDRINUSE",
Command: "npm start",
}, nil)
tool := chattool.ProcessOutput(chattool.ProcessToolOptions{
GetWorkspaceConn: func(_ context.Context) (workspacesdk.AgentConn, error) {
return mockConn, nil
},
})
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "process_output",
Input: `{"process_id":"proc-1","wait_timeout":"0s"}`,
})
require.NoError(t, err)
assert.False(t, resp.IsError)
var result chattool.ExecuteResult
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
assert.False(t, result.Success)
assert.Equal(t, 1, result.ExitCode)
assert.Equal(t, "npm start", result.Command)
})
t.Run("ProcessOutputError", func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
mockConn.EXPECT().
StartProcess(gomock.Any(), gomock.Any()).
Return(workspacesdk.StartProcessResponse{ID: "proc-1"}, nil)
// First call: blocking wait fails.
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
Return(workspacesdk.ProcessOutputResponse{}, xerrors.New("agent disconnected"))
// Second call: snapshot fallback also fails.
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
Return(workspacesdk.ProcessOutputResponse{}, xerrors.New("agent disconnected"))
tool := newExecuteTool(t, mockConn)
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: `{"command":"echo hi"}`,
})
require.NoError(t, err)
assert.False(t, resp.IsError)
var result chattool.ExecuteResult
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
assert.False(t, result.Success)
assert.Contains(t, result.Error, "agent disconnected")
// Snapshot fallback should provide the process ID
// so the agent can retry manually.
assert.Equal(t, "proc-1", result.BackgroundProcessID)
})
t.Run("TransportErrorRecoveryProcessDone", func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
exitCode := 0
mockConn.EXPECT().
StartProcess(gomock.Any(), gomock.Any()).
Return(workspacesdk.StartProcessResponse{ID: "proc-1"}, nil)
// Blocking wait fails with transport error.
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
Return(workspacesdk.ProcessOutputResponse{}, xerrors.New("EOF"))
// Snapshot fallback finds the process completed.
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
Return(workspacesdk.ProcessOutputResponse{
Output: "hello\n",
Running: false,
ExitCode: &exitCode,
}, nil)
tool := newExecuteTool(t, mockConn)
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: `{"command":"echo hello"}`,
})
require.NoError(t, err)
assert.False(t, resp.IsError)
var result chattool.ExecuteResult
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
// Transparent recovery: success with real output.
assert.True(t, result.Success)
assert.Equal(t, 0, result.ExitCode)
assert.Equal(t, "hello\n", result.Output)
assert.Empty(t, result.BackgroundProcessID)
})
t.Run("TransportErrorProcessStillRunning", func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
mockConn.EXPECT().
StartProcess(gomock.Any(), gomock.Any()).
Return(workspacesdk.StartProcessResponse{ID: "proc-1"}, nil)
// Blocking wait fails with transport error.
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
Return(workspacesdk.ProcessOutputResponse{}, xerrors.New("EOF"))
// Snapshot fallback: process still running.
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
Return(workspacesdk.ProcessOutputResponse{
Output: "partial output",
Running: true,
}, nil)
tool := newExecuteTool(t, mockConn)
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: `{"command":"sleep 60"}`,
})
require.NoError(t, err)
assert.False(t, resp.IsError)
var result chattool.ExecuteResult
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
assert.False(t, result.Success)
assert.Contains(t, result.Error, "process still running")
assert.Contains(t, result.Error, "process_output")
assert.Equal(t, "partial output", result.Output)
assert.Equal(t, "proc-1", result.BackgroundProcessID)
})
t.Run("GetWorkspaceConnNil", func(t *testing.T) {
t.Parallel()
tool := chattool.Execute(chattool.ExecuteOptions{
GetWorkspaceConn: nil,
})
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: `{"command":"echo hi"}`,
})
require.NoError(t, err)
assert.True(t, resp.IsError)
assert.Contains(t, resp.Content, "not configured")
})
t.Run("GetWorkspaceConnError", func(t *testing.T) {
t.Parallel()
tool := chattool.Execute(chattool.ExecuteOptions{
GetWorkspaceConn: func(_ context.Context) (workspacesdk.AgentConn, error) {
return nil, xerrors.New("workspace offline")
},
})
ctx := testutil.Context(t, testutil.WaitMedium)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: `{"command":"echo hi"}`,
})
require.NoError(t, err)
assert.True(t, resp.IsError)
assert.Contains(t, resp.Content, "workspace offline")
})
}
func TestDetectFileDump(t *testing.T) {
t.Parallel()
tests := []struct {
name string
command string
wantHit bool
}{
{
name: "CatFile",
command: "cat foo.txt",
wantHit: true,
},
{
name: "NotCatPrefix",
command: "concatenate foo",
wantHit: false,
},
{
name: "GrepIncludeAll",
command: "grep --include-all pattern",
wantHit: true,
},
{
name: "RgListFiles",
command: "rg -l pattern",
wantHit: true,
},
{
name: "GrepRecursive",
command: "grep -r pattern",
wantHit: false,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
mockConn.EXPECT().
StartProcess(gomock.Any(), gomock.Any()).
Return(workspacesdk.StartProcessResponse{ID: "proc-1"}, nil)
exitCode := 0
mockConn.EXPECT().
ProcessOutput(gomock.Any(), "proc-1", gomock.Any()).
Return(workspacesdk.ProcessOutputResponse{
Running: false,
ExitCode: &exitCode,
Output: "output",
}, nil)
tool := newExecuteTool(t, mockConn)
ctx := testutil.Context(t, testutil.WaitMedium)
input, err := json.Marshal(map[string]any{
"command": tc.command,
})
require.NoError(t, err)
resp, err := tool.Run(ctx, fantasy.ToolCall{
ID: "call-1",
Name: "execute",
Input: string(input),
})
require.NoError(t, err)
var result chattool.ExecuteResult
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
if tc.wantHit {
assert.Contains(t, result.Note, "read_file",
"expected advisory note for %q", tc.command)
} else {
assert.Empty(t, result.Note,
"expected no note for %q", tc.command)
}
})
}
}
// newExecuteTool creates an Execute tool wired to the given mock.
func newExecuteTool(t *testing.T, mockConn *agentconnmock.MockAgentConn) fantasy.AgentTool {
t.Helper()
return chattool.Execute(chattool.ExecuteOptions{
GetWorkspaceConn: func(_ context.Context) (workspacesdk.AgentConn, error) {
return mockConn, nil
},
})
}
func ptr[T any](v T) *T {
return &v
}