Files
coder/codersdk/toolsdk/bash_test.go
T
Hugo Dutka 19b72a7bed fix(codersdk/toolsdk): remove background test startup timing assumptions (#29043)
The background bash tests assumed their initial output would arrive
within a two-second deadline. Separate completed-command output capture
from timeout behavior, and gate background completion until after the
handler returns. Each run now uses its own temporary FIFO and completion
marker instead of a fixed sleep and shared file.

Fixes https://linear.app/codercom/issue/CODAGT-1004

<details>
<summary>Investigation and validation evidence</summary>

Injecting a three-second delay into external command execution
reproduced both original failures with exit code 124 and no initial
output. This demonstrates the unsafe startup-timing assumption; it does
not establish the precise source of the CI runner's delay or implicate
the output-drain window.

The updated tests passed three repetitions with the same injected delay,
five race-enabled repetitions, the full `toolsdk` package, and package
lint checks (zero issues).

The continuation test releases its FIFO only after the timeout result
and verifies the unique completion marker afterward. Output capture is
covered separately for stdout and stderr.

</details>

Generated by Coder Agents on behalf of @hugodutka.
2026-09-08 16:15:05 +02:00

445 lines
15 KiB
Go

package toolsdk_test
import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"testing"
"github.com/kballard/go-shellquote"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/coder/coder/v2/agent/agenttest"
"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/codersdk/toolsdk"
"github.com/coder/coder/v2/testutil"
)
func TestWorkspaceBash(t *testing.T) {
t.Parallel()
if runtime.GOOS == "windows" {
t.Skip("Skipping on Windows: Workspace MCP bash tools rely on a Unix-like shell (bash) and POSIX/SSH semantics. Use Linux/macOS or WSL for these tests.")
}
t.Run("ValidateArgs", func(t *testing.T) {
t.Parallel()
deps := toolsdk.Deps{}
ctx := context.Background()
// Test empty workspace name
args := toolsdk.WorkspaceBashArgs{
Workspace: "",
Command: "echo test",
}
_, err := toolsdk.WorkspaceBash.Handler(ctx, deps, args)
require.Error(t, err)
require.Contains(t, err.Error(), "workspace name cannot be empty")
// Test empty command
args = toolsdk.WorkspaceBashArgs{
Workspace: "test-workspace",
Command: "",
}
_, err = toolsdk.WorkspaceBash.Handler(ctx, deps, args)
require.Error(t, err)
require.Contains(t, err.Error(), "command cannot be empty")
})
t.Run("ErrorScenarios", func(t *testing.T) {
t.Parallel()
deps := toolsdk.Deps{}
ctx := context.Background()
// Test input validation errors (these should fail before client access)
t.Run("EmptyWorkspace", func(t *testing.T) {
args := toolsdk.WorkspaceBashArgs{
Workspace: "", // Empty workspace should be caught by validation
Command: "echo test",
}
_, err := toolsdk.WorkspaceBash.Handler(ctx, deps, args)
require.Error(t, err)
require.Contains(t, err.Error(), "workspace name cannot be empty")
})
t.Run("EmptyCommand", func(t *testing.T) {
args := toolsdk.WorkspaceBashArgs{
Workspace: "test-workspace",
Command: "", // Empty command should be caught by validation
}
_, err := toolsdk.WorkspaceBash.Handler(ctx, deps, args)
require.Error(t, err)
require.Contains(t, err.Error(), "command cannot be empty")
})
})
t.Run("ToolMetadata", func(t *testing.T) {
t.Parallel()
tool := toolsdk.WorkspaceBash
require.Equal(t, toolsdk.ToolNameWorkspaceBash, tool.Name)
require.NotEmpty(t, tool.Description)
require.Contains(t, tool.Description, "Execute a bash command in a Coder workspace")
require.Contains(t, tool.Description, "output is trimmed of leading and trailing whitespace")
require.Contains(t, tool.Schema.Required, "workspace")
require.Contains(t, tool.Schema.Required, "command")
// Check that schema has the required properties
require.Contains(t, tool.Schema.Properties, "workspace")
require.Contains(t, tool.Schema.Properties, "command")
})
t.Run("GenericTool", func(t *testing.T) {
t.Parallel()
genericTool := toolsdk.WorkspaceBash.Generic()
require.Equal(t, toolsdk.ToolNameWorkspaceBash, genericTool.Name)
require.NotEmpty(t, genericTool.Description)
require.NotNil(t, genericTool.Handler)
require.False(t, genericTool.UserClientOptional)
})
}
func TestAllToolsIncludesBash(t *testing.T) {
t.Parallel()
if runtime.GOOS == "windows" {
t.Skip("Skipping on Windows: Workspace MCP bash tools rely on a Unix-like shell (bash) and POSIX/SSH semantics. Use Linux/macOS or WSL for these tests.")
}
// Verify that WorkspaceBash is included in the All slice
found := false
for _, tool := range toolsdk.All {
if tool.Name == toolsdk.ToolNameWorkspaceBash {
found = true
break
}
}
require.True(t, found, "WorkspaceBash tool should be included in toolsdk.All")
}
// Note: Unit testing ExecuteCommandWithTimeout is challenging because it expects
// a concrete SSH session type. The integration tests above demonstrate the
// timeout functionality with a real SSH connection and mock clock.
func TestWorkspaceBashTimeout(t *testing.T) {
t.Parallel()
if runtime.GOOS == "windows" {
t.Skip("Skipping on Windows: Workspace MCP bash tools rely on a Unix-like shell (bash) and POSIX/SSH semantics. Use Linux/macOS or WSL for these tests.")
}
t.Run("TimeoutDefaultValue", func(t *testing.T) {
t.Parallel()
// Test that the TimeoutMs field can be set and read correctly
args := toolsdk.WorkspaceBashArgs{
TimeoutMs: 0, // Should default to 60000 in handler
}
// Verify that the TimeoutMs field exists and can be set
require.Equal(t, 0, args.TimeoutMs)
// Test setting a positive value
args.TimeoutMs = 5000
require.Equal(t, 5000, args.TimeoutMs)
})
t.Run("TimeoutNegativeValue", func(t *testing.T) {
t.Parallel()
// Test that negative values can be set and will be handled by the default logic
args := toolsdk.WorkspaceBashArgs{
TimeoutMs: -100,
}
require.Equal(t, -100, args.TimeoutMs)
// The actual defaulting to 60000 happens inside the handler
// We can't test it without a full integration test setup
})
t.Run("TimeoutSchemaValidation", func(t *testing.T) {
t.Parallel()
tool := toolsdk.WorkspaceBash
// Check that timeout_ms is in the schema
require.Contains(t, tool.Schema.Properties, "timeout_ms")
timeoutProperty := tool.Schema.Properties["timeout_ms"].(map[string]any)
require.Equal(t, "integer", timeoutProperty["type"])
require.Equal(t, 60000, timeoutProperty["default"])
require.Equal(t, 1, timeoutProperty["minimum"])
require.Contains(t, timeoutProperty["description"], "timeout in milliseconds")
})
t.Run("TimeoutDescriptionUpdated", func(t *testing.T) {
t.Parallel()
tool := toolsdk.WorkspaceBash
// Check that description mentions timeout functionality
require.Contains(t, tool.Description, "timeout_ms parameter")
require.Contains(t, tool.Description, "defaults to 60000ms")
require.Contains(t, tool.Description, "timeout_ms: 30000")
})
t.Run("TimeoutCommandScenario", func(t *testing.T) {
t.Parallel()
// Scenario: echo "123"; sleep 60; echo "456" with 5ms timeout
// In this scenario, we'd expect to see "123" in the output and a cancellation message
args := toolsdk.WorkspaceBashArgs{
Workspace: "test-workspace",
Command: `echo "123"; sleep 60; echo "456"`, // This command would take 60+ seconds
TimeoutMs: 5, // 5ms timeout - should timeout after first echo
}
// Verify the args are structured correctly for the intended test scenario
require.Equal(t, "test-workspace", args.Workspace)
require.Contains(t, args.Command, `echo "123"`)
require.Contains(t, args.Command, "sleep 60")
require.Contains(t, args.Command, `echo "456"`)
require.Equal(t, 5, args.TimeoutMs)
// Note: The actual timeout behavior would need to be tested with a real workspace
// This test just verifies the structure is correct for the timeout scenario
})
}
func TestWorkspaceBashTimeoutIntegration(t *testing.T) {
t.Parallel()
if runtime.GOOS == "windows" {
t.Skip("Skipping on Windows: Workspace MCP bash tools rely on a Unix-like shell (bash) and POSIX/SSH semantics. Use Linux/macOS or WSL for these tests.")
}
t.Run("ActualTimeoutBehavior", func(t *testing.T) {
t.Parallel()
// Scenario: echo "123"; sleep 60; echo "456" with 5s timeout
// In this scenario, we'd expect to see "123" in the output and a cancellation message
client, workspace, agentToken := setupWorkspaceForAgent(t, nil)
// Start the agent and wait for it to be fully ready
_ = agenttest.New(t, client.URL, agentToken)
// Wait for workspace agents to be ready like other SSH tests do
coderdtest.NewWorkspaceAgentWaiter(t, client, workspace.ID).Wait()
// Use real clock for integration test
deps, err := toolsdk.NewDeps(client)
require.NoError(t, err)
args := toolsdk.WorkspaceBashArgs{
Workspace: workspace.Name,
Command: `echo "123" && sleep 60 && echo "456"`, // This command would take 60+ seconds
TimeoutMs: 2000, // 2 seconds timeout - should timeout after first echo
}
result, err := testTool(t, toolsdk.WorkspaceBash, deps, args)
// Should not error (timeout is handled gracefully)
require.NoError(t, err)
t.Logf("Test results: exitCode=%d, output=%q, error=%v", result.ExitCode, result.Output, err)
// Should have a non-zero exit code (timeout or error)
require.NotEqual(t, 0, result.ExitCode, "Expected non-zero exit code for timeout")
t.Logf("result.Output: %s", result.Output)
// Should contain the first echo output
require.Contains(t, result.Output, "123")
// Should NOT contain the second echo (it never executed due to timeout)
require.NotContains(t, result.Output, "456", "Should not contain output after sleep")
})
t.Run("NormalCommandExecution", func(t *testing.T) {
t.Parallel()
// Test that normal commands still work with timeout functionality present
client, workspace, agentToken := setupWorkspaceForAgent(t, nil)
// Start the agent and wait for it to be fully ready
_ = agenttest.New(t, client.URL, agentToken)
// Wait for workspace agents to be ready
coderdtest.NewWorkspaceAgentWaiter(t, client, workspace.ID).Wait()
deps, err := toolsdk.NewDeps(client)
require.NoError(t, err)
args := toolsdk.WorkspaceBashArgs{
Workspace: workspace.Name,
Command: `echo "normal command"`, // Quick command that should complete normally
TimeoutMs: 5000, // 5 second timeout - plenty of time
}
// Use testTool to register the tool as tested and satisfy coverage validation
result, err := testTool(t, toolsdk.WorkspaceBash, deps, args)
// Should not error
require.NoError(t, err)
t.Logf("result.Output: %s", result.Output)
// Should have exit code 0 (success)
require.Equal(t, 0, result.ExitCode)
// Should contain the expected output
require.Equal(t, "normal command", result.Output)
// Should NOT contain timeout message
require.NotContains(t, result.Output, "Command canceled due to timeout")
})
}
func TestWorkspaceBashBackgroundIntegration(t *testing.T) {
t.Parallel()
if runtime.GOOS == "windows" {
t.Skip("Skipping on Windows: Workspace MCP bash tools rely on a Unix-like shell (bash) and POSIX/SSH semantics. Use Linux/macOS or WSL for these tests.")
}
t.Run("BackgroundCommandCapturesOutput", func(t *testing.T) {
t.Parallel()
client, workspace, agentToken := setupWorkspaceForAgent(t, nil)
// Start the agent and wait for it to be fully ready
_ = agenttest.New(t, client.URL, agentToken)
// Wait for workspace agents to be ready
coderdtest.NewWorkspaceAgentWaiter(t, client, workspace.ID).Wait()
deps, err := toolsdk.NewDeps(client)
require.NoError(t, err)
// Output capture must not depend on starting within a short deadline.
args := toolsdk.WorkspaceBashArgs{
Workspace: workspace.Name,
Command: `sh -c 'echo stdout; echo stderr >&2'`,
Background: true,
}
result, err := testTool(t, toolsdk.WorkspaceBash, deps, args)
require.NoError(t, err)
require.Equal(t, 0, result.ExitCode)
require.Contains(t, result.Output, "stdout")
require.Contains(t, result.Output, "stderr")
})
t.Run("BackgroundVsNormalExecution", func(t *testing.T) {
t.Parallel()
client, workspace, agentToken := setupWorkspaceForAgent(t, nil)
// Start the agent and wait for it to be fully ready
_ = agenttest.New(t, client.URL, agentToken)
// Wait for workspace agents to be ready
coderdtest.NewWorkspaceAgentWaiter(t, client, workspace.ID).Wait()
deps, err := toolsdk.NewDeps(client)
require.NoError(t, err)
// First run the same command in normal mode
normalArgs := toolsdk.WorkspaceBashArgs{
Workspace: workspace.Name,
Command: `echo "hello world"`,
Background: false,
}
normalResult, err := toolsdk.WorkspaceBash.Handler(t.Context(), deps, normalArgs)
require.NoError(t, err)
// Normal mode should return the actual output
require.Equal(t, 0, normalResult.ExitCode)
require.Equal(t, "hello world", normalResult.Output)
// Now run the same command in background mode
backgroundArgs := toolsdk.WorkspaceBashArgs{
Workspace: workspace.Name,
Command: `echo "hello world"`,
Background: true,
}
backgroundResult, err := testTool(t, toolsdk.WorkspaceBash, deps, backgroundArgs)
require.NoError(t, err)
t.Logf("Normal result: %q", normalResult.Output)
t.Logf("Background result: %q", backgroundResult.Output)
// Background mode should also return the actual output since command completes quickly
require.Equal(t, 0, backgroundResult.ExitCode)
require.Equal(t, "hello world", backgroundResult.Output)
})
t.Run("BackgroundCommandContinuesAfterTimeout", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
dir := t.TempDir()
gatePath := filepath.Join(dir, "continue")
donePath := filepath.Join(dir, "done")
output, err := exec.CommandContext(ctx, "mkfifo", gatePath).CombinedOutput()
require.NoError(t, err, "%s", output)
// Opening both ends lets the test release the command even if shell
// startup takes longer than the timeout. Close after agent cleanup so
// an in-flight command can still read the release token.
gate, err := os.OpenFile(gatePath, os.O_RDWR, 0)
require.NoError(t, err)
t.Cleanup(func() { _ = gate.Close() })
client, workspace, agentToken := setupWorkspaceForAgent(t, nil)
// Start the agent and wait for it to be fully ready
_ = agenttest.New(t, client.URL, agentToken)
// Wait for workspace agents to be ready
coderdtest.NewWorkspaceAgentWaiter(t, client, workspace.ID).Wait()
deps, err := toolsdk.NewDeps(client)
require.NoError(t, err)
t.Cleanup(func() {
// Release the command before agent cleanup, including on failure.
_, _ = gate.WriteString("continue\n")
})
script := fmt.Sprintf("read -r proceed < %s && echo done > %s",
shellquote.Join(gatePath), shellquote.Join(donePath))
args := toolsdk.WorkspaceBashArgs{
Workspace: workspace.Name,
Command: shellquote.Join("sh", "-c", script),
TimeoutMs: 2000,
Background: true,
}
result, err := testTool(t, toolsdk.WorkspaceBash, deps, args)
require.NoError(t, err)
require.Equal(t, 124, result.ExitCode)
// No output is required before timeout, including during slow startup.
require.Equal(t, "\nCommand continues running in background", result.Output)
_, err = os.Stat(donePath)
require.ErrorIs(t, err, os.ErrNotExist)
// Only allow completion after the timed-out handler has returned.
_, err = gate.WriteString("continue\n")
require.NoError(t, err)
require.EventuallyWithT(t, func(c *assert.CollectT) {
output, err := os.ReadFile(donePath)
require.NoError(c, err)
require.Equal(c, "done\n", string(output))
}, testutil.WaitLong, testutil.IntervalMedium,
"Background command should continue running and complete after timeout")
})
}