fix: clarify task authorization for workspace setup
This commit is contained in:
1 parent
847c350143
commit
f4f61333ce
6 files changed
+122
-14
No files matched your search
@@ -13047,6 +13047,8 @@ func TestChatSystemPrompt(t *testing.T) {
|
|||||||
const workspaceAwareness = `This chat started without an attached workspace. Follow subsequent workspace tool results and context for its current state.
|
const workspaceAwareness = `This chat started without an attached workspace. Follow subsequent workspace tool results and context for its current state.
|
||||||
Use the conversation and available tools, skills, and MCPs when they are sufficient for the request.
|
Use the conversation and available tools, skills, and MCPs when they are sufficient for the request.
|
||||||
Workspace tools such as execute, read_file, write_file, and edit_files require an attached workspace. If no workspace is attached, create a suitable workspace with create_workspace when missing tools, skills, MCPs, or context prevent progress, or workspace-backed work is needed. Use the workspace's available context and capabilities to continue the user's request. Workspace readiness does not guarantee that skills, MCP tools, or context have finished loading. Use capabilities that are actually exposed, and continue with workspace file and shell tools where possible instead of recreating the workspace.
|
Workspace tools such as execute, read_file, write_file, and edit_files require an attached workspace. If no workspace is attached, create a suitable workspace with create_workspace when missing tools, skills, MCPs, or context prevent progress, or workspace-backed work is needed. Use the workspace's available context and capabilities to continue the user's request. Workspace readiness does not guarantee that skills, MCP tools, or context have finished loading. Use capabilities that are actually exposed, and continue with workspace file and shell tools where possible instead of recreating the workspace.
|
||||||
|
Requests such as "fix this bug" or "build this app" authorize the workspace setup needed to complete them; the user does not need to request a workspace separately. Do not refuse solely because no workspace is attached. If setup is blocked, explain the specific blocker or required user choice.
|
||||||
|
Answer questions and self-contained code examples directly when the conversation and available tools are sufficient.
|
||||||
If a workspace is needed, use list_templates before create_workspace and follow its next_step. Call read_template only when you need template parameter or preset details.`
|
If a workspace is needed, use list_templates before create_workspace and follow its next_step. Call read_template only when you need template parameter or preset details.`
|
||||||
|
|
||||||
updateChatSystemPrompt := func(t *testing.T, ctx context.Context, req codersdk.UpdateChatSystemPromptRequest) {
|
updateChatSystemPrompt := func(t *testing.T, ctx context.Context, req codersdk.UpdateChatSystemPromptRequest) {
|
||||||
|
|||||||
+105
-11
@@ -31,11 +31,14 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/mock/gomock"
|
"go.uber.org/mock/gomock"
|
||||||
"golang.org/x/xerrors"
|
"golang.org/x/xerrors"
|
||||||
|
"google.golang.org/protobuf/encoding/protojson"
|
||||||
|
"google.golang.org/protobuf/types/known/structpb"
|
||||||
|
|
||||||
"cdr.dev/slog/v3"
|
"cdr.dev/slog/v3"
|
||||||
"cdr.dev/slog/v3/sloggers/slogtest"
|
"cdr.dev/slog/v3/sloggers/slogtest"
|
||||||
"github.com/coder/coder/v2/agent/agentcontextconfig"
|
"github.com/coder/coder/v2/agent/agentcontextconfig"
|
||||||
"github.com/coder/coder/v2/agent/agenttest"
|
"github.com/coder/coder/v2/agent/agenttest"
|
||||||
|
agentproto "github.com/coder/coder/v2/agent/proto"
|
||||||
"github.com/coder/coder/v2/coderd/aibridge"
|
"github.com/coder/coder/v2/coderd/aibridge"
|
||||||
"github.com/coder/coder/v2/coderd/aibridgedtest"
|
"github.com/coder/coder/v2/coderd/aibridgedtest"
|
||||||
"github.com/coder/coder/v2/coderd/coderdtest"
|
"github.com/coder/coder/v2/coderd/coderdtest"
|
||||||
@@ -45,6 +48,7 @@ import (
|
|||||||
"github.com/coder/coder/v2/coderd/database/dbfake"
|
"github.com/coder/coder/v2/coderd/database/dbfake"
|
||||||
"github.com/coder/coder/v2/coderd/database/dbgen"
|
"github.com/coder/coder/v2/coderd/database/dbgen"
|
||||||
"github.com/coder/coder/v2/coderd/database/dbtestutil"
|
"github.com/coder/coder/v2/coderd/database/dbtestutil"
|
||||||
|
"github.com/coder/coder/v2/coderd/database/dbtime"
|
||||||
dbpubsub "github.com/coder/coder/v2/coderd/database/pubsub"
|
dbpubsub "github.com/coder/coder/v2/coderd/database/pubsub"
|
||||||
"github.com/coder/coder/v2/coderd/util/slice"
|
"github.com/coder/coder/v2/coderd/util/slice"
|
||||||
"github.com/coder/coder/v2/coderd/workspacestats"
|
"github.com/coder/coder/v2/coderd/workspacestats"
|
||||||
@@ -4530,9 +4534,19 @@ func filterMessageEvents(events []codersdk.ChatStreamEvent) []codersdk.ChatStrea
|
|||||||
func TestCreateWorkspaceTool_EndToEnd(t *testing.T) {
|
func TestCreateWorkspaceTool_EndToEnd(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
instruction = "Run the workspace checks before reporting completion."
|
||||||
|
skillName = "workspace-checks"
|
||||||
|
skillDescription = "Validate the newly created workspace."
|
||||||
|
mcpToolName = "workspace__check"
|
||||||
|
)
|
||||||
|
|
||||||
ctx := testutil.Context(t, testutil.WaitLong)
|
ctx := testutil.Context(t, testutil.WaitLong)
|
||||||
|
deploymentValues := coderdtest.DeploymentValues(t)
|
||||||
|
// Keep the seeded published snapshot independent of agent discovery.
|
||||||
|
deploymentValues.DisableWorkspaceAgentContextSync = true
|
||||||
client, _, api := coderdtest.NewWithAPI(t, &coderdtest.Options{
|
client, _, api := coderdtest.NewWithAPI(t, &coderdtest.Options{
|
||||||
DeploymentValues: coderdtest.DeploymentValues(t),
|
DeploymentValues: deploymentValues,
|
||||||
IncludeProvisionerDaemon: true,
|
IncludeProvisionerDaemon: true,
|
||||||
})
|
})
|
||||||
aibridgedtest.StartTestAIBridgeDaemon(t.Context(), t, api, nil)
|
aibridgedtest.StartTestAIBridgeDaemon(t.Context(), t, api, nil)
|
||||||
@@ -4558,10 +4572,6 @@ func TestCreateWorkspaceTool_EndToEnd(t *testing.T) {
|
|||||||
coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
|
coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
|
||||||
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
|
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
|
||||||
|
|
||||||
// Start the test workspace agent so create_workspace can wait for
|
|
||||||
// the agent to become reachable before returning.
|
|
||||||
_ = agenttest.New(t, client.URL, agentToken)
|
|
||||||
|
|
||||||
workspaceName := "chat-ws-" + strings.ReplaceAll(uuid.NewString(), "-", "")[:8]
|
workspaceName := "chat-ws-" + strings.ReplaceAll(uuid.NewString(), "-", "")[:8]
|
||||||
createWorkspaceArgs := fmt.Sprintf(
|
createWorkspaceArgs := fmt.Sprintf(
|
||||||
`{"template_id":%q,"name":%q}`,
|
`{"template_id":%q,"name":%q}`,
|
||||||
@@ -4571,7 +4581,7 @@ func TestCreateWorkspaceTool_EndToEnd(t *testing.T) {
|
|||||||
|
|
||||||
var streamedCallCount atomic.Int32
|
var streamedCallCount atomic.Int32
|
||||||
var streamedCallsMu sync.Mutex
|
var streamedCallsMu sync.Mutex
|
||||||
streamedCalls := make([][]chattest.OpenAIMessage, 0, 2)
|
streamedCalls := make([]recordedOpenAIRequest, 0, 2)
|
||||||
|
|
||||||
openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
|
openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
|
||||||
if !req.Stream {
|
if !req.Stream {
|
||||||
@@ -4579,7 +4589,7 @@ func TestCreateWorkspaceTool_EndToEnd(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
streamedCallsMu.Lock()
|
streamedCallsMu.Lock()
|
||||||
streamedCalls = append(streamedCalls, append([]chattest.OpenAIMessage(nil), req.Messages...))
|
streamedCalls = append(streamedCalls, recordOpenAIRequest(req))
|
||||||
streamedCallsMu.Unlock()
|
streamedCallsMu.Unlock()
|
||||||
|
|
||||||
if streamedCallCount.Add(1) == 1 {
|
if streamedCallCount.Add(1) == 1 {
|
||||||
@@ -4605,6 +4615,67 @@ func TestCreateWorkspaceTool_EndToEnd(t *testing.T) {
|
|||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
var workspaceAgentID uuid.UUID
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
got, getErr := expClient.GetChat(ctx, chat.ID)
|
||||||
|
if getErr != nil || got.WorkspaceID == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
workspace, getErr := client.Workspace(ctx, *got.WorkspaceID)
|
||||||
|
if getErr != nil || workspace.LatestBuild.Status != codersdk.WorkspaceStatusRunning {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, resource := range workspace.LatestBuild.Resources {
|
||||||
|
for _, workspaceAgent := range resource.Agents {
|
||||||
|
workspaceAgentID = workspaceAgent.ID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return workspaceAgentID != uuid.Nil
|
||||||
|
}, testutil.WaitLong, testutil.IntervalFast)
|
||||||
|
require.EqualValues(t, 1, streamedCallCount.Load())
|
||||||
|
|
||||||
|
// Publish before the agent connects so the same-turn continuation has
|
||||||
|
// a complete snapshot. This does not test startup discovery readiness.
|
||||||
|
instructionBody, err := protojson.Marshal(&agentproto.InstructionFileBody{Content: []byte(instruction)})
|
||||||
|
require.NoError(t, err)
|
||||||
|
skillBody, err := protojson.Marshal(&agentproto.SkillMetaBody{
|
||||||
|
Name: skillName,
|
||||||
|
Description: skillDescription,
|
||||||
|
Meta: []byte("---\nname: " + skillName + "\ndescription: " + skillDescription + "\n---\nRun workspace checks.\n"),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
schema, err := structpb.NewStruct(map[string]any{"type": "object"})
|
||||||
|
require.NoError(t, err)
|
||||||
|
mcpBody, err := protojson.Marshal(&agentproto.MCPServerBody{
|
||||||
|
ServerName: "workspace",
|
||||||
|
Tools: []*agentproto.MCPTool{{Name: "check", Description: "Check the workspace.", InputSchema: schema}},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
systemCtx := dbauthz.AsSystemRestricted(ctx)
|
||||||
|
now := dbtime.Now()
|
||||||
|
for _, resource := range []database.UpsertWorkspaceAgentContextResourceParams{
|
||||||
|
{Source: "/workspace/AGENTS.md", BodyKind: database.WorkspaceAgentContextBodyKindInstructionFile, Body: instructionBody},
|
||||||
|
{Source: "/workspace/skills/" + skillName, BodyKind: database.WorkspaceAgentContextBodyKindSkill, Body: skillBody},
|
||||||
|
{Source: "workspace", BodyKind: database.WorkspaceAgentContextBodyKindMcpServer, Body: mcpBody},
|
||||||
|
} {
|
||||||
|
resource.WorkspaceAgentID = workspaceAgentID
|
||||||
|
resource.ContentHash = []byte(resource.Source)
|
||||||
|
resource.SizeBytes = int64(len(resource.Body))
|
||||||
|
resource.Status = database.WorkspaceAgentContextResourceStatusOk
|
||||||
|
resource.Now = now
|
||||||
|
_, err = api.Database.UpsertWorkspaceAgentContextResource(systemCtx, resource)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
_, err = api.Database.UpsertWorkspaceAgentContextSnapshot(systemCtx, database.UpsertWorkspaceAgentContextSnapshotParams{
|
||||||
|
WorkspaceAgentID: workspaceAgentID,
|
||||||
|
Version: 1,
|
||||||
|
AggregateHash: []byte("created-workspace-context"),
|
||||||
|
ReceivedAt: now,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
_ = agenttest.New(t, client.URL, agentToken)
|
||||||
|
|
||||||
var chatResult codersdk.Chat
|
var chatResult codersdk.Chat
|
||||||
require.Eventually(t, func() bool {
|
require.Eventually(t, func() bool {
|
||||||
got, getErr := expClient.GetChat(ctx, chat.ID)
|
got, getErr := expClient.GetChat(ctx, chat.ID)
|
||||||
@@ -4650,6 +4721,9 @@ func TestCreateWorkspaceTool_EndToEnd(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
require.True(t, foundCreateWorkspaceResult, "expected create_workspace tool result message")
|
require.True(t, foundCreateWorkspaceResult, "expected create_workspace tool result message")
|
||||||
|
require.Len(t, slice.Filter(chatMsgs.Messages, func(message codersdk.ChatMessage) bool {
|
||||||
|
return message.Role == codersdk.ChatMessageRoleUser
|
||||||
|
}), 1, "workspace context should be available without another user turn")
|
||||||
|
|
||||||
// Verify that the tool waited for startup scripts to
|
// Verify that the tool waited for startup scripts to
|
||||||
// complete. The agent should be in "ready" state by the
|
// complete. The agent should be in "ready" state by the
|
||||||
@@ -4665,14 +4739,34 @@ func TestCreateWorkspaceTool_EndToEnd(t *testing.T) {
|
|||||||
require.Equal(t, codersdk.WorkspaceAgentLifecycleReady, agentLifecycle,
|
require.Equal(t, codersdk.WorkspaceAgentLifecycleReady, agentLifecycle,
|
||||||
"agent should be ready after create_workspace returns; startup scripts were not awaited")
|
"agent should be ready after create_workspace returns; startup scripts were not awaited")
|
||||||
|
|
||||||
require.GreaterOrEqual(t, streamedCallCount.Load(), int32(2))
|
|
||||||
streamedCallsMu.Lock()
|
streamedCallsMu.Lock()
|
||||||
recordedStreamCalls := append([][]chattest.OpenAIMessage(nil), streamedCalls...)
|
recordedStreamCalls := append([]recordedOpenAIRequest(nil), streamedCalls...)
|
||||||
streamedCallsMu.Unlock()
|
streamedCallsMu.Unlock()
|
||||||
require.GreaterOrEqual(t, len(recordedStreamCalls), 2)
|
require.Len(t, recordedStreamCalls, 2)
|
||||||
|
|
||||||
|
for i, call := range recordedStreamCalls {
|
||||||
|
var systemContent string
|
||||||
|
for _, message := range call.Messages {
|
||||||
|
if message.Role == "system" {
|
||||||
|
systemContent += message.Content + "\n"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if i == 0 {
|
||||||
|
require.NotContains(t, systemContent, instruction)
|
||||||
|
require.NotContains(t, systemContent, skillName)
|
||||||
|
require.NotContains(t, call.Tools, mcpToolName)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
require.Contains(t, systemContent, "<workspace-context>")
|
||||||
|
require.Contains(t, systemContent, instruction)
|
||||||
|
require.Contains(t, systemContent, "<available-skills>")
|
||||||
|
require.Contains(t, systemContent, skillName)
|
||||||
|
require.Contains(t, systemContent, skillDescription)
|
||||||
|
require.Contains(t, call.Tools, mcpToolName)
|
||||||
|
}
|
||||||
|
|
||||||
var foundToolResultInSecondCall bool
|
var foundToolResultInSecondCall bool
|
||||||
for _, message := range recordedStreamCalls[1] {
|
for _, message := range recordedStreamCalls[1].Messages {
|
||||||
if message.Role != "tool" {
|
if message.Role != "tool" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -216,6 +216,11 @@ func TestWorkspaceAwarenessSupportsWorkspaceFallback(t *testing.T) {
|
|||||||
require.Contains(t, detached, "create a suitable workspace with create_workspace")
|
require.Contains(t, detached, "create a suitable workspace with create_workspace")
|
||||||
require.NotContains(t, detached, "start_workspace")
|
require.NotContains(t, detached, "start_workspace")
|
||||||
require.Contains(t, detached, "Use the workspace's available context and capabilities to continue the user's request")
|
require.Contains(t, detached, "Use the workspace's available context and capabilities to continue the user's request")
|
||||||
|
require.Contains(t, detached, `Requests such as "fix this bug" or "build this app" authorize the workspace setup needed to complete them`)
|
||||||
|
require.Contains(t, detached, "the user does not need to request a workspace separately")
|
||||||
|
require.Contains(t, detached, "Do not refuse solely because no workspace is attached")
|
||||||
|
require.Contains(t, detached, "If setup is blocked, explain the specific blocker or required user choice")
|
||||||
|
require.Contains(t, detached, "Answer questions and self-contained code examples directly when the conversation and available tools are sufficient")
|
||||||
require.Contains(t, detached, "use list_templates before create_workspace")
|
require.Contains(t, detached, "use list_templates before create_workspace")
|
||||||
require.NotContains(t, detached, "Do not create or start a workspace by default")
|
require.NotContains(t, detached, "Do not create or start a workspace by default")
|
||||||
require.NotContains(t, detached, "Only call create_workspace or start_workspace")
|
require.NotContains(t, detached, "Only call create_workspace or start_workspace")
|
||||||
@@ -224,6 +229,7 @@ func TestWorkspaceAwarenessSupportsWorkspaceFallback(t *testing.T) {
|
|||||||
require.Contains(t, delegated, "This delegated chat cannot create or start a workspace")
|
require.Contains(t, delegated, "This delegated chat cannot create or start a workspace")
|
||||||
require.Contains(t, delegated, "report that need to the parent agent")
|
require.Contains(t, delegated, "report that need to the parent agent")
|
||||||
require.NotContains(t, delegated, "create a suitable workspace with create_workspace")
|
require.NotContains(t, delegated, "create a suitable workspace with create_workspace")
|
||||||
|
require.NotContains(t, delegated, "authorize the workspace setup needed to complete them")
|
||||||
|
|
||||||
attached := workspaceAttachedAwareness
|
attached := workspaceAttachedAwareness
|
||||||
require.Contains(t, attached, "This chat is attached to a workspace")
|
require.Contains(t, attached, "This chat is attached to a workspace")
|
||||||
|
|||||||
@@ -22,6 +22,8 @@ Use the conversation and available tools, skills, and MCPs when they are suffici
|
|||||||
Workspace tools such as execute, read_file, write_file, and edit_files require an attached workspace.`
|
Workspace tools such as execute, read_file, write_file, and edit_files require an attached workspace.`
|
||||||
|
|
||||||
const workspaceDetachedAwareness = workspaceDetachedAwarenessBase + ` If no workspace is attached, create a suitable workspace with create_workspace when missing tools, skills, MCPs, or context prevent progress, or workspace-backed work is needed. Use the workspace's available context and capabilities to continue the user's request. Workspace readiness does not guarantee that skills, MCP tools, or context have finished loading. Use capabilities that are actually exposed, and continue with workspace file and shell tools where possible instead of recreating the workspace.
|
const workspaceDetachedAwareness = workspaceDetachedAwarenessBase + ` If no workspace is attached, create a suitable workspace with create_workspace when missing tools, skills, MCPs, or context prevent progress, or workspace-backed work is needed. Use the workspace's available context and capabilities to continue the user's request. Workspace readiness does not guarantee that skills, MCP tools, or context have finished loading. Use capabilities that are actually exposed, and continue with workspace file and shell tools where possible instead of recreating the workspace.
|
||||||
|
Requests such as "fix this bug" or "build this app" authorize the workspace setup needed to complete them; the user does not need to request a workspace separately. Do not refuse solely because no workspace is attached. If setup is blocked, explain the specific blocker or required user choice.
|
||||||
|
Answer questions and self-contained code examples directly when the conversation and available tools are sufficient.
|
||||||
If a workspace is needed, use list_templates before create_workspace and follow its ` + chattool.NextStepField + `. Call read_template only when you need template parameter or preset details.`
|
If a workspace is needed, use list_templates before create_workspace and follow its ` + chattool.NextStepField + `. Call read_template only when you need template parameter or preset details.`
|
||||||
|
|
||||||
const workspaceDetachedNoCreateAwareness = workspaceDetachedAwarenessBase + ` This delegated chat cannot create or start a workspace. If workspace-backed work is required, report that need to the parent agent instead of trying workspace tools.`
|
const workspaceDetachedNoCreateAwareness = workspaceDetachedAwarenessBase + ` This delegated chat cannot create or start a workspace. If workspace-backed work is required, report that need to the parent agent instead of trying workspace tools.`
|
||||||
|
|||||||
@@ -77,6 +77,8 @@ Not every chat requires a workspace.
|
|||||||
The agent uses the conversation and available tools when they're sufficient for your request.
|
The agent uses the conversation and available tools when they're sufficient for your request.
|
||||||
If missing tools, skills, MCP integrations, or context block progress, the agent creates a suitable workspace and uses its capabilities to continue.
|
If missing tools, skills, MCP integrations, or context block progress, the agent creates a suitable workspace and uses its capabilities to continue.
|
||||||
Workspaces also support reading code, running commands, and editing files.
|
Workspaces also support reading code, running commands, and editing files.
|
||||||
|
Requests to fix a bug or build an app authorize the necessary workspace setup; you don't need to ask for a workspace separately.
|
||||||
|
Questions and self-contained code examples don't need a workspace when the conversation and available tools are sufficient.
|
||||||
|
|
||||||
This means:
|
This means:
|
||||||
|
|
||||||
|
|||||||
@@ -36,9 +36,11 @@ Two properties hold for every built-in tool:
|
|||||||
|
|
||||||
## Workspace creation tools
|
## Workspace creation tools
|
||||||
|
|
||||||
A chat starts without a workspace, and many requests are answered without
|
A chat can start without a workspace.
|
||||||
one. When the agent needs compute (to read files, run commands, or edit
|
When the conversation and available tools are sufficient, the agent answers without provisioning one.
|
||||||
code), it provisions a workspace using three tools:
|
If missing tools, skills, MCP integrations, or context block progress, or the task needs workspace file access or execution, the agent provisions a workspace.
|
||||||
|
You don't need to request a workspace separately for setup needed to complete your task.
|
||||||
|
The agent uses three tools to provision a workspace:
|
||||||
|
|
||||||
| Tool | Purpose |
|
| Tool | Purpose |
|
||||||
|--------------------|-----------------------------------------------------------|
|
|--------------------|-----------------------------------------------------------|
|
||||||
|
|||||||
Reference in new issue
Block a user