Closes https://linear.app/codercom/issue/CODAGT-1525/allow-computer-use-with-claude-models-from-aws-bedrock-ai-providers Computer use was only available on models from Anthropic and OpenAI AI providers, as we decided support from the AI provider type alone, in five separate places (spawn, turn setup, override validation, telemetry and the settings UI). A Bedrock AI provider serves Claude models over the Anthropic Messages API though, and chatd already routes those through the Anthropic fantasy client, so there's no reason computer use shouldn't work on them. To fix, there's now a single rule, `chatprovider.ComputerUseProvider(providerType, model)`, derived from the client chatd builds for the model (`chatprovider.ClientProvider`, previously `aibridgeClientProvider`). If the model uses the Anthropic client, it gets the Anthropic computer use tool. The OpenAI tool is still only offered on OpenAI AI providers, as I haven't tested it against Bedrock Mantle, and I couldn't find anyone else doing it either. All five places now call this function. The UI can't re-derive Bedrock routing without duplicating the model ID heuristic, so `ChatModel` gets a read-only `computer_use_provider` field (`anthropic`, `openai`, or omitted when computer use isn't supported). The computer use model override endpoint now accepts Bedrock Claude models, and its 400 message has changed. Whilst in here I also fixed a latent bug where the default model credential check looked up the user's keys under the tool flavour (`anthropic`) instead of the provider type (`bedrock`). One caveat: on a Bedrock AI provider using InvokeModel, AI Gateway runs the provider's configured model rather than the chat model, whilst we pick the computer use tool version from the chat model ID. It's not specific to computer use, and should go away with [AIGOV-747](https://linear.app/codercom/issue/AIGOV-747/deprecate-the-bedrock-invokemodel-protocol-with-model-translation). In the meantime, #30752 warns when an InvokeModel provider ignores the chat model identifier.
199 lines
6.8 KiB
Go
199 lines
6.8 KiB
Go
package chatd
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
|
|
fantasyanthropic "charm.land/fantasy/providers/anthropic"
|
|
"github.com/google/uuid"
|
|
"golang.org/x/xerrors"
|
|
|
|
"cdr.dev/slog/v3"
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/database/dbauthz"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
|
|
openaicomputeruse "github.com/coder/coder/v2/coderd/x/chatd/chatopenai/computeruse"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chattool"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
"github.com/coder/coder/v2/codersdk/workspacesdk"
|
|
"github.com/coder/quartz"
|
|
)
|
|
|
|
type computerUseModel struct {
|
|
config database.ChatModelConfig
|
|
// reasoningEffort comes from the organization override, if any.
|
|
reasoningEffort *string
|
|
}
|
|
|
|
func computerUseModelLabel(config database.ChatModelConfig) string {
|
|
if config.DisplayName != "" && config.DisplayName != config.Model {
|
|
return fmt.Sprintf("%q (%s)", config.DisplayName, config.Model)
|
|
}
|
|
return fmt.Sprintf("%q", config.Model)
|
|
}
|
|
|
|
// resolveComputerUseModel picks the model for a computer use subagent: the
|
|
// organization's computer_use override, else the organization's default chat
|
|
// model. A non-empty reason means computer use is unavailable to the owner.
|
|
func (p *Server) resolveComputerUseModel(
|
|
ctx context.Context,
|
|
ownerID uuid.UUID,
|
|
organizationID uuid.UUID,
|
|
) (model computerUseModel, reason string, err error) {
|
|
//nolint:gocritic // Chatd reads organization model configuration for the chat owner.
|
|
chatdCtx := dbauthz.AsChatd(ctx)
|
|
|
|
override, err := p.resolveModelOverride(chatdCtx, modelOverrideSpec{
|
|
context: string(codersdk.ChatModelOverrideContextComputerUse),
|
|
ownerID: ownerID,
|
|
organizationID: organizationID,
|
|
})
|
|
if err != nil {
|
|
return computerUseModel{}, "", err
|
|
}
|
|
|
|
overrideState := "no computer use model is configured"
|
|
if override.Set || override.Unavailable {
|
|
// A usable override returns below, so a reason built after this point
|
|
// means the saved override cannot be used.
|
|
overrideState = "the configured computer use model cannot be used"
|
|
}
|
|
|
|
if override.Set {
|
|
// resolveModelOverride returns only enabled models of enabled AI
|
|
// providers whose credentials the owner can use.
|
|
aiProvider, err := p.db.GetAIProviderByID(chatdCtx, override.Config.AIProviderID.UUID)
|
|
if err != nil {
|
|
return computerUseModel{}, "", xerrors.Errorf("get AI provider for model %s: %w", override.Config.ID, err)
|
|
}
|
|
|
|
if _, ok := chatprovider.ComputerUseProvider(aiProvider.Type, override.Config.Model); ok {
|
|
return computerUseModel{
|
|
config: override.Config,
|
|
reasoningEffort: override.ReasoningEffort,
|
|
}, "", nil
|
|
}
|
|
|
|
// The override API rejects these models, but the model API can still
|
|
// change the saved model to one. Ignore the override then, as the
|
|
// settings page does.
|
|
p.logger.Info(ctx, "computer use model override is unsupported, using default model",
|
|
slog.F("model_config_id", override.Config.ID),
|
|
slog.F("model", override.Config.Model),
|
|
slog.F("ai_provider_type", aiProvider.Type),
|
|
)
|
|
}
|
|
|
|
// Read the default under the owner's model ACLs, as chat turns do.
|
|
modelCtx, err := p.callerModelConfigContext(chatdCtx, ownerID)
|
|
if err != nil {
|
|
return computerUseModel{}, "", err
|
|
}
|
|
|
|
model.config, err = effectiveDefaultChatModelConfig(modelCtx, p.db, organizationID)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return computerUseModel{}, fmt.Sprintf(
|
|
`type "computer_use" is unavailable because %s and the organization has no default chat model available to the chat owner`,
|
|
overrideState,
|
|
), nil
|
|
}
|
|
if err != nil {
|
|
return computerUseModel{}, "", xerrors.Errorf("get default chat model config: %w", err)
|
|
}
|
|
|
|
// The default lookup returns only enabled models of enabled AI providers.
|
|
aiProvider, err := p.db.GetAIProviderByID(chatdCtx, model.config.AIProviderID.UUID)
|
|
if err != nil {
|
|
return computerUseModel{}, "", xerrors.Errorf("get AI provider for model %s: %w", model.config.ID, err)
|
|
}
|
|
|
|
label := computerUseModelLabel(model.config)
|
|
if _, ok := chatprovider.ComputerUseProvider(aiProvider.Type, model.config.Model); !ok {
|
|
return computerUseModel{}, fmt.Sprintf(
|
|
`type "computer_use" is unavailable because %s and model %s uses AI provider %q of type %q; computer use requires %s`,
|
|
overrideState, label, aiProvider.Name, aiProvider.Type, chatprovider.ComputerUseModelRequirement,
|
|
), nil
|
|
}
|
|
|
|
providerKeys, err := p.resolveUserProviderAPIKeys(chatdCtx, ownerID, aiProvider.ID)
|
|
if err != nil {
|
|
return computerUseModel{}, "", xerrors.Errorf("resolve provider API keys: %w", err)
|
|
}
|
|
|
|
if !userCanUseProviderKeys(providerKeys, chatprovider.NormalizeProvider(string(aiProvider.Type))) {
|
|
return computerUseModel{}, fmt.Sprintf(
|
|
`type "computer_use" is unavailable because %s and credentials for AI provider %q, used by model %s, are not configured`,
|
|
overrideState, aiProvider.Name, label,
|
|
), nil
|
|
}
|
|
|
|
return model, "", nil
|
|
}
|
|
|
|
type computerUseProviderToolOptions struct {
|
|
provider codersdk.ChatComputerUseProvider
|
|
toolVersion fantasyanthropic.ComputerUseToolVersion
|
|
isPlanModeTurn bool
|
|
isComputerUse bool
|
|
getWorkspaceConn func(context.Context) (workspacesdk.AgentConn, error)
|
|
storeFile chattool.StoreFileFunc
|
|
clock quartz.Clock
|
|
logger slog.Logger
|
|
}
|
|
|
|
func appendComputerUseProviderTool(
|
|
providerTools []chatloop.ProviderTool,
|
|
opts computerUseProviderToolOptions,
|
|
) ([]chatloop.ProviderTool, error) {
|
|
// This helper is called for every chat turn. Only chats created by the
|
|
// computer_use subagent definition have ChatModeComputerUse, which filters
|
|
// out root, general, and explore chats. Plan mode is separate from Mode, so
|
|
// planning turns stay gated even for computer-use chats.
|
|
if opts.isPlanModeTurn || !opts.isComputerUse {
|
|
return providerTools, nil
|
|
}
|
|
|
|
desktopGeometry := chattool.DefaultComputerUseDesktopGeometry(opts.provider)
|
|
definition, err := chattool.ComputerUseProviderTool(
|
|
opts.provider,
|
|
opts.toolVersion,
|
|
desktopGeometry.DeclaredWidth,
|
|
desktopGeometry.DeclaredHeight,
|
|
)
|
|
if err != nil {
|
|
return providerTools, xerrors.Errorf(
|
|
"build computer use provider tool for provider %q: %w",
|
|
opts.provider,
|
|
err,
|
|
)
|
|
}
|
|
|
|
clock := opts.clock
|
|
if clock == nil {
|
|
clock = quartz.NewReal()
|
|
}
|
|
providerTool := chatloop.ProviderTool{
|
|
Definition: definition,
|
|
Runner: chattool.NewComputerUseTool(
|
|
opts.provider,
|
|
opts.toolVersion,
|
|
desktopGeometry.DeclaredWidth,
|
|
desktopGeometry.DeclaredHeight,
|
|
opts.getWorkspaceConn,
|
|
opts.storeFile,
|
|
clock,
|
|
opts.logger,
|
|
),
|
|
}
|
|
if opts.provider == codersdk.ChatComputerUseProviderOpenAI {
|
|
// OpenAI computer-use image results need detail metadata so the model receives
|
|
// the screenshot at original detail when the chat loop sends the tool result.
|
|
providerTool.ResultProviderMetadata = openaicomputeruse.ResultProviderMetadata
|
|
}
|
|
|
|
return append(providerTools, providerTool), nil
|
|
}
|