fix(coderd/x/chatd): apply Anthropic cache breakpoints to the compaction summary
The summary request now carries the turn's tool definitions so it can reuse the turn's cached prefix, but Anthropic only reads the cache at explicit cache_control breakpoints and the summary prompt had none, so on Anthropic the definitions were pure cost. Run the same addAnthropicPromptCaching pass over the summary prompt. Two follow-ups on the same request: - A compaction override always targets another model. Prompt caches are model-scoped and provider-native tools are model-specific, so the chat model's definitions buy the override request nothing and can get it rejected. Drop them for every override, not only when the provider instance differs. - Treat HTTP 413 as a size rejection in isContextTooLargeError, so a request-body limit on the provider or a gateway also takes the tool-less retry.
This commit is contained in:
1 parent
a7fa7a61e4
commit
e7bd72390c
4 files changed
+65
-36
No files matched your search
@@ -5635,6 +5635,7 @@ func TestActiveServer_ManualCompaction(t *testing.T) {
|
||||
if strings.Contains(body, "You are performing a context compaction") {
|
||||
compactionRequests.Add(1)
|
||||
require.Contains(t, body, "hello from the user")
|
||||
require.True(t, anthropicMessageHasEphemeralCacheControl(t, req.Messages[len(req.Messages)-1]))
|
||||
return anthropicCompactionResponse(compactionSummary)
|
||||
}
|
||||
return chattest.AnthropicNonStreamingResponse("title")
|
||||
@@ -6256,11 +6257,11 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
|
||||
thresholdPercent = int32(70)
|
||||
)
|
||||
|
||||
seedOverrideModel := func(ctx context.Context, t *testing.T, db database.Store, chatModel database.ChatModelConfig, aiProviderID uuid.UUID, modelName, effort string, contextLimit int64) database.ChatModelConfig {
|
||||
seedOverrideModel := func(ctx context.Context, t *testing.T, db database.Store, chatModel database.ChatModelConfig, modelName, effort string, contextLimit int64) database.ChatModelConfig {
|
||||
t.Helper()
|
||||
overrideModel := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
|
||||
Model: modelName,
|
||||
AIProviderID: uuid.NullUUID{UUID: aiProviderID, Valid: true},
|
||||
AIProviderID: chatModel.AIProviderID,
|
||||
ContextLimit: contextLimit,
|
||||
OrganizationID: chatModel.OrganizationID,
|
||||
})
|
||||
@@ -6282,7 +6283,6 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
|
||||
name string
|
||||
overrideModel string
|
||||
effort string
|
||||
separateProvider bool
|
||||
assertSummaryRequest func(t *testing.T, req *chattest.AnthropicRequest)
|
||||
}{
|
||||
{
|
||||
@@ -6316,17 +6316,6 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
|
||||
require.Contains(t, string(req.Thinking), `"type":"adaptive"`)
|
||||
},
|
||||
},
|
||||
{
|
||||
// A separate provider instance shares no prompt cache with the
|
||||
// chat model, so the summary request carries no tool definitions.
|
||||
name: "override on a separate provider instance",
|
||||
overrideModel: overrideModelName,
|
||||
effort: "high",
|
||||
separateProvider: true,
|
||||
assertSummaryRequest: func(t *testing.T, req *chattest.AnthropicRequest) {
|
||||
require.Empty(t, req.Tools)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range routingCases {
|
||||
@@ -6342,9 +6331,7 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
|
||||
if !req.Stream {
|
||||
if strings.Contains(body, "You are performing a context compaction") {
|
||||
require.Equal(t, tc.overrideModel, req.Model)
|
||||
if !tc.separateProvider {
|
||||
require.NotEmpty(t, req.Tools)
|
||||
}
|
||||
require.Empty(t, req.Tools, "an override model shares no prompt cache, so the summary carries no tool definitions")
|
||||
tc.assertSummaryRequest(t, req)
|
||||
return anthropicCompactionResponse(compactionSummary)
|
||||
}
|
||||
@@ -6368,15 +6355,7 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
|
||||
})
|
||||
user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL)
|
||||
model = updateChatModelCompressionThreshold(t, db, model, 100, thresholdPercent)
|
||||
overrideProviderID := model.AIProviderID.UUID
|
||||
if tc.separateProvider {
|
||||
overrideProvider := dbgen.AIProvider(t, db, database.AIProvider{Type: database.AIProviderTypeAnthropic}, func(params *database.InsertAIProviderParams) {
|
||||
params.BaseUrl = anthropicURL
|
||||
})
|
||||
dbgen.AIProviderKey(t, db, database.AIProviderKey{ProviderID: overrideProvider.ID})
|
||||
overrideProviderID = overrideProvider.ID
|
||||
}
|
||||
overrideModel := seedOverrideModel(ctx, t, db, model, overrideProviderID, tc.overrideModel, tc.effort, 1_000_000)
|
||||
overrideModel := seedOverrideModel(ctx, t, db, model, tc.overrideModel, tc.effort, 1_000_000)
|
||||
ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID)
|
||||
|
||||
ctrl := gomock.NewController(t)
|
||||
@@ -6483,7 +6462,7 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
|
||||
// limit makes the effective threshold 70 tokens, so compaction
|
||||
// must trigger.
|
||||
model = updateChatModelCompressionThreshold(t, db, model, 1_000, thresholdPercent)
|
||||
seedOverrideModel(ctx, t, db, model, model.AIProviderID.UUID, overrideModelName, "high", 100)
|
||||
seedOverrideModel(ctx, t, db, model, overrideModelName, "high", 100)
|
||||
ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID)
|
||||
|
||||
ctrl := gomock.NewController(t)
|
||||
|
||||
@@ -434,6 +434,11 @@ func generateCompactionSummary(
|
||||
Role: fantasy.MessageRoleUser,
|
||||
Content: summaryParts,
|
||||
})
|
||||
// Anthropic only reads the cache at explicit breakpoints, so without
|
||||
// these the shared tool and history prefix is never a cache hit.
|
||||
if shouldApplyAnthropicPromptCaching(model) {
|
||||
addAnthropicPromptCaching(summaryPrompt)
|
||||
}
|
||||
|
||||
summaryCtx, finishDebugRun := startCompactionDebugRun(ctx, options)
|
||||
defer func() {
|
||||
@@ -495,14 +500,15 @@ var contextTooLargePhrases = []string{
|
||||
"too long",
|
||||
}
|
||||
|
||||
// isContextTooLargeError reports whether a provider rejected a request
|
||||
// because the prompt exceeded the model's context window.
|
||||
// isContextTooLargeError reports whether a provider rejected a request for
|
||||
// its size: the prompt exceeded the model's context window, or the body
|
||||
// exceeded a request size limit on the provider or a gateway in front of it.
|
||||
func isContextTooLargeError(err error) bool {
|
||||
providerErr, ok := errors.AsType[*fantasy.ProviderError](err)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if providerErr.IsContextTooLarge() {
|
||||
if providerErr.IsContextTooLarge() || providerErr.StatusCode == http.StatusRequestEntityTooLarge {
|
||||
return true
|
||||
}
|
||||
if providerErr.StatusCode != http.StatusBadRequest {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"charm.land/fantasy"
|
||||
fantasyanthropic "charm.land/fantasy/providers/anthropic"
|
||||
fantasyopenai "charm.land/fantasy/providers/openai"
|
||||
"github.com/google/uuid"
|
||||
"github.com/sqlc-dev/pqtype"
|
||||
@@ -321,6 +322,43 @@ func TestGenerateCompactionSummaryUsesToolDefinitions(t *testing.T) {
|
||||
require.Equal(t, originalMessages, messages)
|
||||
}
|
||||
|
||||
func TestGenerateCompactionSummaryAppliesAnthropicPromptCaching(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
messages := []fantasy.Message{
|
||||
textMessage(fantasy.MessageRoleSystem, "system"),
|
||||
textMessage(fantasy.MessageRoleUser, "hello"),
|
||||
textMessage(fantasy.MessageRoleAssistant, "hi"),
|
||||
}
|
||||
originalMessages := append([]fantasy.Message(nil), messages...)
|
||||
var got fantasy.Call
|
||||
model := &chattest.FakeModel{
|
||||
ProviderName: fantasyanthropic.Name,
|
||||
ModelName: "claude",
|
||||
GenerateFn: func(_ context.Context, call fantasy.Call) (*fantasy.Response, error) {
|
||||
got = call
|
||||
return &fantasy.Response{Content: []fantasy.Content{fantasy.TextContent{Text: "summary"}}}, nil
|
||||
},
|
||||
}
|
||||
|
||||
_, err := generateCompactionSummary(context.Background(), model, messages, CompactionOptions{
|
||||
SummaryPrompt: "summarize",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got.Prompt, 4)
|
||||
cacheControl := fantasy.ProviderOptions{
|
||||
fantasyanthropic.Name: &fantasyanthropic.ProviderCacheControlOptions{
|
||||
CacheControl: fantasyanthropic.CacheControl{Type: "ephemeral"},
|
||||
},
|
||||
}
|
||||
// Breakpoints land on the system message and the final two messages.
|
||||
require.Equal(t, cacheControl, got.Prompt[0].ProviderOptions)
|
||||
require.Nil(t, got.Prompt[1].ProviderOptions)
|
||||
require.Equal(t, cacheControl, got.Prompt[2].ProviderOptions)
|
||||
require.Equal(t, cacheControl, got.Prompt[3].ProviderOptions)
|
||||
require.Equal(t, originalMessages, messages)
|
||||
}
|
||||
|
||||
func TestGenerateCompactionSummaryRetriesWithoutToolsWhenContextTooLarge(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -478,6 +516,14 @@ func TestIsContextTooLargeError(t *testing.T) {
|
||||
}),
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "request entity too large",
|
||||
err: &fantasy.ProviderError{
|
||||
StatusCode: http.StatusRequestEntityTooLarge,
|
||||
Message: "Request Entity Too Large",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "unrelated bad request",
|
||||
err: &fantasy.ProviderError{
|
||||
|
||||
@@ -1022,12 +1022,10 @@ func (s *taskStarter) generateCompaction(
|
||||
compactionOpts.ResolvedModel = overrideModel.resolvedModel
|
||||
compactionOpts.ModelConfigID = overrideModel.dbConfig.ID
|
||||
compactionOpts.SummaryCall = compactionSummaryCall(overrideModel)
|
||||
// A separate provider instance shares no prompt cache with the
|
||||
// chat model and may reject its provider-defined tools or tool
|
||||
// schemas, so the summary request carries no tool definitions.
|
||||
if !sameCompactionProviderIdentity(prepared.Compaction.ChatModelConfig, overrideModel.dbConfig) {
|
||||
compactionOpts.ToolDefinitions = nil
|
||||
}
|
||||
// Prompt caches are model-scoped and provider-native tools are
|
||||
// model-specific, so the chat model's tool definitions buy the
|
||||
// override request nothing and can get it rejected.
|
||||
compactionOpts.ToolDefinitions = nil
|
||||
compactionOpts.Messages = sanitizeCompactionPrompt(
|
||||
ctx,
|
||||
logger,
|
||||
|
||||
Reference in new issue
Block a user