fix(coderd/x/chatd): apply Anthropic cache breakpoints to the compaction summary

The summary request now carries the turn's tool definitions so it can
reuse the turn's cached prefix, but Anthropic only reads the cache at
explicit cache_control breakpoints and the summary prompt had none, so
on Anthropic the definitions were pure cost. Run the same
addAnthropicPromptCaching pass over the summary prompt.

Two follow-ups on the same request:

- A compaction override always targets another model. Prompt caches are
  model-scoped and provider-native tools are model-specific, so the
  chat model's definitions buy the override request nothing and can get
  it rejected. Drop them for every override, not only when the provider
  instance differs.
- Treat HTTP 413 as a size rejection in isContextTooLargeError, so a
  request-body limit on the provider or a gateway also takes the
  tool-less retry.
This commit is contained in:
Michael Suchacz committed 2026-09-16 22:23:21 +00:00
1 parent a7fa7a61e4
commit e7bd72390c
4 files changed
+65 -36

No files matched your search

+6 -27
View File
@@ -5635,6 +5635,7 @@ func TestActiveServer_ManualCompaction(t *testing.T) {
if strings.Contains(body, "You are performing a context compaction") {
compactionRequests.Add(1)
require.Contains(t, body, "hello from the user")
require.True(t, anthropicMessageHasEphemeralCacheControl(t, req.Messages[len(req.Messages)-1]))
return anthropicCompactionResponse(compactionSummary)
}
return chattest.AnthropicNonStreamingResponse("title")
@@ -6256,11 +6257,11 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
thresholdPercent = int32(70)
)
seedOverrideModel := func(ctx context.Context, t *testing.T, db database.Store, chatModel database.ChatModelConfig, aiProviderID uuid.UUID, modelName, effort string, contextLimit int64) database.ChatModelConfig {
seedOverrideModel := func(ctx context.Context, t *testing.T, db database.Store, chatModel database.ChatModelConfig, modelName, effort string, contextLimit int64) database.ChatModelConfig {
t.Helper()
overrideModel := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
Model: modelName,
AIProviderID: uuid.NullUUID{UUID: aiProviderID, Valid: true},
AIProviderID: chatModel.AIProviderID,
ContextLimit: contextLimit,
OrganizationID: chatModel.OrganizationID,
})
@@ -6282,7 +6283,6 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
name string
overrideModel string
effort string
separateProvider bool
assertSummaryRequest func(t *testing.T, req *chattest.AnthropicRequest)
}{
{
@@ -6316,17 +6316,6 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
require.Contains(t, string(req.Thinking), `"type":"adaptive"`)
},
},
{
// A separate provider instance shares no prompt cache with the
// chat model, so the summary request carries no tool definitions.
name: "override on a separate provider instance",
overrideModel: overrideModelName,
effort: "high",
separateProvider: true,
assertSummaryRequest: func(t *testing.T, req *chattest.AnthropicRequest) {
require.Empty(t, req.Tools)
},
},
}
for _, tc := range routingCases {
@@ -6342,9 +6331,7 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
if !req.Stream {
if strings.Contains(body, "You are performing a context compaction") {
require.Equal(t, tc.overrideModel, req.Model)
if !tc.separateProvider {
require.NotEmpty(t, req.Tools)
}
require.Empty(t, req.Tools, "an override model shares no prompt cache, so the summary carries no tool definitions")
tc.assertSummaryRequest(t, req)
return anthropicCompactionResponse(compactionSummary)
}
@@ -6368,15 +6355,7 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
})
user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL)
model = updateChatModelCompressionThreshold(t, db, model, 100, thresholdPercent)
overrideProviderID := model.AIProviderID.UUID
if tc.separateProvider {
overrideProvider := dbgen.AIProvider(t, db, database.AIProvider{Type: database.AIProviderTypeAnthropic}, func(params *database.InsertAIProviderParams) {
params.BaseUrl = anthropicURL
})
dbgen.AIProviderKey(t, db, database.AIProviderKey{ProviderID: overrideProvider.ID})
overrideProviderID = overrideProvider.ID
}
overrideModel := seedOverrideModel(ctx, t, db, model, overrideProviderID, tc.overrideModel, tc.effort, 1_000_000)
overrideModel := seedOverrideModel(ctx, t, db, model, tc.overrideModel, tc.effort, 1_000_000)
ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID)
ctrl := gomock.NewController(t)
@@ -6483,7 +6462,7 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
// limit makes the effective threshold 70 tokens, so compaction
// must trigger.
model = updateChatModelCompressionThreshold(t, db, model, 1_000, thresholdPercent)
seedOverrideModel(ctx, t, db, model, model.AIProviderID.UUID, overrideModelName, "high", 100)
seedOverrideModel(ctx, t, db, model, overrideModelName, "high", 100)
ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID)
ctrl := gomock.NewController(t)
+9 -3
View File
@@ -434,6 +434,11 @@ func generateCompactionSummary(
Role: fantasy.MessageRoleUser,
Content: summaryParts,
})
// Anthropic only reads the cache at explicit breakpoints, so without
// these the shared tool and history prefix is never a cache hit.
if shouldApplyAnthropicPromptCaching(model) {
addAnthropicPromptCaching(summaryPrompt)
}
summaryCtx, finishDebugRun := startCompactionDebugRun(ctx, options)
defer func() {
@@ -495,14 +500,15 @@ var contextTooLargePhrases = []string{
"too long",
}
// isContextTooLargeError reports whether a provider rejected a request
// because the prompt exceeded the model's context window.
// isContextTooLargeError reports whether a provider rejected a request for
// its size: the prompt exceeded the model's context window, or the body
// exceeded a request size limit on the provider or a gateway in front of it.
func isContextTooLargeError(err error) bool {
providerErr, ok := errors.AsType[*fantasy.ProviderError](err)
if !ok {
return false
}
if providerErr.IsContextTooLarge() {
if providerErr.IsContextTooLarge() || providerErr.StatusCode == http.StatusRequestEntityTooLarge {
return true
}
if providerErr.StatusCode != http.StatusBadRequest {
@@ -8,6 +8,7 @@ import (
"time"
"charm.land/fantasy"
fantasyanthropic "charm.land/fantasy/providers/anthropic"
fantasyopenai "charm.land/fantasy/providers/openai"
"github.com/google/uuid"
"github.com/sqlc-dev/pqtype"
@@ -321,6 +322,43 @@ func TestGenerateCompactionSummaryUsesToolDefinitions(t *testing.T) {
require.Equal(t, originalMessages, messages)
}
func TestGenerateCompactionSummaryAppliesAnthropicPromptCaching(t *testing.T) {
t.Parallel()
messages := []fantasy.Message{
textMessage(fantasy.MessageRoleSystem, "system"),
textMessage(fantasy.MessageRoleUser, "hello"),
textMessage(fantasy.MessageRoleAssistant, "hi"),
}
originalMessages := append([]fantasy.Message(nil), messages...)
var got fantasy.Call
model := &chattest.FakeModel{
ProviderName: fantasyanthropic.Name,
ModelName: "claude",
GenerateFn: func(_ context.Context, call fantasy.Call) (*fantasy.Response, error) {
got = call
return &fantasy.Response{Content: []fantasy.Content{fantasy.TextContent{Text: "summary"}}}, nil
},
}
_, err := generateCompactionSummary(context.Background(), model, messages, CompactionOptions{
SummaryPrompt: "summarize",
})
require.NoError(t, err)
require.Len(t, got.Prompt, 4)
cacheControl := fantasy.ProviderOptions{
fantasyanthropic.Name: &fantasyanthropic.ProviderCacheControlOptions{
CacheControl: fantasyanthropic.CacheControl{Type: "ephemeral"},
},
}
// Breakpoints land on the system message and the final two messages.
require.Equal(t, cacheControl, got.Prompt[0].ProviderOptions)
require.Nil(t, got.Prompt[1].ProviderOptions)
require.Equal(t, cacheControl, got.Prompt[2].ProviderOptions)
require.Equal(t, cacheControl, got.Prompt[3].ProviderOptions)
require.Equal(t, originalMessages, messages)
}
func TestGenerateCompactionSummaryRetriesWithoutToolsWhenContextTooLarge(t *testing.T) {
t.Parallel()
@@ -478,6 +516,14 @@ func TestIsContextTooLargeError(t *testing.T) {
}),
want: true,
},
{
name: "request entity too large",
err: &fantasy.ProviderError{
StatusCode: http.StatusRequestEntityTooLarge,
Message: "Request Entity Too Large",
},
want: true,
},
{
name: "unrelated bad request",
err: &fantasy.ProviderError{
+4 -6
View File
@@ -1022,12 +1022,10 @@ func (s *taskStarter) generateCompaction(
compactionOpts.ResolvedModel = overrideModel.resolvedModel
compactionOpts.ModelConfigID = overrideModel.dbConfig.ID
compactionOpts.SummaryCall = compactionSummaryCall(overrideModel)
// A separate provider instance shares no prompt cache with the
// chat model and may reject its provider-defined tools or tool
// schemas, so the summary request carries no tool definitions.
if !sameCompactionProviderIdentity(prepared.Compaction.ChatModelConfig, overrideModel.dbConfig) {
compactionOpts.ToolDefinitions = nil
}
// Prompt caches are model-scoped and provider-native tools are
// model-specific, so the chat model's tool definitions buy the
// override request nothing and can get it rejected.
compactionOpts.ToolDefinitions = nil
compactionOpts.Messages = sanitizeCompactionPrompt(
ctx,
logger,