fix(coderd/x/chatd): retry compaction summary without tools on context overflow

Sending the turn's tool definitions with the compaction summary request
makes it larger than the generation that just overflowed, so a chat at
the provider's real context window can no longer recover through
compaction: both manual /compact and the automatic retrigger fail with
the same context-window rejection.

When the provider rejects the tool-carrying summary request for size,
regenerate the summary once without tools. The tool-less request is the
shape compaction sent before and still fits whenever the history alone
does. The context-window check reuses fantasy's parsed classification
and also matches the bad-request wordings fantasy does not parse, such
as the OpenAI Responses API "Your input exceeds the context window of
this model."
This commit is contained in:
Michael Suchacz committed 2026-09-16 21:51:41 +00:00
1 parent 32d261f94f
commit 8799db2f0e
3 files changed
+301

No files matched your search

+63
View File
@@ -5773,6 +5773,69 @@ func TestActiveServer_ManualCompaction(t *testing.T) {
"prompt history contains the compressed summary boundary")
})
t.Run("retries the summary without tools when the prompt exceeds the context window", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
db, ps := dbtestutil.NewDB(t)
var mu sync.Mutex
var compactionToolCounts []int
anthropicURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse {
body := anthropicRequestBody(t, *req)
if !req.Stream {
if strings.Contains(body, "You are performing a context compaction") {
mu.Lock()
compactionToolCounts = append(compactionToolCounts, len(req.Tools))
mu.Unlock()
// The tool definitions push the summary request past
// the window that the tool-less request still fits.
if len(req.Tools) > 0 {
return chattest.AnthropicResponse{Error: &chattest.ErrorResponse{
StatusCode: http.StatusBadRequest,
Type: "invalid_request_error",
Message: "prompt is too long: 20000 tokens > 16385 maximum",
}}
}
return anthropicCompactionResponse(compactionSummary)
}
return chattest.AnthropicNonStreamingResponse("title")
}
return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunks("assistant answer")...)
})
user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL)
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath()))
})
chat := createChatThroughServer(ctx, t, db, server, org.ID, user.ID, model.ID, "hello from the user")
chat = waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
compacted, err := server.CompactChat(ctx, chat)
require.NoError(t, err)
require.Equal(t, database.ChatStatusRunning, compacted.Status)
chat = waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
require.False(t, chat.LastError.Valid)
require.False(t, chat.CompactionRequestedAt.Valid)
mu.Lock()
toolCounts := slices.Clone(compactionToolCounts)
mu.Unlock()
require.Len(t, toolCounts, 2, "one rejected request with tools, one tool-less retry")
require.Positive(t, toolCounts[0], "the first summary request carries the turn's tool definitions")
require.Zero(t, toolCounts[1], "the retry drops the tool definitions")
messages := chatMessages(ctx, t, db, chat.ID)
promptMessages, err := db.GetChatMessagesForPromptByChatID(ctx, chat.ID)
require.NoError(t, err)
compressed := compressedChatSummarizedMessages(t, append(promptMessages, messages...))
require.Len(t, compressed.results, 1)
resultPart := singlePartOfType(t, compressed.results[0], codersdk.ChatMessagePartTypeToolResult)
var result map[string]any
require.NoError(t, json.Unmarshal(resultPart.Result, &result))
require.Equal(t, compactionSummary, result["summary"])
})
t.Run("busy chat rejects manual compaction", func(t *testing.T) {
t.Parallel()
+45
View File
@@ -3,6 +3,8 @@ package chatloop
import (
"context"
"encoding/json"
"errors"
"net/http"
"strings"
"time"
@@ -453,6 +455,15 @@ func generateCompactionSummary(
call.Prompt = summaryPrompt
call.Tools = options.ToolDefinitions
response, err := model.Generate(summaryCtx, call)
if err != nil && len(call.Tools) > 0 && isContextTooLargeError(err) {
// Tool definitions keep the summary request on the parent turn's
// cacheable prefix, but they also make it larger than the turn
// that just overflowed. Compaction is the recovery path for an
// over-limit conversation, so fall back to the tool-less request,
// which still fits whenever the history alone does.
call.Tools = nil
response, err = model.Generate(summaryCtx, call)
}
if err != nil {
return "", xerrors.Errorf("generate summary text: %w", err)
}
@@ -471,3 +482,37 @@ func generateCompactionSummary(
}
return strings.TrimSpace(strings.Join(parts, " ")), nil
}
// contextTooLargePhrases are context-window rejections fantasy does not
// parse, such as the OpenAI Responses API "Your input exceeds the context
// window of this model." and Anthropic-family "too long" rejections
// without token counts. The response body is included so the OpenAI
// error code "context_length_exceeded" matches regardless of wording.
var contextTooLargePhrases = []string{
"context window",
"context length",
"context_length",
"too long",
}
// isContextTooLargeError reports whether a provider rejected a request
// because the prompt exceeded the model's context window.
func isContextTooLargeError(err error) bool {
providerErr, ok := errors.AsType[*fantasy.ProviderError](err)
if !ok {
return false
}
if providerErr.IsContextTooLarge() {
return true
}
if providerErr.StatusCode != http.StatusBadRequest {
return false
}
text := strings.ToLower(providerErr.Error() + " " + string(providerErr.ResponseBody))
for _, phrase := range contextTooLargePhrases {
if strings.Contains(text, phrase) {
return true
}
}
return false
}
@@ -3,6 +3,7 @@ package chatloop
import (
"context"
"encoding/json"
"net/http"
"testing"
"time"
@@ -320,6 +321,198 @@ func TestGenerateCompactionSummaryUsesToolDefinitions(t *testing.T) {
require.Equal(t, originalMessages, messages)
}
func TestGenerateCompactionSummaryRetriesWithoutToolsWhenContextTooLarge(t *testing.T) {
t.Parallel()
toolDefinitions := []fantasy.Tool{
fantasy.FunctionTool{Name: "read_file", InputSchema: map[string]any{"type": "object"}},
}
toolChoice := fantasy.ToolChoiceNone
var calls []fantasy.Call
model := &chattest.FakeModel{
ProviderName: "fake",
ModelName: "fake-model",
GenerateFn: func(_ context.Context, call fantasy.Call) (*fantasy.Response, error) {
calls = append(calls, call)
if len(call.Tools) > 0 {
return nil, &fantasy.ProviderError{
Title: "bad request",
Message: "Your input exceeds the context window of this model. Please adjust your input and try again.",
StatusCode: http.StatusBadRequest,
}
}
return &fantasy.Response{Content: []fantasy.Content{fantasy.TextContent{Text: "summary"}}}, nil
},
}
summary, err := generateCompactionSummary(context.Background(), model,
[]fantasy.Message{textMessage(fantasy.MessageRoleUser, "hello")},
CompactionOptions{
SummaryPrompt: "summarize",
SummaryCall: fantasy.Call{ToolChoice: &toolChoice},
ToolDefinitions: toolDefinitions,
},
)
require.NoError(t, err)
require.Equal(t, "summary", summary)
require.Len(t, calls, 2)
require.Equal(t, toolDefinitions, calls[0].Tools)
require.Nil(t, calls[1].Tools)
require.Equal(t, calls[0].Prompt, calls[1].Prompt)
require.Equal(t, calls[0].ToolChoice, calls[1].ToolChoice)
}
func TestGenerateCompactionSummaryDoesNotRetryOtherFailures(t *testing.T) {
t.Parallel()
contextTooLarge := &fantasy.ProviderError{
Message: "Your input exceeds the context window of this model.",
StatusCode: http.StatusBadRequest,
}
cases := []struct {
name string
toolDefinitions []fantasy.Tool
err error
}{
{
name: "unrelated bad request",
toolDefinitions: []fantasy.Tool{fantasy.FunctionTool{Name: "read_file"}},
err: &fantasy.ProviderError{
Message: "Unsupported parameter: 'temperature' is not supported with this model.",
StatusCode: http.StatusBadRequest,
},
},
{
name: "server error",
toolDefinitions: []fantasy.Tool{fantasy.FunctionTool{Name: "read_file"}},
err: &fantasy.ProviderError{
Message: "context window service unavailable",
StatusCode: http.StatusServiceUnavailable,
},
},
{
name: "non-provider error",
toolDefinitions: []fantasy.Tool{fantasy.FunctionTool{Name: "read_file"}},
err: xerrors.New("context window"),
},
{
name: "no tools to drop",
err: contextTooLarge,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
calls := 0
model := &chattest.FakeModel{
ProviderName: "fake",
ModelName: "fake-model",
GenerateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) {
calls++
return nil, tc.err
},
}
_, err := generateCompactionSummary(context.Background(), model,
[]fantasy.Message{textMessage(fantasy.MessageRoleUser, "hello")},
CompactionOptions{
SummaryPrompt: "summarize",
ToolDefinitions: tc.toolDefinitions,
},
)
require.ErrorIs(t, err, tc.err)
require.Equal(t, 1, calls)
})
}
}
func TestIsContextTooLargeError(t *testing.T) {
t.Parallel()
cases := []struct {
name string
err error
want bool
}{
{
name: "parsed by fantasy",
err: &fantasy.ProviderError{
StatusCode: http.StatusBadRequest,
ContextTooLargeErr: true,
ContextMaxTokens: 16385,
ContextUsedTokens: 20000,
},
want: true,
},
{
name: "openai responses wording",
err: &fantasy.ProviderError{
StatusCode: http.StatusBadRequest,
Message: "Your input exceeds the context window of this model. Please adjust your input and try again.",
},
want: true,
},
{
name: "anthropic wording",
err: &fantasy.ProviderError{
StatusCode: http.StatusBadRequest,
Message: "prompt is too long: 213462 tokens > 200000 maximum",
},
want: true,
},
{
name: "openai error code in response body",
err: &fantasy.ProviderError{
StatusCode: http.StatusBadRequest,
Message: "Request too large for this model.",
ResponseBody: []byte(`{"error":{"message":"Request too large for this model.","type":"invalid_request_error","code":"context_length_exceeded"}}`),
},
want: true,
},
{
name: "wrapped provider error",
err: xerrors.Errorf("generate summary text: %w", &fantasy.ProviderError{
StatusCode: http.StatusBadRequest,
Message: "This model's maximum context length is 16385 tokens.",
}),
want: true,
},
{
name: "unrelated bad request",
err: &fantasy.ProviderError{
StatusCode: http.StatusBadRequest,
Message: "Invalid schema for function 'read_file'.",
},
want: false,
},
{
name: "matching wording on a non bad-request status",
err: &fantasy.ProviderError{
StatusCode: http.StatusTooManyRequests,
Message: "context window tokens per minute exceeded",
},
want: false,
},
{
name: "non-provider error",
err: xerrors.New("prompt is too long"),
want: false,
},
{
name: "nil",
err: nil,
want: false,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
require.Equal(t, tc.want, isContextTooLargeError(tc.err))
})
}
}
func TestGenerateCompactionSummary_UsesCallerContext(t *testing.T) {
t.Parallel()