fix(coderd/x/chatd): retry compaction summary without tools on context overflow
Sending the turn's tool definitions with the compaction summary request makes it larger than the generation that just overflowed, so a chat at the provider's real context window can no longer recover through compaction: both manual /compact and the automatic retrigger fail with the same context-window rejection. When the provider rejects the tool-carrying summary request for size, regenerate the summary once without tools. The tool-less request is the shape compaction sent before and still fits whenever the history alone does. The context-window check reuses fantasy's parsed classification and also matches the bad-request wordings fantasy does not parse, such as the OpenAI Responses API "Your input exceeds the context window of this model."
This commit is contained in:
1 parent
32d261f94f
commit
8799db2f0e
3 files changed
+301
No files matched your search
@@ -5773,6 +5773,69 @@ func TestActiveServer_ManualCompaction(t *testing.T) {
|
||||
"prompt history contains the compressed summary boundary")
|
||||
})
|
||||
|
||||
t.Run("retries the summary without tools when the prompt exceeds the context window", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
db, ps := dbtestutil.NewDB(t)
|
||||
var mu sync.Mutex
|
||||
var compactionToolCounts []int
|
||||
anthropicURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse {
|
||||
body := anthropicRequestBody(t, *req)
|
||||
if !req.Stream {
|
||||
if strings.Contains(body, "You are performing a context compaction") {
|
||||
mu.Lock()
|
||||
compactionToolCounts = append(compactionToolCounts, len(req.Tools))
|
||||
mu.Unlock()
|
||||
// The tool definitions push the summary request past
|
||||
// the window that the tool-less request still fits.
|
||||
if len(req.Tools) > 0 {
|
||||
return chattest.AnthropicResponse{Error: &chattest.ErrorResponse{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Type: "invalid_request_error",
|
||||
Message: "prompt is too long: 20000 tokens > 16385 maximum",
|
||||
}}
|
||||
}
|
||||
return anthropicCompactionResponse(compactionSummary)
|
||||
}
|
||||
return chattest.AnthropicNonStreamingResponse("title")
|
||||
}
|
||||
return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunks("assistant answer")...)
|
||||
})
|
||||
user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL)
|
||||
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath()))
|
||||
})
|
||||
chat := createChatThroughServer(ctx, t, db, server, org.ID, user.ID, model.ID, "hello from the user")
|
||||
chat = waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
|
||||
|
||||
compacted, err := server.CompactChat(ctx, chat)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, database.ChatStatusRunning, compacted.Status)
|
||||
|
||||
chat = waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
|
||||
require.False(t, chat.LastError.Valid)
|
||||
require.False(t, chat.CompactionRequestedAt.Valid)
|
||||
|
||||
mu.Lock()
|
||||
toolCounts := slices.Clone(compactionToolCounts)
|
||||
mu.Unlock()
|
||||
require.Len(t, toolCounts, 2, "one rejected request with tools, one tool-less retry")
|
||||
require.Positive(t, toolCounts[0], "the first summary request carries the turn's tool definitions")
|
||||
require.Zero(t, toolCounts[1], "the retry drops the tool definitions")
|
||||
|
||||
messages := chatMessages(ctx, t, db, chat.ID)
|
||||
promptMessages, err := db.GetChatMessagesForPromptByChatID(ctx, chat.ID)
|
||||
require.NoError(t, err)
|
||||
compressed := compressedChatSummarizedMessages(t, append(promptMessages, messages...))
|
||||
require.Len(t, compressed.results, 1)
|
||||
resultPart := singlePartOfType(t, compressed.results[0], codersdk.ChatMessagePartTypeToolResult)
|
||||
var result map[string]any
|
||||
require.NoError(t, json.Unmarshal(resultPart.Result, &result))
|
||||
require.Equal(t, compactionSummary, result["summary"])
|
||||
})
|
||||
|
||||
t.Run("busy chat rejects manual compaction", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ package chatloop
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -453,6 +455,15 @@ func generateCompactionSummary(
|
||||
call.Prompt = summaryPrompt
|
||||
call.Tools = options.ToolDefinitions
|
||||
response, err := model.Generate(summaryCtx, call)
|
||||
if err != nil && len(call.Tools) > 0 && isContextTooLargeError(err) {
|
||||
// Tool definitions keep the summary request on the parent turn's
|
||||
// cacheable prefix, but they also make it larger than the turn
|
||||
// that just overflowed. Compaction is the recovery path for an
|
||||
// over-limit conversation, so fall back to the tool-less request,
|
||||
// which still fits whenever the history alone does.
|
||||
call.Tools = nil
|
||||
response, err = model.Generate(summaryCtx, call)
|
||||
}
|
||||
if err != nil {
|
||||
return "", xerrors.Errorf("generate summary text: %w", err)
|
||||
}
|
||||
@@ -471,3 +482,37 @@ func generateCompactionSummary(
|
||||
}
|
||||
return strings.TrimSpace(strings.Join(parts, " ")), nil
|
||||
}
|
||||
|
||||
// contextTooLargePhrases are context-window rejections fantasy does not
|
||||
// parse, such as the OpenAI Responses API "Your input exceeds the context
|
||||
// window of this model." and Anthropic-family "too long" rejections
|
||||
// without token counts. The response body is included so the OpenAI
|
||||
// error code "context_length_exceeded" matches regardless of wording.
|
||||
var contextTooLargePhrases = []string{
|
||||
"context window",
|
||||
"context length",
|
||||
"context_length",
|
||||
"too long",
|
||||
}
|
||||
|
||||
// isContextTooLargeError reports whether a provider rejected a request
|
||||
// because the prompt exceeded the model's context window.
|
||||
func isContextTooLargeError(err error) bool {
|
||||
providerErr, ok := errors.AsType[*fantasy.ProviderError](err)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if providerErr.IsContextTooLarge() {
|
||||
return true
|
||||
}
|
||||
if providerErr.StatusCode != http.StatusBadRequest {
|
||||
return false
|
||||
}
|
||||
text := strings.ToLower(providerErr.Error() + " " + string(providerErr.ResponseBody))
|
||||
for _, phrase := range contextTooLargePhrases {
|
||||
if strings.Contains(text, phrase) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package chatloop
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -320,6 +321,198 @@ func TestGenerateCompactionSummaryUsesToolDefinitions(t *testing.T) {
|
||||
require.Equal(t, originalMessages, messages)
|
||||
}
|
||||
|
||||
func TestGenerateCompactionSummaryRetriesWithoutToolsWhenContextTooLarge(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
toolDefinitions := []fantasy.Tool{
|
||||
fantasy.FunctionTool{Name: "read_file", InputSchema: map[string]any{"type": "object"}},
|
||||
}
|
||||
toolChoice := fantasy.ToolChoiceNone
|
||||
var calls []fantasy.Call
|
||||
model := &chattest.FakeModel{
|
||||
ProviderName: "fake",
|
||||
ModelName: "fake-model",
|
||||
GenerateFn: func(_ context.Context, call fantasy.Call) (*fantasy.Response, error) {
|
||||
calls = append(calls, call)
|
||||
if len(call.Tools) > 0 {
|
||||
return nil, &fantasy.ProviderError{
|
||||
Title: "bad request",
|
||||
Message: "Your input exceeds the context window of this model. Please adjust your input and try again.",
|
||||
StatusCode: http.StatusBadRequest,
|
||||
}
|
||||
}
|
||||
return &fantasy.Response{Content: []fantasy.Content{fantasy.TextContent{Text: "summary"}}}, nil
|
||||
},
|
||||
}
|
||||
|
||||
summary, err := generateCompactionSummary(context.Background(), model,
|
||||
[]fantasy.Message{textMessage(fantasy.MessageRoleUser, "hello")},
|
||||
CompactionOptions{
|
||||
SummaryPrompt: "summarize",
|
||||
SummaryCall: fantasy.Call{ToolChoice: &toolChoice},
|
||||
ToolDefinitions: toolDefinitions,
|
||||
},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "summary", summary)
|
||||
require.Len(t, calls, 2)
|
||||
require.Equal(t, toolDefinitions, calls[0].Tools)
|
||||
require.Nil(t, calls[1].Tools)
|
||||
require.Equal(t, calls[0].Prompt, calls[1].Prompt)
|
||||
require.Equal(t, calls[0].ToolChoice, calls[1].ToolChoice)
|
||||
}
|
||||
|
||||
func TestGenerateCompactionSummaryDoesNotRetryOtherFailures(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
contextTooLarge := &fantasy.ProviderError{
|
||||
Message: "Your input exceeds the context window of this model.",
|
||||
StatusCode: http.StatusBadRequest,
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
toolDefinitions []fantasy.Tool
|
||||
err error
|
||||
}{
|
||||
{
|
||||
name: "unrelated bad request",
|
||||
toolDefinitions: []fantasy.Tool{fantasy.FunctionTool{Name: "read_file"}},
|
||||
err: &fantasy.ProviderError{
|
||||
Message: "Unsupported parameter: 'temperature' is not supported with this model.",
|
||||
StatusCode: http.StatusBadRequest,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "server error",
|
||||
toolDefinitions: []fantasy.Tool{fantasy.FunctionTool{Name: "read_file"}},
|
||||
err: &fantasy.ProviderError{
|
||||
Message: "context window service unavailable",
|
||||
StatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "non-provider error",
|
||||
toolDefinitions: []fantasy.Tool{fantasy.FunctionTool{Name: "read_file"}},
|
||||
err: xerrors.New("context window"),
|
||||
},
|
||||
{
|
||||
name: "no tools to drop",
|
||||
err: contextTooLarge,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
calls := 0
|
||||
model := &chattest.FakeModel{
|
||||
ProviderName: "fake",
|
||||
ModelName: "fake-model",
|
||||
GenerateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) {
|
||||
calls++
|
||||
return nil, tc.err
|
||||
},
|
||||
}
|
||||
|
||||
_, err := generateCompactionSummary(context.Background(), model,
|
||||
[]fantasy.Message{textMessage(fantasy.MessageRoleUser, "hello")},
|
||||
CompactionOptions{
|
||||
SummaryPrompt: "summarize",
|
||||
ToolDefinitions: tc.toolDefinitions,
|
||||
},
|
||||
)
|
||||
require.ErrorIs(t, err, tc.err)
|
||||
require.Equal(t, 1, calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsContextTooLargeError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "parsed by fantasy",
|
||||
err: &fantasy.ProviderError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
ContextTooLargeErr: true,
|
||||
ContextMaxTokens: 16385,
|
||||
ContextUsedTokens: 20000,
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "openai responses wording",
|
||||
err: &fantasy.ProviderError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Message: "Your input exceeds the context window of this model. Please adjust your input and try again.",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "anthropic wording",
|
||||
err: &fantasy.ProviderError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Message: "prompt is too long: 213462 tokens > 200000 maximum",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "openai error code in response body",
|
||||
err: &fantasy.ProviderError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Message: "Request too large for this model.",
|
||||
ResponseBody: []byte(`{"error":{"message":"Request too large for this model.","type":"invalid_request_error","code":"context_length_exceeded"}}`),
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "wrapped provider error",
|
||||
err: xerrors.Errorf("generate summary text: %w", &fantasy.ProviderError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Message: "This model's maximum context length is 16385 tokens.",
|
||||
}),
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "unrelated bad request",
|
||||
err: &fantasy.ProviderError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Message: "Invalid schema for function 'read_file'.",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "matching wording on a non bad-request status",
|
||||
err: &fantasy.ProviderError{
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
Message: "context window tokens per minute exceeded",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "non-provider error",
|
||||
err: xerrors.New("prompt is too long"),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "nil",
|
||||
err: nil,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
require.Equal(t, tc.want, isContextTooLargeError(tc.err))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCompactionSummary_UsesCallerContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
Reference in new issue
Block a user