feat(coderd/x/chatd): trigger organization compaction at 80% of the override window

The organization override trigger no longer reads the override model's compression threshold. It fires at a fixed 80% of the override model's context window, leaving room for the summary prompt and the summary. An override with a threshold of 100 is no longer treated as absent; only an unknown window falls back to the chat model. The still-over-limit copy names the 80% window instead of an override threshold.
This commit is contained in:
Michael Suchacz committed 2026-10-09 16:36:49 +00:00
1 parent ec718a6b8c
commit 171d5f568e
5 files changed
+57 -203

No files matched your search

+30 -178
View File
@@ -6910,7 +6910,7 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
})
}
t.Run("compaction triggers at the override model's own threshold", func(t *testing.T) {
t.Run("compaction triggers at 80% of the override window, ignoring its own threshold", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
@@ -6937,11 +6937,12 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
}
})
user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL)
// Usage 80: the override trigger fires at 60% of 100 = 60 tokens,
// while the chat threshold on the stricter limit would be 90% of 100 = 90.
// Usage 80 reaches 80% of the override's 100-token window but not the
// chat trigger at 90% of 1,000. The override's own threshold of 100
// would disable it if it were read.
model = updateChatModelCompressionThreshold(t, db, model, 1_000, 90)
overrideModel := seedOverrideModel(ctx, t, db, model, overrideModelName, "high", 100)
updateChatModelCompressionThreshold(t, db, overrideModel, 100, 60)
updateChatModelCompressionThreshold(t, db, overrideModel, 100, 100)
ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID)
ctrl := gomock.NewController(t)
@@ -7018,11 +7019,10 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
}
})
user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL)
// The chat trigger binds (10% of 1,000,000 = 100,000 < 90% of
// 200,000 = 180,000), but the override model summarizes.
// The chat trigger binds (10% of 1,000,000 = 100,000 < 80% of
// 200,000 = 160,000), but the override model summarizes.
model = updateChatModelCompressionThreshold(t, db, model, 1_000_000, 10)
overrideModel := seedOverrideModel(ctx, t, db, model, overrideModelName, "high", overrideContextLimit)
updateChatModelCompressionThreshold(t, db, overrideModel, overrideContextLimit, 90)
seedOverrideModel(ctx, t, db, model, overrideModelName, "high", overrideContextLimit)
ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID)
ctrl := gomock.NewController(t)
@@ -7064,7 +7064,7 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
t.Parallel()
const (
promptTokens = 170_000
promptTokens = 140_000
overrideContextLimit = 200_000
)
ctx := testutil.Context(t, testutil.WaitLong)
@@ -7086,11 +7086,10 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
}, "assistant answer")...)
})
user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL)
// Usage stays below both triggers (18% of 1,000,000 = 180,000 and
// 95% of 200,000 = 190,000); the chat trigger binds.
model = updateChatModelCompressionThreshold(t, db, model, 1_000_000, 18)
overrideModel := seedOverrideModel(ctx, t, db, model, overrideModelName, "high", overrideContextLimit)
updateChatModelCompressionThreshold(t, db, overrideModel, overrideContextLimit, 95)
// Usage stays below both triggers (15% of 1,000,000 = 150,000 and
// 80% of 200,000 = 160,000); the chat trigger binds.
model = updateChatModelCompressionThreshold(t, db, model, 1_000_000, 15)
seedOverrideModel(ctx, t, db, model, overrideModelName, "high", overrideContextLimit)
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath()))
@@ -7111,10 +7110,11 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
const (
chatOverLimitMessage = "Conversation compaction could not reduce the history below your compaction threshold. Raise the compaction threshold in settings, or start a new conversation."
overrideOverLimitMessage = "Conversation compaction could not reduce the history below the organization override's compaction threshold. Start a new conversation, or ask an administrator to raise the organization override's compaction threshold or choose an override model with a larger context window."
bothOverLimitMessage = "Conversation compaction could not reduce the history below your compaction threshold or the organization override's compaction threshold. Start a new conversation, or raise your compaction threshold in settings and ask an administrator to raise the organization override's compaction threshold or choose an override model with a larger context window."
overrideOverLimitMessage = "Conversation compaction could not reduce the history below 80% of the organization override's context window. Start a new conversation, or ask an administrator to choose an override model with a larger context window."
bothOverLimitMessage = "Conversation compaction could not reduce the history below your compaction threshold or 80% of the organization override's context window. Start a new conversation, or raise your compaction threshold in settings and ask an administrator to choose an override model with a larger context window."
)
// Usage stays at 80 tokens and both thresholds are 70%.
// Usage stays at 80 tokens, the chat threshold is 70%, and the override
// trigger is 80% of its window.
for _, tc := range []struct {
name string
chatContextLimit int64
@@ -7122,31 +7122,31 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
wantMessage string
}{
{
// Override point 70 binds; chat point 700.
// Override point 72 binds; chat point 700.
name: "override trigger binds, usage below chat point",
chatContextLimit: 1_000,
overrideContextLimit: 100,
overrideContextLimit: 90,
wantMessage: overrideOverLimitMessage,
},
{
// Override point 70 binds; chat point 77.
// Override point 72 binds; chat point 77.
name: "override trigger binds, usage reaches chat point",
chatContextLimit: 110,
overrideContextLimit: 100,
overrideContextLimit: 90,
wantMessage: bothOverLimitMessage,
},
{
// Chat point 70 binds; override point 700.
// Chat point 70 binds; override point 800.
name: "chat trigger binds, usage below override point",
chatContextLimit: 100,
overrideContextLimit: 1_000,
wantMessage: chatOverLimitMessage,
},
{
// Chat point 70 binds; override point 77.
// Chat point 70 binds; override point 80.
name: "chat trigger binds, usage reaches override point",
chatContextLimit: 100,
overrideContextLimit: 110,
overrideContextLimit: 100,
wantMessage: bothOverLimitMessage,
},
} {
@@ -7175,8 +7175,7 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
})
user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL)
model = updateChatModelCompressionThreshold(t, db, model, tc.chatContextLimit, thresholdPercent)
overrideModel := seedOverrideModel(ctx, t, db, model, overrideModelName, "high", tc.overrideContextLimit)
updateChatModelCompressionThreshold(t, db, overrideModel, tc.overrideContextLimit, thresholdPercent)
seedOverrideModel(ctx, t, db, model, overrideModelName, "high", tc.overrideContextLimit)
ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID)
ctrl := gomock.NewController(t)
@@ -7225,7 +7224,7 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
})
}
t.Run("lowering the override threshold compacts a chat summarized under the old trigger", func(t *testing.T) {
t.Run("a new lower override trigger compacts a chat summarized under the old trigger", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
@@ -7261,8 +7260,6 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
})
user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL)
model = updateChatModelCompressionThreshold(t, db, model, 100, thresholdPercent)
overrideModel := seedOverrideModel(ctx, t, db, model, overrideModelName, "high", 100)
overrideModel = updateChatModelCompressionThreshold(t, db, overrideModel, 100, 100)
ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID)
ctrl := gomock.NewController(t)
@@ -7295,9 +7292,9 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
require.False(t, chat.LastError.Valid)
require.Equal(t, int32(1), chatSummaries.Load())
// The post-compaction usage of 50 is under the old 70% trigger but
// over the new binding override trigger at 40%.
updateChatModelCompressionThreshold(t, db, overrideModel, 100, 40)
// The post-compaction usage of 50 is under the chat trigger at 70 but
// reaches the new binding override trigger at 80% of 60 = 48.
seedOverrideModel(ctx, t, db, model, overrideModelName, "high", 60)
_, err = server.SendMessage(ctx, chatd.SendMessageOptions{
ChatID: chat.ID,
CreatedBy: user.ID,
@@ -7349,8 +7346,7 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
ThresholdPercent: 100,
})
require.NoError(t, err)
overrideModel := seedOverrideModel(ctx, t, db, model, overrideModelName, "high", 100)
updateChatModelCompressionThreshold(t, db, overrideModel, 100, 70)
seedOverrideModel(ctx, t, db, model, overrideModelName, "high", 100)
ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID)
ctrl := gomock.NewController(t)
@@ -7388,150 +7384,6 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
require.Len(t, compressed.summaries, 1)
requireTextPart(t, messages[len(messages)-1], "continued after compaction")
})
t.Run("override model with a higher own point does not compact early", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
db, ps := dbtestutil.NewDB(t)
var streamCount atomic.Int32
var compactionRequests atomic.Int32
anthropicURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse {
body := anthropicRequestBody(t, *req)
if strings.Contains(body, "You are performing a context compaction") {
compactionRequests.Add(1)
return anthropicCompactionResponse(t, req, compactionSummary)
}
if !req.Stream {
return chattest.AnthropicNonStreamingResponse("title")
}
switch streamCount.Add(1) {
case 1:
return highUsageReadFileResponse("/tmp/a.txt")
default:
return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunksWithCacheUsage(chattest.AnthropicUsage{
InputTokens: 20,
OutputTokens: 5,
}, "continued without compaction")...)
}
})
user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL)
model = updateChatModelCompressionThreshold(t, db, model, 1_000, thresholdPercent)
overrideModel := seedOverrideModel(ctx, t, db, model, overrideModelName, "high", 100)
updateChatModelCompressionThreshold(t, db, overrideModel, 100, 95)
ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID)
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
setupToolExecutionAgentConn(t, mockConn)
mockConn.EXPECT().ReadFileLines(gomock.Any(), "/tmp/a.txt", int64(1), int64(0), gomock.Any()).
Return(workspacesdk.ReadFileLinesResponse{Success: true, FileSize: 12, TotalLines: 1, LinesRead: 1, Content: "1\tpackage main"}, nil).
Times(1)
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath()))
cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) {
require.Equal(t, dbAgent.ID, agentID)
return mockConn, func() {}, nil
}
})
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
OrganizationID: org.ID,
OwnerID: user.ID,
WorkspaceID: uuid.NullUUID{UUID: ws.ID, Valid: true},
AgentID: uuid.NullUUID{UUID: dbAgent.ID, Valid: true},
Title: "compaction-override-higher-point",
ModelConfigID: model.ID,
InitialUserContent: []codersdk.ChatMessagePart{
codersdk.ChatMessageText("read the file and continue"),
},
})
require.NoError(t, err)
waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
messages := chatMessages(ctx, t, db, chat.ID)
promptMessages, err := db.GetChatMessagesForPromptByChatID(ctx, chat.ID)
require.NoError(t, err)
compressed := compressedChatSummarizedMessages(t, append(promptMessages, messages...))
require.Empty(t, compressed.summaries)
require.Zero(t, compactionRequests.Load())
requireTextPart(t, messages[len(messages)-1], "continued without compaction")
})
t.Run("override model with compaction disabled is not used", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
db, ps := dbtestutil.NewDB(t)
var streamCount atomic.Int32
anthropicURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse {
body := anthropicRequestBody(t, *req)
if strings.Contains(body, "You are performing a context compaction") {
require.Equal(t, chatModelName, req.Model)
return anthropicCompactionResponse(t, req, compactionSummary)
}
if !req.Stream {
return chattest.AnthropicNonStreamingResponse("title")
}
switch streamCount.Add(1) {
case 1:
return highUsageReadFileResponse("/tmp/a.txt")
default:
require.Contains(t, body, compactionSummary)
return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunksWithCacheUsage(chattest.AnthropicUsage{
InputTokens: 20,
OutputTokens: 5,
}, "continued after compaction")...)
}
})
user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL)
model = updateChatModelCompressionThreshold(t, db, model, 100, thresholdPercent)
overrideModel := seedOverrideModel(ctx, t, db, model, overrideModelName, "high", 100)
updateChatModelCompressionThreshold(t, db, overrideModel, 100, 100)
ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID)
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
setupToolExecutionAgentConn(t, mockConn)
mockConn.EXPECT().ReadFileLines(gomock.Any(), "/tmp/a.txt", int64(1), int64(0), gomock.Any()).
Return(workspacesdk.ReadFileLinesResponse{Success: true, FileSize: 12, TotalLines: 1, LinesRead: 1, Content: "1\tpackage main"}, nil).
Times(1)
reg := prometheus.NewRegistry()
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
cfg.PrometheusRegistry = reg
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath()))
cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) {
require.Equal(t, dbAgent.ID, agentID)
return mockConn, func() {}, nil
}
})
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
OrganizationID: org.ID,
OwnerID: user.ID,
WorkspaceID: uuid.NullUUID{UUID: ws.ID, Valid: true},
AgentID: uuid.NullUUID{UUID: dbAgent.ID, Valid: true},
Title: "compaction-override-disabled",
ModelConfigID: model.ID,
InitialUserContent: []codersdk.ChatMessagePart{
codersdk.ChatMessageText("read the file and continue"),
},
})
require.NoError(t, err)
waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
messages := chatMessages(ctx, t, db, chat.ID)
promptMessages, err := db.GetChatMessagesForPromptByChatID(ctx, chat.ID)
require.NoError(t, err)
compressed := compressedChatSummarizedMessages(t, append(promptMessages, messages...))
require.Len(t, compressed.summaries, 1)
requireTextPart(t, messages[len(messages)-1], "continued after compaction")
requireChatdMetricCounter(t, reg, "coderd_chatd_compaction_total", 1, map[string]string{
"provider": "anthropic",
"model": chatModelName,
"result": "success",
})
})
}
func TestActiveServer_AnthropicModelReasoningEffort(t *testing.T) {
+3 -2
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
@@ -159,9 +160,9 @@ func compactionStillOverLimitError(source compactionTriggerSource, otherOverLimi
message := "Conversation compaction could not reduce the history below your compaction threshold. Raise the compaction threshold in settings, or start a new conversation."
switch {
case otherOverLimit:
message = "Conversation compaction could not reduce the history below your compaction threshold or the organization override's compaction threshold. Start a new conversation, or raise your compaction threshold in settings and ask an administrator to raise the organization override's compaction threshold or choose an override model with a larger context window."
message = fmt.Sprintf("Conversation compaction could not reduce the history below your compaction threshold or %d%% of the organization override's context window. Start a new conversation, or raise your compaction threshold in settings and ask an administrator to choose an override model with a larger context window.", compactionOverrideWindowPercent)
case source == compactionTriggerSourceOrganization:
message = "Conversation compaction could not reduce the history below the organization override's compaction threshold. Start a new conversation, or ask an administrator to raise the organization override's compaction threshold or choose an override model with a larger context window."
message = fmt.Sprintf("Conversation compaction could not reduce the history below %d%% of the organization override's context window. Start a new conversation, or ask an administrator to choose an override model with a larger context window.", compactionOverrideWindowPercent)
}
return chaterror.WithClassification(
xerrors.Errorf("%s trigger at %d%% of %d tokens: %w", source, thresholdPercent, contextLimit, errCompactionStillOverLimit),
+2 -2
View File
@@ -83,8 +83,8 @@ func TestCompactionStillOverLimitError(t *testing.T) {
const (
chatMessage = "Conversation compaction could not reduce the history below your compaction threshold. Raise the compaction threshold in settings, or start a new conversation."
overrideMessage = "Conversation compaction could not reduce the history below the organization override's compaction threshold. Start a new conversation, or ask an administrator to raise the organization override's compaction threshold or choose an override model with a larger context window."
bothMessage = "Conversation compaction could not reduce the history below your compaction threshold or the organization override's compaction threshold. Start a new conversation, or raise your compaction threshold in settings and ask an administrator to raise the organization override's compaction threshold or choose an override model with a larger context window."
overrideMessage = "Conversation compaction could not reduce the history below 80% of the organization override's context window. Start a new conversation, or ask an administrator to choose an override model with a larger context window."
bothMessage = "Conversation compaction could not reduce the history below your compaction threshold or 80% of the organization override's context window. Start a new conversation, or raise your compaction threshold in settings and ask an administrator to choose an override model with a larger context window."
)
require.Equal(t, chatMessage, chaterror.Classify(compactionStillOverLimitError(compactionTriggerSourceChat, false, 70, 100)).Message)
require.Equal(t, overrideMessage, chaterror.Classify(compactionStillOverLimitError(compactionTriggerSourceOrganization, false, 70, 100)).Message)
+6 -2
View File
@@ -847,10 +847,10 @@ func (server *Server) prepareGeneration(
var otherTrigger compactionTrigger
if resolvedCompactionOverride.Set {
overrideTrigger := compactionTrigger{
thresholdPercent: resolvedCompactionOverride.Config.CompressionThreshold,
thresholdPercent: compactionOverrideWindowPercent,
contextLimit: resolvedCompactionOverride.Config.ContextLimit,
}
// With its own trigger disabled, nothing bounds the history by the
// With an unknown window, nothing bounds the history by the
// override's window, so the chat model summarizes instead.
if overrideTrigger.enabled() {
compactionOverride = &resolvedCompactionOverride
@@ -959,6 +959,10 @@ func latestPromptUsage(messages []database.ChatMessage) fantasy.Usage {
return fantasy.Usage{}
}
// compactionOverrideWindowPercent leaves room in the override's window for
// the summary prompt and the summary.
const compactionOverrideWindowPercent = int32(80)
type compactionTrigger struct {
thresholdPercent int32
contextLimit int64
@@ -911,6 +911,9 @@ func TestBindingCompactionTriggerSource(t *testing.T) {
trigger := func(threshold int32, limit int64) compactionTrigger {
return compactionTrigger{thresholdPercent: threshold, contextLimit: limit}
}
override := func(limit int64) compactionTrigger {
return trigger(compactionOverrideWindowPercent, limit)
}
cases := []struct {
name string
@@ -921,56 +924,50 @@ func TestBindingCompactionTriggerSource(t *testing.T) {
{
name: "lower override point wins",
chat: trigger(70, 200_000),
override: trigger(70, 32_000),
override: override(32_000),
want: compactionTriggerSourceOrganization,
},
{
name: "lower chat point wins over higher override point",
chat: trigger(70, 100_000),
override: trigger(95, 80_000),
chat: trigger(70, 80_000),
override: override(80_000),
want: compactionTriggerSourceChat,
},
{
name: "tie prefers the chat trigger",
chat: trigger(70, 100_000),
override: trigger(70, 100_000),
chat: trigger(80, 100_000),
override: override(100_000),
want: compactionTriggerSourceChat,
},
{
name: "chat trigger disabled by threshold 100 yields override",
chat: trigger(100, 200_000),
override: trigger(70, 32_000),
override: override(32_000),
want: compactionTriggerSourceOrganization,
},
{
name: "chat trigger disabled by zero limit yields override",
chat: trigger(70, 0),
override: trigger(70, 32_000),
override: override(32_000),
want: compactionTriggerSourceOrganization,
},
{
name: "override disabled by threshold 100 yields chat",
chat: trigger(70, 200_000),
override: trigger(100, 32_000),
want: compactionTriggerSourceChat,
},
{
name: "override disabled by zero limit yields chat",
chat: trigger(70, 200_000),
override: trigger(70, 0),
override: override(0),
want: compactionTriggerSourceChat,
},
{
name: "both disabled yields chat",
chat: trigger(100, 200_000),
override: trigger(100, 32_000),
override: override(0),
want: compactionTriggerSourceChat,
},
{
name: "threshold zero fires immediately and wins",
chat: trigger(70, 200_000),
override: trigger(0, 32_000),
want: compactionTriggerSourceOrganization,
name: "chat threshold zero fires immediately and wins",
chat: trigger(0, 200_000),
override: override(32_000),
want: compactionTriggerSourceChat,
},
}
for _, tc := range cases {