On Claude Opus 5.5, chat title generation and the turn-status label each made four upstream requests, three of them rejected with a 400. The temperature retry in `generateQuickgenObject` called `generateObject` again, which restarted from the forced tool call the model had already rejected. Each turn left about six `bad_request` rows in `aibridge_interceptions` and added 1.5 to 3.5 seconds of latency. This replaces the per-call fallbacks with `quickgenParamModel`, a wrapper applied in `resolveModelCall`. It falls back to plain JSON text when the model rejects the forced `tool_choice`, drops `temperature` when the model rejects it, and records each rejection in a replica-wide `sync.Map` keyed by AI provider ID, model, and parameter, following the existing `thinkingDropBlock` map. Entries expire after 6 hours so a provider that starts accepting a parameter again is picked up without a restart. Only quickgen calls `GenerateObject`, so chat turns are unaffected. | Opus 5.5 | Before | First job per replica (and after expiry) | Later jobs | |---|---|---|---| | Title / status label | 4 (3 rejected) | 3 (2 rejected) | 1 | | Summary | 2 (1 rejected) | 2 (1 rejected) | 1 | Rejections are still detected by matching `tool_choice` or `temperature` in a 400 body, as before. A false match now persists for the TTL on that replica rather than one job; the worst case is text mode or no temperature for that model. Closes [CODAGT-1411](https://linear.app/codercom/issue/CODAGT-1411/quickgen-side-jobs-make-3-rejected-requests-per-run-on-claude-opus-55) <details> <summary>Decision log</summary> - Opus 5.5 reports one rejected parameter per 400, so a job that knows nothing yet needs 3 requests; the issue's "2 requests" target for the minimal fix isn't reachable. - A per-job flag (first iteration) only got jobs to 3 requests. Sharing rejections across jobs gets steady state to 1 without a hardcoded model list, which wouldn't match gateway aliases such as Bedrock IDs anyway. - Per-replica memory over a database column or admin setting: no migration, no request-path writes, self-healing via TTL. Each replica pays the learning cost once per model per TTL window. - Keyed by provider ID plus model so the same model name behind different providers is learned independently. </details> > Generated by Coder Agents on behalf of @deansheather.
1649 lines
55 KiB
Go
1649 lines
55 KiB
Go
package chatd
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"charm.land/fantasy"
|
|
"charm.land/fantasy/object"
|
|
fantasyanthropic "charm.land/fantasy/providers/anthropic"
|
|
fantasyopenai "charm.land/fantasy/providers/openai"
|
|
fantasyopenaicompat "charm.land/fantasy/providers/openaicompat"
|
|
"github.com/google/uuid"
|
|
"github.com/sqlc-dev/pqtype"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/mock/gomock"
|
|
"golang.org/x/xerrors"
|
|
|
|
"cdr.dev/slog/v3/sloggers/slogtest"
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/database/dbgen"
|
|
"github.com/coder/coder/v2/coderd/database/dbmock"
|
|
"github.com/coder/coder/v2/coderd/database/dbtestutil"
|
|
dbpubsub "github.com/coder/coder/v2/coderd/database/pubsub"
|
|
"github.com/coder/coder/v2/coderd/database/pubsub/psmock"
|
|
coderdpubsub "github.com/coder/coder/v2/coderd/pubsub"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatprompt"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatstate"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
"github.com/coder/coder/v2/testutil"
|
|
"github.com/coder/quartz"
|
|
)
|
|
|
|
func Test_extractManualTitleTurns(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
pasteFileID := uuid.New()
|
|
|
|
tests := []struct {
|
|
name string
|
|
messages []database.ChatMessage
|
|
pasteText map[uuid.UUID]string
|
|
want []manualTitleTurn
|
|
}{
|
|
{
|
|
name: "paste only user message resolves via paste text",
|
|
messages: []database.ChatMessage{
|
|
mustChatMessage(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessageFile(pasteFileID, "text/plain", "pasted-text-2026-01-02-03-04-05.txt"),
|
|
),
|
|
},
|
|
pasteText: map[uuid.UUID]string{pasteFileID: "pasted panic output"},
|
|
want: []manualTitleTurn{{role: "user", text: "pasted panic output"}},
|
|
},
|
|
{
|
|
name: "filters to visible user and assistant text turns",
|
|
messages: []database.ChatMessage{
|
|
mustChatMessage(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeText, Text: " review quickgen helpers "},
|
|
),
|
|
mustChatMessage(t, database.ChatMessageRoleAssistant, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeText, Text: " drafted a plan "},
|
|
),
|
|
mustChatMessage(t, database.ChatMessageRoleSystem, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeText, Text: "system prompt"},
|
|
),
|
|
mustChatMessage(t, database.ChatMessageRoleTool, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeText, Text: "tool output"},
|
|
),
|
|
mustChatMessage(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityModel,
|
|
codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeText, Text: "hidden model note"},
|
|
),
|
|
mustChatMessage(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeText, Text: " "},
|
|
),
|
|
mustChatMessage(t, database.ChatMessageRoleAssistant, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeReasoning, Text: "reasoning only"},
|
|
),
|
|
mustChatMessage(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeFile, MediaType: "text/plain"},
|
|
),
|
|
},
|
|
want: []manualTitleTurn{
|
|
{role: "user", text: "review quickgen helpers"},
|
|
{role: "assistant", text: "drafted a plan"},
|
|
},
|
|
},
|
|
{
|
|
name: "reuses text extraction for multi-part content",
|
|
messages: []database.ChatMessage{
|
|
mustChatMessage(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeText, Text: "first chunk"},
|
|
codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeReasoning, Text: "skip me"},
|
|
codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeText, Text: " second chunk "},
|
|
),
|
|
},
|
|
want: []manualTitleTurn{{role: "user", text: "first chunk second chunk"}},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got := extractManualTitleTurns(tt.messages, tt.pasteText)
|
|
require.Equal(t, tt.want, got)
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_selectManualTitleTurnIndexes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
turns []manualTitleTurn
|
|
want []int
|
|
}{
|
|
{
|
|
name: "single user turn",
|
|
turns: []manualTitleTurn{
|
|
{role: "user", text: "one"},
|
|
},
|
|
want: []int{0},
|
|
},
|
|
{
|
|
name: "first user plus trailing window",
|
|
turns: []manualTitleTurn{
|
|
{role: "user", text: "one"},
|
|
{role: "assistant", text: "two"},
|
|
{role: "user", text: "three"},
|
|
{role: "assistant", text: "four"},
|
|
{role: "user", text: "five"},
|
|
},
|
|
want: []int{0, 2, 3, 4},
|
|
},
|
|
{
|
|
name: "two turns returns both",
|
|
turns: []manualTitleTurn{
|
|
{role: "user", text: "one"},
|
|
{role: "assistant", text: "two"},
|
|
},
|
|
want: []int{0, 1},
|
|
},
|
|
{
|
|
name: "prepends first user when before trailing window",
|
|
turns: []manualTitleTurn{
|
|
{role: "assistant", text: "intro"},
|
|
{role: "assistant", text: "setup"},
|
|
{role: "user", text: "goal"},
|
|
{role: "assistant", text: "a"},
|
|
{role: "assistant", text: "b"},
|
|
{role: "assistant", text: "c"},
|
|
},
|
|
want: []int{2, 3, 4, 5},
|
|
},
|
|
{
|
|
name: "ten plus turns keeps first user and last three",
|
|
turns: []manualTitleTurn{
|
|
{role: "assistant", text: "0"},
|
|
{role: "assistant", text: "1"},
|
|
{role: "user", text: "2"},
|
|
{role: "assistant", text: "3"},
|
|
{role: "assistant", text: "4"},
|
|
{role: "assistant", text: "5"},
|
|
{role: "assistant", text: "6"},
|
|
{role: "assistant", text: "7"},
|
|
{role: "assistant", text: "8"},
|
|
{role: "user", text: "9"},
|
|
{role: "assistant", text: "10"},
|
|
{role: "user", text: "11"},
|
|
},
|
|
want: []int{2, 9, 10, 11},
|
|
},
|
|
{
|
|
name: "no user turns",
|
|
turns: []manualTitleTurn{
|
|
{role: "assistant", text: "one"},
|
|
{role: "assistant", text: "two"},
|
|
},
|
|
want: nil,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got := selectManualTitleTurnIndexes(tt.turns)
|
|
require.Equal(t, tt.want, got)
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_buildManualTitleContext(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
longConversationText := strings.Repeat("a", 3500)
|
|
longLatestUserText := strings.Repeat("z", 1200)
|
|
|
|
tests := []struct {
|
|
name string
|
|
turns []manualTitleTurn
|
|
selected []int
|
|
wantConversation string
|
|
wantConversationEmpty bool
|
|
wantConversationHasGap bool
|
|
wantConversationRunes int
|
|
wantLatestUser string
|
|
wantLatestUserRunes int
|
|
wantLatestUserContains string
|
|
wantLatestUserNotEmpty bool
|
|
}{
|
|
{
|
|
name: "adds gap marker when selected turns skip earlier context",
|
|
turns: []manualTitleTurn{
|
|
{role: "user", text: "open pull request"},
|
|
{role: "assistant", text: "checked CI"},
|
|
{role: "user", text: "review logs"},
|
|
{role: "assistant", text: "found flaky test"},
|
|
{role: "user", text: "update chat title"},
|
|
},
|
|
selected: []int{0, 3, 4},
|
|
wantConversationHasGap: true,
|
|
wantLatestUser: "update chat title",
|
|
},
|
|
{
|
|
name: "omits gap marker for contiguous selection",
|
|
turns: []manualTitleTurn{
|
|
{role: "user", text: "open pull request"},
|
|
{role: "assistant", text: "checked CI"},
|
|
{role: "user", text: "update chat title"},
|
|
},
|
|
selected: []int{0, 1, 2},
|
|
wantConversation: "[user]: open pull request\n[assistant]: checked CI\n[user]: update chat title",
|
|
wantConversationHasGap: false,
|
|
wantLatestUser: "update chat title",
|
|
},
|
|
{
|
|
name: "single useful user turn returns empty conversation block",
|
|
turns: []manualTitleTurn{{role: "user", text: "rename helper"}},
|
|
selected: []int{0},
|
|
wantConversationEmpty: true,
|
|
wantLatestUser: "rename helper",
|
|
},
|
|
{
|
|
name: "truncates conversation block at six thousand runes",
|
|
turns: []manualTitleTurn{
|
|
{role: "user", text: longConversationText},
|
|
{role: "assistant", text: longConversationText},
|
|
{role: "user", text: "latest"},
|
|
},
|
|
selected: []int{0, 1, 2},
|
|
wantConversationRunes: 6000,
|
|
wantLatestUser: "latest",
|
|
},
|
|
{
|
|
name: "truncates latest user message at one thousand runes",
|
|
turns: []manualTitleTurn{
|
|
{role: "user", text: "first"},
|
|
{role: "assistant", text: "reply"},
|
|
{role: "user", text: longLatestUserText},
|
|
},
|
|
selected: []int{0, 1, 2},
|
|
wantLatestUserRunes: 1000,
|
|
wantLatestUserContains: strings.Repeat("z", 1000),
|
|
wantLatestUserNotEmpty: true,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
conversationBlock, latestUserMsg := buildManualTitleContext(tt.turns, tt.selected)
|
|
|
|
if tt.wantConversationEmpty {
|
|
require.Empty(t, conversationBlock)
|
|
}
|
|
if tt.wantConversation != "" {
|
|
require.Equal(t, tt.wantConversation, conversationBlock)
|
|
}
|
|
if tt.wantConversationHasGap {
|
|
require.Contains(t, conversationBlock, "[... 2 earlier turns omitted ...]")
|
|
} else if !tt.wantConversationEmpty {
|
|
require.NotContains(t, conversationBlock, "earlier turns omitted")
|
|
}
|
|
if tt.wantConversationRunes > 0 {
|
|
require.Len(t, []rune(conversationBlock), tt.wantConversationRunes)
|
|
}
|
|
if tt.wantLatestUser != "" {
|
|
require.Equal(t, tt.wantLatestUser, latestUserMsg)
|
|
}
|
|
if tt.wantLatestUserRunes > 0 {
|
|
require.Len(t, []rune(latestUserMsg), tt.wantLatestUserRunes)
|
|
}
|
|
if tt.wantLatestUserContains != "" {
|
|
require.Equal(t, tt.wantLatestUserContains, latestUserMsg)
|
|
}
|
|
if tt.wantLatestUserNotEmpty {
|
|
require.NotEmpty(t, latestUserMsg)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_renderManualTitlePrompt(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
longFirstUserText := strings.Repeat("b", 1501)
|
|
|
|
tests := []struct {
|
|
name string
|
|
conversationBlock string
|
|
firstUserText string
|
|
latestUserMsg string
|
|
wantConversationSample bool
|
|
wantLatestSection bool
|
|
}{
|
|
{
|
|
name: "includes conversation sample when provided",
|
|
conversationBlock: "[user]: inspect logs\n[assistant]: found flaky test",
|
|
firstUserText: "inspect logs",
|
|
latestUserMsg: "update quickgen title",
|
|
wantConversationSample: true,
|
|
wantLatestSection: true,
|
|
},
|
|
{
|
|
name: "omits optional sections when not needed",
|
|
conversationBlock: "",
|
|
firstUserText: "inspect logs",
|
|
latestUserMsg: "inspect logs",
|
|
wantConversationSample: false,
|
|
wantLatestSection: false,
|
|
},
|
|
{
|
|
name: "latest section compares trimmed text",
|
|
conversationBlock: "",
|
|
firstUserText: "inspect logs",
|
|
latestUserMsg: " inspect logs ",
|
|
wantConversationSample: false,
|
|
wantLatestSection: false,
|
|
},
|
|
{
|
|
name: "omits latest section when same message truncated",
|
|
conversationBlock: "",
|
|
firstUserText: longFirstUserText,
|
|
latestUserMsg: truncateRunes(longFirstUserText, 1000),
|
|
wantConversationSample: false,
|
|
wantLatestSection: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
prompt := renderManualTitlePrompt(tt.conversationBlock, tt.firstUserText, tt.latestUserMsg)
|
|
|
|
require.Contains(t, prompt, "Primary user objective:")
|
|
require.Contains(t, prompt, "Requirements:")
|
|
require.Contains(t, prompt, "- Keep the title to 2-8 words.")
|
|
require.Contains(t, prompt, "Do not answer the user or describe the title-writing task")
|
|
require.Contains(t, prompt, "stay close to the user's wording")
|
|
require.Contains(t, prompt, "same language as the user's messages")
|
|
|
|
if tt.wantConversationSample {
|
|
require.Contains(t, prompt, "Conversation sample:")
|
|
require.Contains(t, prompt, tt.conversationBlock)
|
|
} else {
|
|
require.NotContains(t, prompt, "Conversation sample:")
|
|
}
|
|
|
|
if tt.wantLatestSection {
|
|
require.Contains(t, prompt, "The user's most recent message:")
|
|
require.Contains(t, prompt, "Note: Weight the overall conversation arc more heavily than just the latest message.")
|
|
require.Contains(t, prompt, strings.TrimSpace(tt.latestUserMsg))
|
|
} else {
|
|
require.NotContains(t, prompt, "The user's most recent message:")
|
|
require.NotContains(t, prompt, "Weight the overall conversation arc more heavily")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_titleInput(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
pasteFileID := uuid.New()
|
|
pasteContent := "pasted stack trace with details"
|
|
pasteMessage := mustChatMessage(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessageFile(pasteFileID, "text/plain", "pasted-text-2026-01-02-03-04-05.txt"),
|
|
)
|
|
textMessage := mustChatMessage(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessageText("summarize build logs"),
|
|
)
|
|
|
|
tests := []struct {
|
|
name string
|
|
chat database.Chat
|
|
messages []database.ChatMessage
|
|
pasteText map[uuid.UUID]string
|
|
wantInput string
|
|
wantOK bool
|
|
}{
|
|
{
|
|
name: "text message with fallback title is eligible",
|
|
chat: database.Chat{Title: chatprompt.FallbackTitle("summarize build logs"), TitleSource: database.ChatTitleSourceFallback},
|
|
messages: []database.ChatMessage{textMessage},
|
|
wantInput: "summarize build logs",
|
|
wantOK: true,
|
|
},
|
|
{
|
|
name: "placeholder title of a chat created without a message is eligible",
|
|
chat: database.Chat{Title: chatprompt.DefaultChatTitle, TitleSource: database.ChatTitleSourceFallback},
|
|
messages: []database.ChatMessage{textMessage},
|
|
wantInput: "summarize build logs",
|
|
wantOK: true,
|
|
},
|
|
{
|
|
name: "paste only message with resolved paste text is eligible",
|
|
chat: database.Chat{Title: chatprompt.FallbackTitle(pasteContent), TitleSource: database.ChatTitleSourceFallback},
|
|
messages: []database.ChatMessage{pasteMessage},
|
|
pasteText: map[uuid.UUID]string{pasteFileID: pasteContent},
|
|
wantInput: pasteContent,
|
|
wantOK: true,
|
|
},
|
|
{
|
|
name: "paste only message without resolved paste text is skipped",
|
|
chat: database.Chat{Title: chatprompt.DefaultChatTitle, TitleSource: database.ChatTitleSourceFallback},
|
|
messages: []database.ChatMessage{pasteMessage},
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "user title identical to fallback text is skipped",
|
|
chat: database.Chat{Title: chatprompt.FallbackTitle(pasteContent), TitleSource: database.ChatTitleSourceUser},
|
|
messages: []database.ChatMessage{pasteMessage},
|
|
pasteText: map[uuid.UUID]string{pasteFileID: pasteContent},
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "already generated title is skipped",
|
|
chat: database.Chat{Title: "Build log summary", TitleSource: database.ChatTitleSourceGenerated},
|
|
messages: []database.ChatMessage{textMessage},
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "assistant reply disables generation",
|
|
chat: database.Chat{Title: chatprompt.FallbackTitle(pasteContent), TitleSource: database.ChatTitleSourceFallback},
|
|
messages: []database.ChatMessage{
|
|
pasteMessage,
|
|
mustChatMessage(t, database.ChatMessageRoleAssistant, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessageText("done"),
|
|
),
|
|
},
|
|
pasteText: map[uuid.UUID]string{pasteFileID: pasteContent},
|
|
wantOK: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
input, ok := titleInput(tt.chat, tt.messages, tt.pasteText)
|
|
require.Equal(t, tt.wantOK, ok)
|
|
require.Equal(t, tt.wantInput, input)
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_titlePasteText(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
pasteFileID := uuid.New()
|
|
pasteMessage := mustChatMessage(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessageFile(pasteFileID, "text/plain", "pasted-text-2026-01-02-03-04-05.txt"),
|
|
)
|
|
|
|
t.Run("skips fetch when user messages have text", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctrl := gomock.NewController(t)
|
|
// No GetChatFileDataPrefixesByIDs expectation: a fetch would
|
|
// fail the test.
|
|
db := dbmock.NewMockStore(ctrl)
|
|
|
|
pasteText, err := titlePasteText(context.Background(), db, []database.ChatMessage{
|
|
mustChatMessage(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessageText("typed text"),
|
|
codersdk.ChatMessageFile(pasteFileID, "text/plain", "pasted-text-2026-01-02-03-04-05.txt"),
|
|
),
|
|
})
|
|
require.NoError(t, err)
|
|
require.Nil(t, pasteText)
|
|
})
|
|
|
|
t.Run("resolves paste content for paste only user messages", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctrl := gomock.NewController(t)
|
|
db := dbmock.NewMockStore(ctrl)
|
|
db.EXPECT().GetChatFileDataPrefixesByIDs(gomock.Any(), database.GetChatFileDataPrefixesByIDsParams{
|
|
IDs: []uuid.UUID{pasteFileID},
|
|
PrefixBytes: chatprompt.TitlePasteBytePrefix,
|
|
}).Return([]database.GetChatFileDataPrefixesByIDsRow{
|
|
{ID: pasteFileID, DataPrefix: []byte("pasted content")},
|
|
}, nil)
|
|
|
|
pasteText, err := titlePasteText(context.Background(), db, []database.ChatMessage{pasteMessage})
|
|
require.NoError(t, err)
|
|
require.Equal(t, map[uuid.UUID]string{pasteFileID: "pasted content"}, pasteText)
|
|
})
|
|
|
|
t.Run("propagates fetch errors", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctrl := gomock.NewController(t)
|
|
db := dbmock.NewMockStore(ctrl)
|
|
db.EXPECT().GetChatFileDataPrefixesByIDs(gomock.Any(), gomock.Any()).Return(nil, sql.ErrConnDone)
|
|
|
|
_, err := titlePasteText(context.Background(), db, []database.ChatMessage{pasteMessage})
|
|
require.ErrorIs(t, err, sql.ErrConnDone)
|
|
})
|
|
|
|
t.Run("ignores non synthetic file only messages", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctrl := gomock.NewController(t)
|
|
db := dbmock.NewMockStore(ctrl)
|
|
|
|
pasteText, err := titlePasteText(context.Background(), db, []database.ChatMessage{
|
|
mustChatMessage(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessageFile(uuid.New(), "image/png", "photo.png"),
|
|
),
|
|
})
|
|
require.NoError(t, err)
|
|
require.Nil(t, pasteText)
|
|
})
|
|
}
|
|
|
|
func TestMaybeGenerateChatTitle(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const userPrompt = "summarize failed workspace build logs"
|
|
fallbackTitle := chatprompt.FallbackTitle(userPrompt)
|
|
|
|
newFakeModel := func(t *testing.T, beforeReturn func(), title string, err error) *chattest.FakeModel {
|
|
return &chattest.FakeModel{
|
|
GenerateObjectFn: func(_ context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
|
|
assert.Equal(t, "propose_title", call.SchemaName)
|
|
if beforeReturn != nil {
|
|
beforeReturn()
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &fantasy.ObjectResponse{Object: map[string]any{"title": title}}, nil
|
|
},
|
|
}
|
|
}
|
|
|
|
run := func(t *testing.T, db database.Store, ps dbpubsub.Pubsub, chat database.Chat, model *chattest.FakeModel) {
|
|
t.Helper()
|
|
message := mustChatMessage(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, codersdk.ChatMessageText(userPrompt))
|
|
message.ID = 1
|
|
server := &Server{db: db, pubsub: ps, logger: slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}), chatLimits: Limits{}.withDefaults()}
|
|
server.maybeGenerateChatTitle(
|
|
testutil.Context(t, testutil.WaitMedium),
|
|
chat,
|
|
[]database.ChatMessage{message},
|
|
nil,
|
|
resolvedModelCall{
|
|
model: chatprovider.NewModel(model, nil),
|
|
dbConfig: database.ChatModelConfig{Model: "test-model"},
|
|
},
|
|
server.logger,
|
|
nil,
|
|
)
|
|
}
|
|
|
|
t.Run("WritesGeneratedTitle", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
cases := []struct {
|
|
name string
|
|
title string
|
|
}{
|
|
{name: "new text", title: "Failed workspace logs"},
|
|
{name: "same text as fallback", title: fallbackTitle},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, _ := dbtestutil.NewDB(t)
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
owner, chat := seedTitleChat(t, db, fallbackTitle, database.ChatTitleSourceFallback)
|
|
ps := dbpubsub.NewInMemory()
|
|
events := subscribeChatWatchEvents(t, ps, owner.ID)
|
|
|
|
run(t, db, ps, chat, newFakeModel(t, nil, tc.title, nil))
|
|
|
|
fetched, err := db.GetChatByID(ctx, chat.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, tc.title, fetched.Title)
|
|
require.Equal(t, database.ChatTitleSourceGenerated, fetched.TitleSource)
|
|
require.True(t, fetched.UpdatedAt.Equal(chat.UpdatedAt), "title writes must not reorder chat lists")
|
|
|
|
event := testutil.RequireReceive(ctx, t, events)
|
|
require.Equal(t, codersdk.ChatWatchEventKindTitleChange, event.Kind)
|
|
require.Equal(t, tc.title, event.Chat.Title)
|
|
require.Equal(t, codersdk.ChatTitleSourceGenerated, event.Chat.TitleSource)
|
|
require.True(t, event.Chat.TitleUpdatedAt.After(chat.TitleUpdatedAt), "a title write must advance title_updated_at")
|
|
})
|
|
}
|
|
})
|
|
|
|
t.Run("KeepsCurrentTitleWhenNoTitleIsWritten", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const renamed = "My build investigation"
|
|
cases := []struct {
|
|
name string
|
|
renameDuringCall bool
|
|
modelErr error
|
|
wantTitle string
|
|
wantSource database.ChatTitleSource
|
|
}{
|
|
{
|
|
name: "write refused by a rename during the call",
|
|
renameDuringCall: true,
|
|
wantTitle: renamed,
|
|
wantSource: database.ChatTitleSourceUser,
|
|
},
|
|
{
|
|
name: "model call failed",
|
|
modelErr: xerrors.New("provider returned status 400: invalid request"),
|
|
wantTitle: fallbackTitle,
|
|
wantSource: database.ChatTitleSourceFallback,
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, _ := dbtestutil.NewDB(t)
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
_, chat := seedTitleChat(t, db, fallbackTitle, database.ChatTitleSourceFallback)
|
|
ps := psmock.NewMockPubsub(gomock.NewController(t))
|
|
|
|
var rename func()
|
|
if tc.renameDuringCall {
|
|
rename = func() {
|
|
_, err := db.UpdateChatTitleByID(ctx, database.UpdateChatTitleByIDParams{
|
|
ID: chat.ID,
|
|
Title: renamed,
|
|
TitleSource: database.ChatTitleSourceUser,
|
|
})
|
|
assert.NoError(t, err)
|
|
}
|
|
}
|
|
model := newFakeModel(t, rename, "Generated title", tc.modelErr)
|
|
|
|
run(t, db, ps, chat, model)
|
|
|
|
fetched, err := db.GetChatByID(ctx, chat.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, tc.wantTitle, fetched.Title)
|
|
require.Equal(t, tc.wantSource, fetched.TitleSource)
|
|
})
|
|
}
|
|
})
|
|
}
|
|
|
|
func seedTitleChat(t *testing.T, db database.Store, title string, source database.ChatTitleSource) (database.User, database.Chat) {
|
|
t.Helper()
|
|
owner := dbgen.User(t, db, database.User{})
|
|
org := dbgen.Organization(t, db, database.Organization{})
|
|
dbgen.OrganizationMember(t, db, database.OrganizationMember{
|
|
UserID: owner.ID,
|
|
OrganizationID: org.ID,
|
|
})
|
|
dbgen.ChatProvider(t, db, database.ChatProvider{
|
|
Provider: "openai",
|
|
DisplayName: "OpenAI",
|
|
APIKey: "test-key",
|
|
Enabled: true,
|
|
CentralApiKeyEnabled: true,
|
|
})
|
|
modelConfig := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{Model: "test-model"})
|
|
chat := dbgen.Chat(t, db, database.Chat{
|
|
OrganizationID: org.ID,
|
|
OwnerID: owner.ID,
|
|
LastModelConfigID: modelConfig.ID,
|
|
Title: title,
|
|
TitleSource: source,
|
|
Status: database.ChatStatusWaiting,
|
|
ClientType: database.ChatClientTypeUi,
|
|
})
|
|
return owner, chat
|
|
}
|
|
|
|
func subscribeChatWatchEvents(t *testing.T, ps dbpubsub.Pubsub, ownerID uuid.UUID) <-chan codersdk.ChatWatchEvent {
|
|
t.Helper()
|
|
events := make(chan codersdk.ChatWatchEvent, 8)
|
|
cancel, err := ps.SubscribeWithErr(
|
|
coderdpubsub.ChatWatchEventChannel(ownerID),
|
|
coderdpubsub.HandleChatWatchEvent(func(_ context.Context, event codersdk.ChatWatchEvent, err error) {
|
|
assert.NoError(t, err)
|
|
events <- event
|
|
}),
|
|
)
|
|
require.NoError(t, err)
|
|
t.Cleanup(cancel)
|
|
return events
|
|
}
|
|
|
|
// TestQuickgenFollowsConfiguredRetries verifies that title and summary
|
|
// generation stop retrying at the server's configured retry limit. The
|
|
// unconfigured default is not exercised because 25 real backoffs take minutes.
|
|
func TestQuickgenFollowsConfiguredRetries(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
limits := Limits{MaxGenerationRetries: 1}
|
|
rateLimited := &fantasy.ProviderError{StatusCode: http.StatusTooManyRequests, Message: "rate limited"}
|
|
|
|
t.Run("Title", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, ps := dbtestutil.NewDB(t)
|
|
owner := dbgen.User(t, db, database.User{})
|
|
org := dbgen.Organization(t, db, database.Organization{})
|
|
dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: owner.ID, OrganizationID: org.ID})
|
|
modelConfig := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{Model: "test-model"})
|
|
userPrompt := "summarize failed workspace build logs"
|
|
chat := dbgen.Chat(t, db, database.Chat{
|
|
OrganizationID: org.ID,
|
|
OwnerID: owner.ID,
|
|
LastModelConfigID: modelConfig.ID,
|
|
Title: chatprompt.FallbackTitle(userPrompt),
|
|
Status: database.ChatStatusWaiting,
|
|
ClientType: database.ChatClientTypeUi,
|
|
})
|
|
message := mustChatMessage(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessageText(userPrompt))
|
|
message.ID = 1
|
|
|
|
var calls atomic.Int32
|
|
model := &chattest.FakeModel{
|
|
GenerateObjectFn: func(context.Context, fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
|
|
calls.Add(1)
|
|
return nil, rateLimited
|
|
},
|
|
}
|
|
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
|
server := newInternalTestServer(t, db, ps, chatprovider.ProviderAPIKeys{},
|
|
withInternalTestServerLimits(limits))
|
|
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
server.maybeGenerateChatTitle(
|
|
ctx,
|
|
chat,
|
|
[]database.ChatMessage{message},
|
|
nil,
|
|
resolvedModelCall{
|
|
model: chatprovider.NewModel(model, nil),
|
|
dbConfig: database.ChatModelConfig{Model: "test-model"},
|
|
},
|
|
logger,
|
|
nil,
|
|
)
|
|
|
|
require.EqualValues(t, 2, calls.Load(), "one configured retry allows exactly two model calls")
|
|
})
|
|
|
|
t.Run("Summary", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, ps := dbtestutil.NewDB(t)
|
|
user := dbgen.User(t, db, database.User{})
|
|
org := dbgen.Organization(t, db, database.Organization{})
|
|
dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user.ID, OrganizationID: org.ID})
|
|
provider := dbgen.AIProviderWithOptionalKey(t, db, database.AIProvider{
|
|
Type: database.AIProviderTypeOpenai,
|
|
}, "test-key")
|
|
modelConfig := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
|
|
Model: "gpt-4o-mini",
|
|
AIProviderID: uuid.NullUUID{UUID: provider.ID, Valid: true},
|
|
OrganizationID: org.ID,
|
|
}, func(p *database.InsertChatModelConfigParams) {
|
|
p.Enabled = true
|
|
})
|
|
|
|
var requests atomic.Int32
|
|
factory := &aibridgeTestFactory{rt: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
|
requests.Add(1)
|
|
return &http.Response{
|
|
StatusCode: http.StatusTooManyRequests,
|
|
Header: http.Header{"Content-Type": []string{"application/json"}},
|
|
Body: io.NopCloser(strings.NewReader(`{"error":{"message":"rate limited","type":"rate_limit_error"}}`)),
|
|
}, nil
|
|
})}
|
|
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
|
server := newInternalTestServer(t, db, ps, chatprovider.ProviderAPIKeys{},
|
|
withInternalTestServerTransportFactory(factory),
|
|
withInternalTestServerLimits(limits),
|
|
)
|
|
|
|
ctx := chatdTestContext(t)
|
|
created, err := chatstate.CreateChat(ctx, db, ps, chatstate.CreateChatInput{
|
|
OrganizationID: org.ID,
|
|
OwnerID: user.ID,
|
|
LastModelConfigID: modelConfig.ID,
|
|
Title: "summary retries",
|
|
ClientType: database.ChatClientTypeApi,
|
|
InitialMessages: []chatstate.Message{{
|
|
Role: database.ChatMessageRoleUser,
|
|
Content: mustMarshalText(t, strings.Repeat("Investigate the failing workspace build. ", 10)),
|
|
Visibility: database.ChatMessageVisibilityBoth,
|
|
ModelConfigID: uuid.NullUUID{UUID: modelConfig.ID, Valid: true},
|
|
CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
|
ContentVersion: chatprompt.CurrentContentVersion,
|
|
}},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
server.generateAndStoreChatSummary(ctx, logger, created.Chat)
|
|
|
|
require.EqualValues(t, 2, requests.Load(), "one configured retry allows exactly two provider requests")
|
|
fetched, err := db.GetChatByID(ctx, created.Chat.ID)
|
|
require.NoError(t, err)
|
|
require.False(t, fetched.Summary.Valid)
|
|
})
|
|
}
|
|
|
|
func TestMaybeGenerateChatTitleAppliesModelConfigReasoningEffort(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitShort)
|
|
chat, messages := titleOverrideTestChatAndMessages(t)
|
|
reasoningEffort := "high"
|
|
maxReasoningEffort := "max"
|
|
modelConfigRaw, err := json.Marshal(codersdk.ChatModelCallConfig{
|
|
ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{
|
|
Default: &reasoningEffort,
|
|
Max: &maxReasoningEffort,
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
model := &chattest.FakeModel{
|
|
ProviderName: fantasyopenai.Name,
|
|
ModelName: "gpt-4o-mini",
|
|
GenerateObjectFn: func(_ context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
|
|
require.NotNil(t, call.MaxOutputTokens)
|
|
require.Equal(t, titleMaxOutputTokens, *call.MaxOutputTokens)
|
|
providerOptions, ok := call.ProviderOptions[fantasyopenai.Name].(*fantasyopenai.ResponsesProviderOptions)
|
|
require.True(t, ok, "%T", call.ProviderOptions[fantasyopenai.Name])
|
|
require.NotNil(t, providerOptions.ReasoningEffort)
|
|
require.Equal(t, fantasyopenai.ReasoningEffortHigh, *providerOptions.ReasoningEffort)
|
|
return &fantasy.ObjectResponse{
|
|
Object: map[string]any{"title": "Reasoning title"},
|
|
}, nil
|
|
},
|
|
}
|
|
|
|
db := dbmock.NewMockStore(gomock.NewController(t))
|
|
db.EXPECT().UpdateChatTitleByID(gomock.Any(), database.UpdateChatTitleByIDParams{
|
|
ID: chat.ID,
|
|
Title: "Reasoning title",
|
|
TitleSource: database.ChatTitleSourceGenerated,
|
|
}).Return(chatWithGeneratedTitle(chat, "Reasoning title"), nil)
|
|
|
|
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
|
server := titleOverrideTestServer(db, logger)
|
|
fallbackModel := chatprovider.NewModel(model, nil)
|
|
fallbackConfig := database.ChatModelConfig{Model: "gpt-4o-mini", Options: modelConfigRaw}
|
|
callConfig, err := parseModelConfigOptions(fallbackConfig.Options)
|
|
require.NoError(t, err)
|
|
server.maybeGenerateChatTitle(
|
|
ctx,
|
|
chat,
|
|
messages,
|
|
nil,
|
|
resolvedModelCall{
|
|
model: fallbackModel,
|
|
dbConfig: fallbackConfig,
|
|
providerOptions: chatprovider.ProviderOptionsForCall(fallbackModel, callConfig, nil),
|
|
},
|
|
logger,
|
|
nil,
|
|
)
|
|
}
|
|
|
|
func Test_titleGenerationPrompt_UsesSlimRules(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
require.Contains(t, titleGenerationPrompt, "Keep the title to 2-8 words")
|
|
require.Contains(t, titleGenerationPrompt, "Do not answer the user or describe the title-writing task")
|
|
require.Contains(t, titleGenerationPrompt, "stay close to the user's wording")
|
|
require.Contains(t, titleGenerationPrompt, "same language as the user's message")
|
|
require.Contains(t, titleGenerationPrompt, "Examples:")
|
|
require.NotContains(t, titleGenerationPrompt, "I am a title generator")
|
|
}
|
|
|
|
func Test_generateManualTitle_UsesTimeout(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
mustChatMessage(
|
|
t,
|
|
database.ChatMessageRoleUser,
|
|
database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessageText("refresh chat title"),
|
|
),
|
|
}
|
|
|
|
model := &chattest.FakeModel{
|
|
GenerateObjectFn: func(ctx context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
|
|
deadline, ok := ctx.Deadline()
|
|
require.True(t, ok, "manual title generation should set a deadline")
|
|
require.WithinDuration(
|
|
t,
|
|
time.Now().Add(30*time.Second),
|
|
deadline,
|
|
2*time.Second,
|
|
)
|
|
require.Len(t, call.Prompt, 2)
|
|
require.Equal(t, "propose_title", call.SchemaName)
|
|
return &fantasy.ObjectResponse{Object: map[string]any{"title": "Refresh title"}}, nil
|
|
},
|
|
}
|
|
|
|
title, err := generateManualTitle(
|
|
context.Background(),
|
|
messages,
|
|
nil,
|
|
model,
|
|
titleObjectCall(resolvedModelCall{}),
|
|
codersdk.DefaultChatMaxGenerationRetries,
|
|
)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "Refresh title", title)
|
|
}
|
|
|
|
func Test_generateManualTitle_TruncatesFirstUserInput(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
longFirstUserText := strings.Repeat("a", 1500)
|
|
messages := []database.ChatMessage{
|
|
mustChatMessage(
|
|
t,
|
|
database.ChatMessageRoleUser,
|
|
database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessageText(longFirstUserText),
|
|
),
|
|
}
|
|
|
|
model := &chattest.FakeModel{
|
|
GenerateObjectFn: func(_ context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
|
|
require.Len(t, call.Prompt, 2)
|
|
systemText, ok := call.Prompt[0].Content[0].(fantasy.TextPart)
|
|
require.True(t, ok)
|
|
require.Contains(t, systemText.Text, truncateRunes(longFirstUserText, 1000))
|
|
|
|
userText, ok := call.Prompt[1].Content[0].(fantasy.TextPart)
|
|
require.True(t, ok)
|
|
require.Equal(t, truncateRunes(longFirstUserText, 1000), userText.Text)
|
|
return &fantasy.ObjectResponse{Object: map[string]any{"title": "Refresh title"}}, nil
|
|
},
|
|
}
|
|
|
|
_, err := generateManualTitle(
|
|
context.Background(),
|
|
messages,
|
|
nil,
|
|
model,
|
|
titleObjectCall(resolvedModelCall{}),
|
|
codersdk.DefaultChatMaxGenerationRetries,
|
|
)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func Test_generateManualTitle_ErrorsOnEmptyNormalizedTitle(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
mustChatMessage(
|
|
t,
|
|
database.ChatMessageRoleUser,
|
|
database.ChatMessageVisibilityBoth,
|
|
codersdk.ChatMessageText("refresh chat title"),
|
|
),
|
|
}
|
|
|
|
model := &chattest.FakeModel{
|
|
GenerateObjectFn: func(_ context.Context, _ fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
|
|
return &fantasy.ObjectResponse{
|
|
Object: map[string]any{"title": "\"\""},
|
|
Usage: fantasy.Usage{
|
|
InputTokens: 11,
|
|
OutputTokens: 7,
|
|
TotalTokens: 18,
|
|
},
|
|
}, nil
|
|
},
|
|
}
|
|
|
|
_, err := generateManualTitle(
|
|
context.Background(),
|
|
messages,
|
|
nil,
|
|
model,
|
|
titleObjectCall(resolvedModelCall{}),
|
|
codersdk.DefaultChatMaxGenerationRetries,
|
|
)
|
|
require.ErrorContains(t, err, "generated title was empty")
|
|
}
|
|
|
|
func Test_selectPreferredConfiguredShortTextModelConfig(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("chooses the highest-priority configured lightweight model", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
configs := []database.GetEnabledChatModelConfigsByOrganizationRow{
|
|
{ChatModelConfig: database.ChatModelConfig{Model: preferredTitleModels[2].model}, Provider: preferredTitleModels[2].provider},
|
|
{ChatModelConfig: database.ChatModelConfig{Model: preferredTitleModels[1].model}, Provider: preferredTitleModels[1].provider},
|
|
{ChatModelConfig: database.ChatModelConfig{Model: "gpt-4.1"}, Provider: "openai"},
|
|
}
|
|
|
|
got, ok := selectPreferredConfiguredShortTextModelConfig(configs)
|
|
require.True(t, ok)
|
|
require.Equal(t, preferredTitleModels[1].model, got.Model)
|
|
})
|
|
|
|
t.Run("matches dated snapshots of a preferred model", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
for _, model := range []string{"claude-haiku-4-5-20251001", "Claude-Haiku-4-5-2025-10-01"} {
|
|
got, ok := selectPreferredConfiguredShortTextModelConfig([]database.GetEnabledChatModelConfigsByOrganizationRow{
|
|
{ChatModelConfig: database.ChatModelConfig{Model: "claude-haiku-4-5-preview"}, Provider: "anthropic"},
|
|
{ChatModelConfig: database.ChatModelConfig{Model: model}, Provider: "anthropic"},
|
|
})
|
|
require.True(t, ok, model)
|
|
require.Equal(t, model, got.Model)
|
|
}
|
|
})
|
|
|
|
t.Run("returns false when no preferred lightweight model is configured", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got, ok := selectPreferredConfiguredShortTextModelConfig([]database.GetEnabledChatModelConfigsByOrganizationRow{{
|
|
ChatModelConfig: database.ChatModelConfig{Model: "gpt-4.1"},
|
|
Provider: "openai",
|
|
}})
|
|
require.False(t, ok)
|
|
require.Equal(t, database.ChatModelConfig{}, got)
|
|
})
|
|
}
|
|
|
|
func TestNormalizeTurnStatusLabel(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
input string
|
|
want string
|
|
ok bool
|
|
}{
|
|
{name: "accepts short label", input: "Finished unit tests", want: "Finished unit tests", ok: true},
|
|
{name: "accepts two word label", input: "Submitted PR", want: "Submitted PR", ok: true},
|
|
{name: "trims quotes and trailing punctuation", input: `"Submitted PR."`, want: "Submitted PR", ok: true},
|
|
{name: "keeps version punctuation", input: "Updated v2.1 config", want: "Updated v2.1 config", ok: true},
|
|
{name: "accepts five word label", input: "Updated workspace proxy routing rules", want: "Updated workspace proxy routing rules", ok: true},
|
|
{name: "rejects agent phrasing", input: "Agent identified failing tests", ok: false},
|
|
{name: "rejects agent possessive", input: "Agent's findings reviewed", ok: false},
|
|
{name: "rejects i contraction", input: "I've fixed tests", ok: false},
|
|
{name: "rejects it contraction", input: "It's still running", ok: false},
|
|
{name: "rejects we contraction", input: "We're almost done", ok: false},
|
|
{name: "rejects agent phrase without prefix", input: "Found agent identified bugs", ok: false},
|
|
{name: "rejects chat phrasing", input: "The chat is waiting now", ok: false},
|
|
{name: "rejects multiline labels", input: "Fixed bug\nAdded tests", ok: false},
|
|
{name: "rejects multi sentence labels", input: "Fixed bug. Added tests", ok: false},
|
|
{name: "rejects single word", input: "Fixed", ok: false},
|
|
{name: "rejects long labels", input: "Fixed the bug and added tests", ok: false},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got, ok := normalizeTurnStatusLabel(tt.input)
|
|
require.Equal(t, tt.ok, ok)
|
|
require.Equal(t, tt.want, got)
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_normalizeTitleOutput(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
cases := []struct {
|
|
name string
|
|
input string
|
|
want string
|
|
}{
|
|
{name: "empty after normalization", input: " \"\" ", want: ""},
|
|
{name: "collapses whitespace and strips quotes", input: "\"Fix pq duplicate\tkey\"", want: "Fix pq duplicate key"},
|
|
{name: "keeps titles within the word budget", input: "Review risky changes in acme/webapp PR #123", want: "Review risky changes in acme/webapp PR #123"},
|
|
{
|
|
name: "truncates overlong titles to the word budget",
|
|
input: "Re-capture Tasks-enabled evidence for the Coder pull request feature flag",
|
|
want: "Re-capture Tasks-enabled evidence for the Coder pull request",
|
|
},
|
|
{name: "truncates to 80 runes", input: strings.Repeat("a", 100), want: strings.Repeat("a", 80)},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
require.Equal(t, tc.want, normalizeTitleOutput(tc.input))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFallbackTurnStatusLabel(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
status database.ChatStatus
|
|
want string
|
|
}{
|
|
{status: database.ChatStatusWaiting, want: "Finished latest turn"},
|
|
{status: database.ChatStatusPaused, want: "Finished latest turn"},
|
|
{status: database.ChatStatusRequiresAction, want: "Waiting for user input"},
|
|
{status: database.ChatStatusError, want: "Hit an error"},
|
|
{status: database.ChatStatus("unknown"), want: "Updated chat status"},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(string(tt.status), func(t *testing.T) {
|
|
t.Parallel()
|
|
require.Equal(t, tt.want, fallbackTurnStatusLabel(tt.status))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGenerateStructuredTitleWithUsage_OpenAICompatibleRequiredToolChoice(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
server, requests := newOpenAICompatStructuredOutputServer(t, "propose_title", `{"title":"Failed workspace logs"}`)
|
|
model := openAICompatTestModel(t, server.URL)
|
|
|
|
title, _, err := generateStructuredTitleWithUsage(
|
|
t.Context(),
|
|
model.LanguageModel(),
|
|
titleObjectCall(resolvedModelCall{}),
|
|
codersdk.DefaultChatMaxGenerationRetries,
|
|
titleGenerationPrompt,
|
|
"summarize failed workspace build logs",
|
|
)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "Failed workspace logs", title)
|
|
|
|
body := testutil.TryReceive(t.Context(), t, requests)
|
|
require.Equal(t, "required", body["tool_choice"])
|
|
require.Equal(t, quickgenTemperature, body["temperature"],
|
|
"title generation should pin temperature for repeatable output")
|
|
}
|
|
|
|
// newTemperatureRejectedError mirrors the bad-request error returned
|
|
// through AI Bridge by models that do not accept the temperature
|
|
// parameter.
|
|
func newTemperatureRejectedError() *fantasy.ProviderError {
|
|
return &fantasy.ProviderError{
|
|
Title: "bad request",
|
|
Message: `POST "http://coder-aibridge/v1/messages": 400 Bad Request ` +
|
|
`{"error":{"message":"` + "`temperature`" + ` is deprecated for this model.",` +
|
|
`"type":"invalid_request_error"},"request_id":"","type":"error"}`,
|
|
StatusCode: http.StatusBadRequest,
|
|
}
|
|
}
|
|
|
|
func TestGenerateStructuredTitleWithUsage_DropsRejectedTemperature(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var sawTemperature []bool
|
|
model := newQuickgenParamTestModel(&Server{clock: quartz.NewMock(t)}, uuid.New(), &chattest.FakeModel{
|
|
GenerateObjectFn: func(_ context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
|
|
sawTemperature = append(sawTemperature, call.Temperature != nil)
|
|
if call.Temperature != nil {
|
|
return nil, newTemperatureRejectedError()
|
|
}
|
|
return &fantasy.ObjectResponse{
|
|
Object: map[string]any{"title": "Failed workspace logs"},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
title, _, err := generateStructuredTitleWithUsage(
|
|
t.Context(),
|
|
model,
|
|
titleObjectCall(resolvedModelCall{}),
|
|
codersdk.DefaultChatMaxGenerationRetries,
|
|
titleGenerationPrompt,
|
|
"summarize failed workspace build logs",
|
|
)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "Failed workspace logs", title)
|
|
require.Equal(t, []bool{true, false}, sawTemperature,
|
|
"generation should retry without temperature after the model rejects it")
|
|
}
|
|
|
|
// newQuickgenParamTestModel wraps lm the way resolveModelCall does, so
|
|
// models wrapped through the same server share learned rejections.
|
|
func newQuickgenParamTestModel(server *Server, providerID uuid.UUID, lm *chattest.FakeModel) fantasy.LanguageModel {
|
|
lm.ProviderName = fantasyanthropic.Name
|
|
lm.ModelName = "claude-opus-5-5"
|
|
return server.withQuickgenParamFallback(chatprovider.NewModel(lm, nil), providerID).LanguageModel()
|
|
}
|
|
|
|
// Mirrors Claude Opus 5.5, which rejects both the forced tool_choice of
|
|
// tool-mode object generation and the temperature parameter.
|
|
func TestGenerateStructuredTitleWithUsage_RemembersRejectedParams(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
type request struct {
|
|
toolMode bool
|
|
temperature bool
|
|
}
|
|
var requests []request
|
|
lm := &chattest.FakeModel{
|
|
GenerateObjectFn: func(_ context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
|
|
requests = append(requests, request{toolMode: true, temperature: call.Temperature != nil})
|
|
return nil, &fantasy.ProviderError{
|
|
Title: "bad request",
|
|
Message: `tool_choice: type "tool" and "any" are not supported for this model.`,
|
|
StatusCode: http.StatusBadRequest,
|
|
}
|
|
},
|
|
GenerateFn: func(_ context.Context, call fantasy.Call) (*fantasy.Response, error) {
|
|
requests = append(requests, request{temperature: call.Temperature != nil})
|
|
if call.Temperature != nil {
|
|
return nil, newTemperatureRejectedError()
|
|
}
|
|
return &fantasy.Response{
|
|
Content: fantasy.ResponseContent{fantasy.TextContent{Text: `{"title":"Failed workspace logs"}`}},
|
|
}, nil
|
|
},
|
|
}
|
|
clock := quartz.NewMock(t)
|
|
server := &Server{clock: clock}
|
|
providerID := uuid.New()
|
|
// Each job resolves its own model, as title generation does per turn.
|
|
runJob := func(providerID uuid.UUID) []request {
|
|
requests = nil
|
|
title, _, err := generateStructuredTitleWithUsage(
|
|
t.Context(),
|
|
newQuickgenParamTestModel(server, providerID, lm),
|
|
titleObjectCall(resolvedModelCall{}),
|
|
codersdk.DefaultChatMaxGenerationRetries,
|
|
titleGenerationPrompt,
|
|
"summarize failed workspace build logs",
|
|
)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "Failed workspace logs", title)
|
|
return requests
|
|
}
|
|
|
|
learning := []request{{toolMode: true, temperature: true}, {temperature: true}, {}}
|
|
require.Equal(t, learning, runJob(providerID),
|
|
"the first job should learn each rejected parameter once")
|
|
require.Equal(t, []request{{}}, runJob(providerID),
|
|
"later jobs should skip parameters the model already rejected")
|
|
require.Equal(t, learning, runJob(uuid.New()),
|
|
"rejections should not apply to a different provider")
|
|
|
|
clock.Advance(quickgenParamRejectionTTL).MustWait(t.Context())
|
|
require.Equal(t, learning, runJob(providerID),
|
|
"expired rejections should be learned again")
|
|
}
|
|
|
|
// Budget thinking would reject the forced tool_choice of tool-mode generation,
|
|
// so no configured effort may turn it on for older Claude quickgen models.
|
|
func TestQuickgenCallsKeepBudgetThinkingOff(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
calls := map[string]func(resolvedModelCall) fantasy.ObjectCall{
|
|
"title": titleObjectCall,
|
|
"summary": summaryObjectCall,
|
|
"label": turnStatusLabelObjectCall,
|
|
}
|
|
efforts := []fantasyanthropic.Effort{
|
|
fantasyanthropic.EffortMinimal,
|
|
fantasyanthropic.EffortLow,
|
|
fantasyanthropic.EffortMedium,
|
|
fantasyanthropic.EffortHigh,
|
|
fantasyanthropic.EffortXHigh,
|
|
fantasyanthropic.EffortMax,
|
|
}
|
|
for name, newCall := range calls {
|
|
for _, effort := range efforts {
|
|
t.Run(name+"/"+string(effort), func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
requests := make(chan *chattest.AnthropicRequest, 1)
|
|
serverURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse {
|
|
select {
|
|
case requests <- req:
|
|
default:
|
|
}
|
|
return chattest.AnthropicResponse{Error: &chattest.ErrorResponse{
|
|
StatusCode: http.StatusBadRequest,
|
|
Type: "invalid_request_error",
|
|
Message: "request captured",
|
|
}}
|
|
})
|
|
client, err := fantasyanthropic.New(
|
|
fantasyanthropic.WithAPIKey("test-key"),
|
|
fantasyanthropic.WithBaseURL(serverURL),
|
|
)
|
|
require.NoError(t, err)
|
|
model, err := client.LanguageModel(t.Context(), "claude-haiku-4-5")
|
|
require.NoError(t, err)
|
|
|
|
call := newCall(resolvedModelCall{providerOptions: fantasy.ProviderOptions{
|
|
fantasyanthropic.Name: &fantasyanthropic.ProviderOptions{Effort: &effort},
|
|
}})
|
|
call.Prompt = fantasy.Prompt{fantasy.NewUserMessage("fix the login bug")}
|
|
_, err = object.Generate[generatedTitle](t.Context(), model, call)
|
|
require.Error(t, err)
|
|
|
|
req := testutil.RequireReceive(t.Context(), t, requests)
|
|
require.Empty(t, req.Thinking, "max_tokens %d enabled budget thinking", req.MaxTokens)
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGenerateStructuredTitleWithUsage_TruncatesOverlongTitle(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
model := &chattest.FakeModel{
|
|
GenerateObjectFn: func(_ context.Context, _ fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
|
|
return &fantasy.ObjectResponse{
|
|
Object: map[string]any{
|
|
"title": "Re-capture Tasks-enabled evidence for the Coder pull request feature flag",
|
|
},
|
|
}, nil
|
|
},
|
|
}
|
|
|
|
title, _, err := generateStructuredTitleWithUsage(
|
|
t.Context(),
|
|
model,
|
|
titleObjectCall(resolvedModelCall{}),
|
|
codersdk.DefaultChatMaxGenerationRetries,
|
|
titleGenerationPrompt,
|
|
"re-capture UI evidence for a Coder pull request",
|
|
)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "Re-capture Tasks-enabled evidence for the Coder pull request", title,
|
|
"an overlong generated title should be truncated to the word budget, not rejected")
|
|
}
|
|
|
|
func newOpenAICompatStructuredOutputServer(
|
|
t *testing.T,
|
|
toolName string,
|
|
arguments string,
|
|
) (*httptest.Server, <-chan map[string]any) {
|
|
t.Helper()
|
|
|
|
requests := make(chan map[string]any, 10)
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
var body map[string]any
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
requests <- body
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"id": "chatcmpl-structured-output",
|
|
"object": "chat.completion",
|
|
"created": time.Now().Unix(),
|
|
"model": "anthropic/claude-4-5-sonnet",
|
|
"choices": []map[string]any{
|
|
{
|
|
"index": 0,
|
|
"message": map[string]any{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": []map[string]any{
|
|
{
|
|
"id": "call_structured_output",
|
|
"type": "function",
|
|
"function": map[string]any{
|
|
"name": toolName,
|
|
"arguments": arguments,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
"finish_reason": "tool_calls",
|
|
},
|
|
},
|
|
"usage": map[string]any{
|
|
"prompt_tokens": 10,
|
|
"completion_tokens": 5,
|
|
"total_tokens": 15,
|
|
},
|
|
})
|
|
}))
|
|
t.Cleanup(server.Close)
|
|
return server, requests
|
|
}
|
|
|
|
func openAICompatTestModel(t *testing.T, baseURL string) chatprovider.Model {
|
|
t.Helper()
|
|
|
|
model, err := chatprovider.ModelFromConfig(
|
|
fantasyopenaicompat.Name,
|
|
"anthropic/claude-4-5-sonnet",
|
|
chatprovider.ProviderAPIKeys{
|
|
ByProvider: map[string]string{
|
|
fantasyopenaicompat.Name: "test-key",
|
|
},
|
|
BaseURLByProvider: map[string]string{
|
|
fantasyopenaicompat.Name: baseURL,
|
|
},
|
|
},
|
|
chatprovider.UserAgent(),
|
|
nil,
|
|
nil,
|
|
nil,
|
|
)
|
|
require.NoError(t, err)
|
|
return model
|
|
}
|
|
|
|
func TestGenerateStructuredTurnStatusLabel(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("returns compact label", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
model := &chattest.FakeModel{
|
|
GenerateObjectFn: func(_ context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
|
|
require.Equal(t, "propose_turn_status_label", call.SchemaName)
|
|
require.NotNil(t, call.MaxOutputTokens)
|
|
require.Equal(t, turnStatusLabelMaxOutputTokens, *call.MaxOutputTokens)
|
|
require.NotNil(t, call.Temperature)
|
|
require.Equal(t, quickgenTemperature, *call.Temperature)
|
|
return &fantasy.ObjectResponse{
|
|
Object: map[string]any{"label": "Submitted PR"},
|
|
}, nil
|
|
},
|
|
}
|
|
|
|
label, err := generateStructuredTurnStatusLabel(t.Context(), model, turnStatusLabelObjectCall(resolvedModelCall{}), codersdk.DefaultChatMaxGenerationRetries, turnStatusLabelPrompt, "done")
|
|
require.NoError(t, err)
|
|
require.Equal(t, "Submitted PR", label)
|
|
})
|
|
|
|
t.Run("sends required tool_choice to openai-compatible provider", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
server, requests := newOpenAICompatStructuredOutputServer(t, "propose_turn_status_label", `{"label":"Submitted PR"}`)
|
|
model := openAICompatTestModel(t, server.URL)
|
|
|
|
label, err := generateStructuredTurnStatusLabel(t.Context(), model.LanguageModel(), turnStatusLabelObjectCall(resolvedModelCall{}), codersdk.DefaultChatMaxGenerationRetries, turnStatusLabelPrompt, "done")
|
|
require.NoError(t, err)
|
|
require.Equal(t, "Submitted PR", label)
|
|
require.Len(t, requests, 1)
|
|
|
|
body := testutil.TryReceive(t.Context(), t, requests)
|
|
require.Equal(t, "required", body["tool_choice"])
|
|
require.Equal(t, quickgenTemperature, body["temperature"],
|
|
"status-label generation should pin temperature for repeatable output")
|
|
})
|
|
|
|
t.Run("drops temperature when model rejects it", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var sawTemperature []bool
|
|
model := newQuickgenParamTestModel(&Server{clock: quartz.NewMock(t)}, uuid.New(), &chattest.FakeModel{
|
|
GenerateObjectFn: func(_ context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
|
|
sawTemperature = append(sawTemperature, call.Temperature != nil)
|
|
if call.Temperature != nil {
|
|
return nil, newTemperatureRejectedError()
|
|
}
|
|
return &fantasy.ObjectResponse{
|
|
Object: map[string]any{"label": "Submitted PR"},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
label, err := generateStructuredTurnStatusLabel(t.Context(), model, turnStatusLabelObjectCall(resolvedModelCall{}), codersdk.DefaultChatMaxGenerationRetries, turnStatusLabelPrompt, "done")
|
|
require.NoError(t, err)
|
|
require.Equal(t, "Submitted PR", label)
|
|
require.Equal(t, []bool{true, false}, sawTemperature,
|
|
"generation should retry without temperature after the model rejects it")
|
|
})
|
|
|
|
t.Run("surfaces unrelated bad request errors", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var calls int
|
|
model := &chattest.FakeModel{
|
|
GenerateObjectFn: func(_ context.Context, _ fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
|
|
calls++
|
|
return nil, &fantasy.ProviderError{
|
|
Title: "bad request",
|
|
Message: "tools.0.custom.input_schema: JSON schema is invalid",
|
|
StatusCode: http.StatusBadRequest,
|
|
}
|
|
},
|
|
}
|
|
|
|
_, err := generateStructuredTurnStatusLabel(t.Context(), model, turnStatusLabelObjectCall(resolvedModelCall{}), codersdk.DefaultChatMaxGenerationRetries, turnStatusLabelPrompt, "done")
|
|
require.ErrorContains(t, err, "JSON schema is invalid")
|
|
require.Equal(t, 1, calls,
|
|
"bad requests unrelated to temperature should not trigger a second attempt")
|
|
})
|
|
|
|
t.Run("rejects narrative label", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
model := &chattest.FakeModel{
|
|
GenerateObjectFn: func(_ context.Context, _ fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
|
|
return &fantasy.ObjectResponse{
|
|
Object: map[string]any{"label": "Agent identified failing tests"},
|
|
}, nil
|
|
},
|
|
}
|
|
|
|
_, err := generateStructuredTurnStatusLabel(t.Context(), model, turnStatusLabelObjectCall(resolvedModelCall{}), codersdk.DefaultChatMaxGenerationRetries, turnStatusLabelPrompt, "done")
|
|
require.ErrorContains(t, err, "generated turn status label was invalid")
|
|
})
|
|
|
|
t.Run("rejects empty input", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
model := &chattest.FakeModel{}
|
|
_, err := generateStructuredTurnStatusLabel(t.Context(), model, turnStatusLabelObjectCall(resolvedModelCall{}), codersdk.DefaultChatMaxGenerationRetries, turnStatusLabelPrompt, " ")
|
|
require.ErrorContains(t, err, "turn status label input was empty")
|
|
})
|
|
}
|
|
|
|
func TestIsTemperatureRejectedError(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
err error
|
|
want bool
|
|
}{
|
|
{
|
|
name: "nil error",
|
|
err: nil,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "plain error mentioning temperature",
|
|
err: xerrors.New("temperature is deprecated for this model"),
|
|
want: false,
|
|
},
|
|
{
|
|
name: "bad request rejecting temperature",
|
|
err: newTemperatureRejectedError(),
|
|
want: true,
|
|
},
|
|
{
|
|
name: "wrapped bad request rejecting temperature",
|
|
err: xerrors.Errorf("tool-based generation failed: %w", newTemperatureRejectedError()),
|
|
want: true,
|
|
},
|
|
{
|
|
name: "bad request with temperature only in response body",
|
|
err: &fantasy.ProviderError{
|
|
Title: "bad request",
|
|
Message: "provider request failed",
|
|
StatusCode: http.StatusBadRequest,
|
|
ResponseBody: []byte(`{"error":{"message":"Unsupported parameter: 'temperature' is not supported with this model."}}`),
|
|
},
|
|
want: true,
|
|
},
|
|
{
|
|
name: "bad request unrelated to temperature",
|
|
err: &fantasy.ProviderError{
|
|
Title: "bad request",
|
|
Message: "tools.0.custom.input_schema: JSON schema is invalid",
|
|
StatusCode: http.StatusBadRequest,
|
|
},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "server error mentioning temperature",
|
|
err: &fantasy.ProviderError{
|
|
Title: "internal server error",
|
|
Message: "temperature processing failed",
|
|
StatusCode: http.StatusInternalServerError,
|
|
},
|
|
want: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
require.Equal(t, tt.want, isTemperatureRejectedError(tt.err))
|
|
})
|
|
}
|
|
}
|
|
|
|
func mustChatMessage(
|
|
t *testing.T,
|
|
role database.ChatMessageRole,
|
|
visibility database.ChatMessageVisibility,
|
|
parts ...codersdk.ChatMessagePart,
|
|
) database.ChatMessage {
|
|
t.Helper()
|
|
|
|
content, err := json.Marshal(parts)
|
|
require.NoError(t, err)
|
|
|
|
return database.ChatMessage{
|
|
Role: role,
|
|
Visibility: visibility,
|
|
Content: pqtype.NullRawMessage{
|
|
RawMessage: content,
|
|
Valid: len(content) > 0,
|
|
},
|
|
ContentVersion: chatprompt.CurrentContentVersion,
|
|
}
|
|
}
|