Files
coder/coderd/x/chatd/hooks_test.go
T
Mathias Fredriksson d639ced1e0 fix: send only changed chat history over the stream (#29662)
A client could resume the chat stream only with after_id, a message
ID. A message ID does not identify what a client has received. A
reconnect could skip a message that another client sent, send no
history_reset for an edit the client missed, or resend the whole
history.

A client can now resume with the chat's snapshot_version, which the
newest messages page returns and the stream's events then include.
The stream sends what changed at that version or later. For an edit
it sends the ID of the edited message and the messages written
since, not the whole history. A stream opened without
snapshot_version sends the same events as before.

A write outside a chat state transition does not increase
snapshot_version, and a stream can miss it. The queries that only
tests used for such writes are removed.

When the stream cannot be set up, the server now returns 500 before
the WebSocket upgrade. It used to upgrade and send an error event,
which the web client shows as a chat error.

The Tasks to Chats migration guide told clients to reconnect with
after_id. It now describes snapshot_version.
2026-11-13 13:02:38 +02:00

906 lines
34 KiB
Go

package chatd_test
import (
"encoding/json"
"net/http"
"net/http/httptest"
"slices"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/google/uuid"
"github.com/prometheus/client_golang/prometheus"
"github.com/sqlc-dev/pqtype"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"cdr.dev/slog/v3/sloggers/slogtest"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbgen"
"github.com/coder/coder/v2/coderd/database/dbtestutil"
dbpubsub "github.com/coder/coder/v2/coderd/database/pubsub"
"github.com/coder/coder/v2/coderd/x/agenthooks/dispatch"
"github.com/coder/coder/v2/coderd/x/chatd"
"github.com/coder/coder/v2/coderd/x/chatd/chathooks"
"github.com/coder/coder/v2/coderd/x/chatd/chatprompt"
"github.com/coder/coder/v2/coderd/x/chatd/chatstate"
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/codersdk/x/agenthooks"
"github.com/coder/coder/v2/testutil"
)
func TestSendMessageUserPromptSubmitHook(t *testing.T) {
t.Parallel()
t.Run("override", func(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
user, org, model := seedChatDependencies(t, db)
chat := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: model.ID,
})
_, err := db.InsertUserSkill(ctx, database.InsertUserSkillParams{
ID: uuid.New(), UserID: user.ID, Name: "notes", Content: "---\nname: notes\ndescription: Notes\n---\n\nTake notes.\n",
})
require.NoError(t, err)
skill := codersdk.ChatMessageSkillInvocation(codersdk.ChatSkillSourcePersonal, "notes")
submitted := []codersdk.ChatMessagePart{
codersdk.ChatMessageText("before"),
codersdk.ChatMessageFileReference("main.go", 1, 3, "package main"),
skill,
}
consumer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var request agenthooks.Request
require.NoError(t, json.NewDecoder(r.Body).Decode(&request))
data := decodeHookData[agenthooks.UserPromptSubmitData](t, request)
require.Equal(t, "before /personal/notes", data.Prompt)
var hookParts []codersdk.ChatMessagePart
require.NoError(t, json.Unmarshal(data.Parts, &hookParts))
require.Equal(t, submitted, hookParts, "hook payload must carry non-text parts without skill bodies")
require.NotNil(t, request.Meta.TurnID)
_, err := w.Write([]byte(`{"permission":{"decision":"allow","input_override":{"prompt":"after"}},"model_context":"model only","user_message":"user only"}`))
require.NoError(t, err)
}))
t.Cleanup(consumer.Close)
server := newHookTestServer(t, db, ps, consumer)
result, err := server.SendMessage(ctx, chatd.SendMessageOptions{
ChatID: chat.ID,
CreatedBy: user.ID,
Content: submitted,
})
require.NoError(t, err)
parts, err := chatprompt.ParseContent(result.Message)
require.NoError(t, err)
storedSkill := skill
storedSkill.SkillBody = "Take notes."
require.Equal(t, []codersdk.ChatMessagePart{
codersdk.ChatMessageText("after"),
codersdk.ChatMessageFileReference("main.go", 1, 3, "package main"),
storedSkill,
{Type: codersdk.ChatMessagePartTypeHookContext, Text: "model only"},
{Type: codersdk.ChatMessagePartTypeHookNotice, Text: "user only"},
}, parts)
require.Len(t, result.InsertedMessages, 1)
require.Equal(t, result.Message.ID, result.InsertedMessages[0].ID)
})
t.Run("deny", func(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
user, org, model := seedChatDependencies(t, db)
chat := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: model.ID,
})
consumer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, err := w.Write([]byte(`{"permission":{"decision":"deny"},"user_message":"blocked"}`))
require.NoError(t, err)
}))
t.Cleanup(consumer.Close)
server := newHookTestServer(t, db, ps, consumer)
_, err := server.SendMessage(ctx, chatd.SendMessageOptions{
ChatID: chat.ID,
CreatedBy: user.ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("blocked prompt")},
})
var denied *chathooks.UserPromptDeniedError
require.ErrorAs(t, err, &denied)
require.Equal(t, "blocked", denied.UserMessage)
messages, err := db.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ChatID: chat.ID})
require.NoError(t, err)
require.Empty(t, messages)
})
}
func newHookDispatcher(t *testing.T, _ database.Store, consumer *httptest.Server) *dispatch.Dispatcher {
t.Helper()
return dispatch.New(
slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}),
consumer.Client(),
consumer.URL,
false,
"test-hook-secret-32-bytes-minimum!!",
time.Second,
"test-deployment",
"test-version",
prometheus.NewRegistry(),
)
}
func newHookTestServer(t *testing.T, db database.Store, ps dbpubsub.Pubsub, consumer *httptest.Server) *chatd.Server {
t.Helper()
return newTestServer(t, db, ps, uuid.New(), func(cfg *chatd.Config) {
cfg.HookDispatcher = newHookDispatcher(t, db, consumer)
})
}
func TestHookDispatcherRequiresExperiment(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
user, org, model := seedChatDependencies(t, db)
chat := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: model.ID,
})
var hookRequests atomic.Int32
consumer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
hookRequests.Add(1)
w.WriteHeader(http.StatusOK)
}))
t.Cleanup(consumer.Close)
server := newTestServer(t, db, ps, uuid.New(), func(cfg *chatd.Config) {
cfg.HookDispatcher = newHookDispatcher(t, db, consumer)
cfg.Experiments = slices.DeleteFunc(
slices.Clone(codersdk.ExperimentsKnown),
func(e codersdk.Experiment) bool { return e == codersdk.ExperimentAgentLifecycleHooks },
)
})
result, err := server.SendMessage(ctx, chatd.SendMessageOptions{
ChatID: chat.ID,
CreatedBy: user.ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("hello")},
})
require.NoError(t, err)
parts, err := chatprompt.ParseContent(result.Message)
require.NoError(t, err)
require.Equal(t, []codersdk.ChatMessagePart{codersdk.ChatMessageText("hello")}, parts)
require.Zero(t, hookRequests.Load())
}
func TestSendMessageUserPromptSubmitPassthrough(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
user, org, model := seedChatDependencies(t, db)
chat := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: model.ID,
})
var received agenthooks.Request
consumer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
require.NoError(t, json.NewDecoder(r.Body).Decode(&received))
_, err := w.Write([]byte(`{}`))
require.NoError(t, err)
}))
t.Cleanup(consumer.Close)
server := newHookTestServer(t, db, ps, consumer)
result, err := server.SendMessage(ctx, chatd.SendMessageOptions{
ChatID: chat.ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("passthrough")},
})
require.NoError(t, err)
require.Equal(t, "passthrough", hookMessageText(t, result.Message))
require.Equal(t, agenthooks.EventUserPromptSubmit, received.Type)
promptData := decodeHookData[agenthooks.UserPromptSubmitData](t, received)
require.Equal(t, "passthrough", promptData.Prompt)
// The persisted content is jsonb-normalized, so compare JSON
// semantics rather than raw bytes.
require.JSONEq(t, string(result.Message.Content.RawMessage), string(promptData.Parts))
}
func TestSendMessageUserPromptSubmitQueue(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
user, org, model := seedChatDependencies(t, db)
chat, err := newTestServer(t, db, ps, uuid.New()).CreateChat(ctx, chatd.CreateOptions{
OrganizationID: org.ID,
OwnerID: user.ID,
Title: "queued hook",
ModelConfigID: model.ID,
InitialUserContent: []codersdk.ChatMessagePart{codersdk.ChatMessageText("running")},
})
require.NoError(t, err)
var received agenthooks.Request
consumer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
require.NoError(t, json.NewDecoder(r.Body).Decode(&received))
_, err := w.Write([]byte(`{"permission":{"decision":"allow","input_override":{"prompt":"queued override"}},"model_context":"queued context","user_message":"queued notice"}`))
require.NoError(t, err)
}))
t.Cleanup(consumer.Close)
server := newHookTestServer(t, db, ps, consumer)
result, err := server.SendMessage(ctx, chatd.SendMessageOptions{
ChatID: chat.ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("queued original")},
BusyBehavior: chatd.SendMessageBusyBehaviorQueue,
})
require.NoError(t, err)
require.True(t, result.Queued)
require.NotNil(t, result.QueuedMessage)
queuedParts, err := chatprompt.ParseContent(database.ChatMessage{
Role: database.ChatMessageRoleUser,
Content: pqtype.NullRawMessage{RawMessage: result.QueuedMessage.Content, Valid: true},
ContentVersion: chatprompt.CurrentContentVersion,
})
require.NoError(t, err)
wantQueuedParts := []codersdk.ChatMessagePart{
codersdk.ChatMessageText("queued override"),
{Type: codersdk.ChatMessagePartTypeHookContext, Text: "queued context"},
{Type: codersdk.ChatMessagePartTypeHookNotice, Text: "queued notice"},
}
require.Equal(t, wantQueuedParts, queuedParts)
queued, err := db.GetChatQueuedMessages(ctx, chat.ID)
require.NoError(t, err)
require.Len(t, queued, 1)
persistedParts, err := chatprompt.ParseContent(database.ChatMessage{
Role: database.ChatMessageRoleUser,
Content: pqtype.NullRawMessage{RawMessage: queued[0].Content, Valid: true},
ContentVersion: chatprompt.CurrentContentVersion,
})
require.NoError(t, err)
require.Equal(t, wantQueuedParts, persistedParts)
require.Equal(t, agenthooks.EventUserPromptSubmit, received.Type)
require.Equal(t, "queued original", decodeHookData[agenthooks.UserPromptSubmitData](t, received).Prompt)
}
func TestSendMessageUserPromptSubmitQueuedRejections(t *testing.T) {
t.Parallel()
tests := []struct {
name string
statusCode int
response string
assertErr func(*testing.T, error)
}{
{
name: "deny",
statusCode: http.StatusOK,
response: `{"permission":{"decision":"deny"},"user_message":"blocked"}`,
assertErr: func(t *testing.T, err error) {
var denied *chathooks.UserPromptDeniedError
require.ErrorAs(t, err, &denied)
},
},
{
name: "dispatch failure",
statusCode: http.StatusInternalServerError,
assertErr: func(t *testing.T, err error) {
var dispatchErr *dispatch.Error
require.ErrorAs(t, err, &dispatchErr)
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
user, org, model := seedChatDependencies(t, db)
chat, err := newTestServer(t, db, ps, uuid.New()).CreateChat(ctx, chatd.CreateOptions{
OrganizationID: org.ID,
OwnerID: user.ID,
Title: "queued rejection",
ModelConfigID: model.ID,
InitialUserContent: []codersdk.ChatMessagePart{codersdk.ChatMessageText("running")},
})
require.NoError(t, err)
consumer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(test.statusCode)
if test.response != "" {
_, err := w.Write([]byte(test.response))
require.NoError(t, err)
}
}))
t.Cleanup(consumer.Close)
server := newHookTestServer(t, db, ps, consumer)
_, err = server.SendMessage(ctx, chatd.SendMessageOptions{
ChatID: chat.ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("queued")},
BusyBehavior: chatd.SendMessageBusyBehaviorQueue,
})
test.assertErr(t, err)
queued, err := db.GetChatQueuedMessages(ctx, chat.ID)
require.NoError(t, err)
require.Empty(t, queued)
updated, err := db.GetChatByID(ctx, chat.ID)
require.NoError(t, err)
require.Equal(t, database.ChatStatusRunning, updated.Status)
require.False(t, updated.LastError.Valid)
})
}
}
func TestSubagentSpawnHookDispatchFailureFailsTurn(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
db, ps := dbtestutil.NewDB(t)
openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
if !req.Stream {
return chattest.OpenAINonStreamingResponse("title")
}
chunk := chattest.OpenAIToolCallChunk("spawn_agent", `{"type":"general","prompt":"child admission prompt"}`)
chunk.Choices[0].ToolCalls[0].ID = "call_spawn"
return chattest.OpenAIStreamingResponse(chunk)
})
user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL)
consumer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var request agenthooks.Request
require.NoError(t, json.NewDecoder(r.Body).Decode(&request))
if request.Type == agenthooks.EventUserPromptSubmit {
data := decodeHookData[agenthooks.UserPromptSubmitData](t, request)
if data.Prompt == "child admission prompt" {
w.WriteHeader(http.StatusInternalServerError)
return
}
}
_, err := w.Write([]byte(`{}`))
require.NoError(t, err)
}))
t.Cleanup(consumer.Close)
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
cfg.HookDispatcher = newHookDispatcher(t, db, consumer)
})
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
OrganizationID: org.ID,
OwnerID: user.ID,
Title: "spawn-hook-failure",
ModelConfigID: model.ID,
InitialUserContent: []codersdk.ChatMessagePart{
codersdk.ChatMessageText("spawn a child"),
},
})
require.NoError(t, err)
failed := waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusError)
require.Contains(t, chatLastErrorMessage(failed.LastError), "hook dispatch failed: user_prompt_submit: http_error")
messages := chatMessages(ctx, t, db, chat.ID)
require.Len(t, messages, 3)
require.Equal(t, database.ChatMessageRoleUser, messages[0].Role)
require.Equal(t, database.ChatMessageRoleAssistant, messages[1].Role)
require.Equal(t, database.ChatMessageRoleTool, messages[2].Role)
require.Contains(t, string(messages[2].Content.RawMessage), "lifecycle hook returned HTTP status 500")
chats, err := db.GetChats(ctx, database.GetChatsParams{
OwnedOnly: true,
ViewerID: user.ID,
AfterID: uuid.Nil,
OffsetOpt: 0,
LimitOpt: 100,
})
require.NoError(t, err)
require.Len(t, chats, 1)
}
func TestSendMessageUserPromptSubmitDispatchFailure(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
user, org, model := seedChatDependencies(t, db)
chat := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: model.ID,
})
var received agenthooks.Request
consumer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
require.NoError(t, json.NewDecoder(r.Body).Decode(&received))
w.WriteHeader(http.StatusInternalServerError)
}))
t.Cleanup(consumer.Close)
server := newHookTestServer(t, db, ps, consumer)
_, err := server.SendMessage(ctx, chatd.SendMessageOptions{
ChatID: chat.ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("fails")},
})
var dispatchErr *dispatch.Error
require.ErrorAs(t, err, &dispatchErr)
require.Equal(t, dispatch.ResultHTTPError, dispatchErr.Class)
updated, err := db.GetChatByID(ctx, chat.ID)
require.NoError(t, err)
require.Equal(t, database.ChatStatusError, updated.Status)
var chatErr codersdk.ChatError
require.NoError(t, json.Unmarshal(updated.LastError.RawMessage, &chatErr))
require.Equal(t, "hook dispatch failed: user_prompt_submit: http_error (dispatch "+dispatchErr.DispatchID.String()+")", chatErr.Message)
require.Equal(t, agenthooks.EventUserPromptSubmit, received.Type)
prompt := decodeHookData[agenthooks.UserPromptSubmitData](t, received)
require.Equal(t, "fails", prompt.Prompt)
messages, err := db.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ChatID: chat.ID})
require.NoError(t, err)
require.Empty(t, messages)
queued, err := db.GetChatQueuedMessages(ctx, chat.ID)
require.NoError(t, err)
require.Empty(t, queued)
}
func TestEditMessageUserPromptSubmitHook(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
user, org, model := seedChatDependencies(t, db)
chat := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: model.ID,
})
content, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{codersdk.ChatMessageText("original")})
require.NoError(t, err)
inserted, err := db.InsertChatMessages(ctx, singleChatMessageInsertParams(
chat.ID, database.ChatMessageRoleUser, content, model.ID, user.ID,
))
require.NoError(t, err)
require.Len(t, inserted, 1)
type receivedHook struct {
request agenthooks.Request
claims agenthooks.Claims
}
var receivedMu sync.Mutex
received := make([]receivedHook, 0, 2)
consumer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var request agenthooks.Request
require.NoError(t, json.NewDecoder(r.Body).Decode(&request))
claims, err := agenthooks.Verify(r.Header.Get("Authorization"), []byte("test-hook-secret-32-bytes-minimum!!"))
require.NoError(t, err)
receivedMu.Lock()
received = append(received, receivedHook{request: request, claims: claims})
receivedMu.Unlock()
response := `{"model_context":"clear context","user_message":"clear notice"}`
if request.Type == agenthooks.EventUserPromptSubmit {
response = `{"permission":{"decision":"allow","input_override":{"prompt":"edited override"}},"model_context":"edit context","user_message":"edit notice"}`
}
_, err = w.Write([]byte(response))
require.NoError(t, err)
}))
t.Cleanup(consumer.Close)
server := newHookTestServer(t, db, ps, consumer)
chatFile, err := db.InsertChatFile(ctx, database.InsertChatFileParams{
OwnerID: user.ID,
OrganizationID: org.ID,
Name: "edited.png",
Mimetype: "image/png",
Data: []byte("png-bytes"),
})
require.NoError(t, err)
upload := codersdk.ChatMessageFile(chatFile.ID, chatFile.Mimetype, chatFile.Name)
reference := codersdk.ChatMessageFileReference("main.go", 1, 3, "package main")
result, err := server.EditMessage(ctx, chatd.EditMessageOptions{
ChatID: chat.ID,
CreatedBy: user.ID,
EditedMessageID: inserted[0].ID,
Content: []codersdk.ChatMessagePart{
reference,
codersdk.ChatMessageText("edited original"),
upload,
},
})
require.NoError(t, err)
linkedFiles, err := db.GetChatFileMetadataByChatID(ctx, chat.ID)
require.NoError(t, err)
require.Len(t, linkedFiles, 1)
require.Equal(t, chatFile.ID, linkedFiles[0].ID)
parts, err := chatprompt.ParseContent(result.Message)
require.NoError(t, err)
require.Equal(t, []codersdk.ChatMessagePart{
reference,
codersdk.ChatMessageText("edited override"),
upload,
{Type: codersdk.ChatMessagePartTypeHookContext, Text: "edit context"},
{Type: codersdk.ChatMessagePartTypeHookNotice, Text: "edit notice"},
}, parts)
receivedMu.Lock()
received = slices.Clone(received)
receivedMu.Unlock()
require.Len(t, received, 2)
require.Equal(t, agenthooks.EventSessionStart, received[0].request.Type)
require.Equal(t, agenthooks.SessionStartData{Source: "clear"}, decodeHookData[agenthooks.SessionStartData](t, received[0].request))
require.Equal(t, received[0].request.Meta.DispatchID, received[0].claims.JTI)
require.Equal(t, agenthooks.EventUserPromptSubmit, received[1].request.Type)
prompt := decodeHookData[agenthooks.UserPromptSubmitData](t, received[1].request)
require.Equal(t, "edited original", prompt.Prompt)
require.NotNil(t, received[0].request.Meta.TurnID)
require.Equal(t, received[0].request.Meta.TurnID, received[1].request.Meta.TurnID)
require.Equal(t, received[1].request.Meta.DispatchID, received[1].claims.JTI)
require.NotEqual(t, received[0].claims.JTI, received[1].claims.JTI)
rows, err := db.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ChatID: chat.ID})
require.NoError(t, err)
var foundNotice bool
for _, row := range rows {
if row.Role == database.ChatMessageRoleSystem && row.Visibility == database.ChatMessageVisibilityUser && hookMessageText(t, row) == "clear notice" {
foundNotice = true
}
}
require.True(t, foundNotice)
promptRows, err := db.GetChatMessagesForPromptByChatID(ctx, chat.ID)
require.NoError(t, err)
var foundContext bool
for _, row := range promptRows {
if row.Visibility == database.ChatMessageVisibilityModel && hookMessageText(t, row) == "clear context" {
foundContext = true
}
}
require.True(t, foundContext)
}
// seedErroredChatWithQueuedMessage returns an errored chat (E1) with one
// queued row, a state that admits EditQueuedMessage.
func seedErroredChatWithQueuedMessage(t *testing.T, db database.Store, text string) (database.Chat, database.ChatQueuedMessage) {
t.Helper()
ctx := testutil.Context(t, testutil.WaitShort)
user, org, model := seedChatDependencies(t, db)
chat := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: model.ID,
Status: database.ChatStatusError,
})
content, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{codersdk.ChatMessageText(text)})
require.NoError(t, err)
queued, err := db.InsertChatQueuedMessageWithCreator(ctx, database.InsertChatQueuedMessageWithCreatorParams{
ChatID: chat.ID,
Content: content.RawMessage,
ModelConfigID: uuid.NullUUID{UUID: model.ID, Valid: true},
CreatedBy: user.ID,
BusyBehavior: database.ChatBusyBehaviorQueue,
})
require.NoError(t, err)
return chat, queued
}
func queuedMessageParts(t *testing.T, queued database.ChatQueuedMessage) []codersdk.ChatMessagePart {
t.Helper()
parts, err := chatprompt.ParseContent(database.ChatMessage{
Role: database.ChatMessageRoleUser,
Content: pqtype.NullRawMessage{RawMessage: queued.Content, Valid: true},
ContentVersion: chatprompt.CurrentContentVersion,
})
require.NoError(t, err)
return parts
}
func TestEditQueuedMessageUserPromptSubmitHook(t *testing.T) {
t.Parallel()
t.Run("override", func(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
chat, queued := seedErroredChatWithQueuedMessage(t, db, "original")
var received agenthooks.Request
consumer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
require.NoError(t, json.NewDecoder(r.Body).Decode(&received))
_, err := w.Write([]byte(`{"permission":{"decision":"allow","input_override":{"prompt":"queued edit override"}},"model_context":"edit context","user_message":"edit notice"}`))
require.NoError(t, err)
}))
t.Cleanup(consumer.Close)
server := newHookTestServer(t, db, ps, consumer)
require.NoError(t, server.EditQueuedMessage(ctx, chatd.EditQueuedMessageOptions{
ChatID: chat.ID,
QueuedMessageID: queued.ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("edited")},
}))
require.Equal(t, agenthooks.EventUserPromptSubmit, received.Type)
require.Equal(t, "edited", decodeHookData[agenthooks.UserPromptSubmitData](t, received).Prompt)
stored, err := db.GetChatQueuedMessageByID(ctx, database.GetChatQueuedMessageByIDParams{ID: queued.ID, ChatID: chat.ID})
require.NoError(t, err)
require.Equal(t, []codersdk.ChatMessagePart{
codersdk.ChatMessageText("queued edit override"),
{Type: codersdk.ChatMessagePartTypeHookContext, Text: "edit context"},
{Type: codersdk.ChatMessagePartTypeHookNotice, Text: "edit notice"},
}, queuedMessageParts(t, stored))
})
t.Run("deny", func(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
chat, queued := seedErroredChatWithQueuedMessage(t, db, "original")
consumer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, err := w.Write([]byte(`{"permission":{"decision":"deny"},"user_message":"blocked"}`))
require.NoError(t, err)
}))
t.Cleanup(consumer.Close)
server := newHookTestServer(t, db, ps, consumer)
err := server.EditQueuedMessage(ctx, chatd.EditQueuedMessageOptions{
ChatID: chat.ID,
QueuedMessageID: queued.ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("blocked edit")},
})
var denied *chathooks.UserPromptDeniedError
require.ErrorAs(t, err, &denied)
require.Equal(t, "blocked", denied.UserMessage)
stored, err := db.GetChatQueuedMessageByID(ctx, database.GetChatQueuedMessageByIDParams{ID: queued.ID, ChatID: chat.ID})
require.NoError(t, err)
require.Equal(t, []codersdk.ChatMessagePart{codersdk.ChatMessageText("original")}, queuedMessageParts(t, stored),
"a denied edit leaves the queued content unchanged")
})
}
// TestEditQueuedMessageUserPromptSubmitDispatchFailure: the hook
// consumer removes the edited row and idles the chat before failing, so
// the chat is waiting when the dispatch error returns. The failed edit
// must not move that chat to error.
func TestEditQueuedMessageUserPromptSubmitDispatchFailure(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
chat, queued := seedErroredChatWithQueuedMessage(t, db, "original")
consumer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, err := db.DeleteChatQueuedMessageReturningCount(r.Context(), database.DeleteChatQueuedMessageReturningCountParams{
ID: queued.ID, ChatID: chat.ID,
})
require.NoError(t, err)
_, err = db.UpdateChatStatus(r.Context(), database.UpdateChatStatusParams{
ID: chat.ID,
Status: database.ChatStatusWaiting,
})
require.NoError(t, err)
w.WriteHeader(http.StatusInternalServerError)
}))
t.Cleanup(consumer.Close)
server := newHookTestServer(t, db, ps, consumer)
err := server.EditQueuedMessage(ctx, chatd.EditQueuedMessageOptions{
ChatID: chat.ID,
QueuedMessageID: queued.ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("edited")},
})
var dispatchErr *dispatch.Error
require.ErrorAs(t, err, &dispatchErr)
updated, err := db.GetChatByID(ctx, chat.ID)
require.NoError(t, err)
require.Equal(t, database.ChatStatusWaiting, updated.Status)
require.False(t, updated.LastError.Valid)
}
func TestEditMessageInvalidTargetSkipsHooks(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
user, org, model := seedChatDependencies(t, db)
chat := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: model.ID,
})
var dispatched atomic.Int32
consumer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
dispatched.Add(1)
_, err := w.Write([]byte(`{}`))
require.NoError(t, err)
}))
t.Cleanup(consumer.Close)
server := newHookTestServer(t, db, ps, consumer)
_, err := server.EditMessage(ctx, chatd.EditMessageOptions{
ChatID: chat.ID,
CreatedBy: user.ID,
EditedMessageID: 999999,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("edit of nothing")},
})
require.ErrorIs(t, err, chatd.ErrEditedMessageNotFound)
// dbgen.Chat ignores seed.Archived; archive explicitly.
archived := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: model.ID,
})
_, err = db.ArchiveChatByID(ctx, archived.ID)
require.NoError(t, err)
_, err = server.SendMessage(ctx, chatd.SendMessageOptions{
ChatID: archived.ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("send to archived")},
})
require.ErrorIs(t, err, chatd.ErrChatArchived)
_, err = server.EditMessage(ctx, chatd.EditMessageOptions{
ChatID: archived.ID,
CreatedBy: user.ID,
EditedMessageID: 1,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("edit archived")},
})
require.ErrorIs(t, err, chatd.ErrChatArchived)
require.Zero(t, dispatched.Load(), "invalid targets must not dispatch hooks")
}
func TestPromptHooksAdmissionPreflight(t *testing.T) {
t.Parallel()
// newPreflightServer returns a server whose hook consumer reports every
// dispatched request on the returned channel.
newPreflightServer := func(t *testing.T, limits chatd.Limits) (database.Store, *chatd.Server, <-chan agenthooks.Request) {
t.Helper()
db, ps := dbtestutil.NewDB(t)
received := make(chan agenthooks.Request, 8)
consumer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var request agenthooks.Request
if !assert.NoError(t, json.NewDecoder(r.Body).Decode(&request)) {
w.WriteHeader(http.StatusBadRequest)
return
}
received <- request
_, err := w.Write([]byte(`{}`))
assert.NoError(t, err)
}))
t.Cleanup(consumer.Close)
server := newTestServer(t, db, ps, uuid.New(), func(cfg *chatd.Config) {
cfg.HookDispatcher = newHookDispatcher(t, db, consumer)
cfg.Limits = limits
})
return db, server, received
}
t.Run("InvalidModel", func(t *testing.T) {
t.Parallel()
db, server, received := newPreflightServer(t, chatd.Limits{})
user, org, model := seedChatDependencies(t, db)
chat := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: model.ID,
})
ctx := testutil.Context(t, testutil.WaitLong)
_, err := server.SendMessage(ctx, chatd.SendMessageOptions{
ChatID: chat.ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("bad model")},
ModelConfigID: uuid.New(),
})
require.ErrorIs(t, err, chatd.ErrInvalidModelConfigID)
content, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{codersdk.ChatMessageText("original")})
require.NoError(t, err)
inserted, err := db.InsertChatMessages(ctx, singleChatMessageInsertParams(
chat.ID, database.ChatMessageRoleUser, content, model.ID, user.ID,
))
require.NoError(t, err)
require.Len(t, inserted, 1)
_, err = server.EditMessage(ctx, chatd.EditMessageOptions{
ChatID: chat.ID,
CreatedBy: user.ID,
EditedMessageID: inserted[0].ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("bad model edit")},
ModelConfigID: uuid.New(),
})
require.ErrorIs(t, err, chatd.ErrInvalidModelConfigID)
select {
case request := <-received:
t.Fatalf("admission-rejected prompt dispatched %s", request.Type)
default:
}
})
queueTests := []struct {
name string
limits chatd.Limits
maxQueued int
}{
{name: "QueueFullDefault", maxQueued: codersdk.DefaultChatMaxQueuedMessagesPerChat},
{name: "QueueFullConfigured2", limits: chatd.Limits{MaxQueuedMessagesPerChat: 2}, maxQueued: 2},
}
for _, tt := range queueTests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
db, server, received := newPreflightServer(t, tt.limits)
user, org, model := seedChatDependencies(t, db)
busy := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: model.ID,
Status: database.ChatStatusRunning,
})
ctx := testutil.Context(t, testutil.WaitLong)
queuedContent, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{codersdk.ChatMessageText("queued")})
require.NoError(t, err)
for range tt.maxQueued - 1 {
_, err = db.InsertChatQueuedMessageWithCreator(ctx, database.InsertChatQueuedMessageWithCreatorParams{
ChatID: busy.ID,
Content: queuedContent.RawMessage,
ModelConfigID: uuid.NullUUID{UUID: model.ID, Valid: true},
CreatedBy: user.ID,
BusyBehavior: database.ChatBusyBehaviorQueue,
})
require.NoError(t, err)
}
result, err := server.SendMessage(ctx, chatd.SendMessageOptions{
ChatID: busy.ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("last free slot")},
})
require.NoError(t, err)
require.True(t, result.Queued)
request := testutil.RequireReceive(ctx, t, received)
require.Equal(t, agenthooks.EventUserPromptSubmit, request.Type)
_, err = server.SendMessage(ctx, chatd.SendMessageOptions{
ChatID: busy.ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("queue full")},
})
var queueFull *chatstate.MessageQueueFullError
require.ErrorAs(t, err, &queueFull)
require.EqualValues(t, tt.maxQueued, queueFull.Max)
select {
case request := <-received:
t.Fatalf("admission-rejected prompt dispatched %s", request.Type)
default:
}
})
}
}
func TestSendMessageHooksDisabled(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
user, org, model := seedChatDependencies(t, db)
chat := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: model.ID,
})
server := newTestServer(t, db, ps, uuid.New())
result, err := server.SendMessage(ctx, chatd.SendMessageOptions{
ChatID: chat.ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("unchanged")},
})
require.NoError(t, err)
require.Equal(t, "unchanged", hookMessageText(t, result.Message))
}
func hookMessageText(t *testing.T, message database.ChatMessage) string {
t.Helper()
parts, err := chatprompt.ParseContent(message)
require.NoError(t, err)
require.Len(t, parts, 1)
return parts[0].Text
}
func decodeHookData[T any](t *testing.T, request agenthooks.Request) T {
t.Helper()
var data T
require.NoError(t, json.Unmarshal(request.Data, &data))
return data
}