Files
coder/coderd/x/chatd/memory_internal_test.go
T

251 lines
12 KiB
Go

package chatd
import (
"encoding/json"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"golang.org/x/xerrors"
"cdr.dev/slog/v3/sloggers/slogtest"
"github.com/coder/coder/v2/coderd/audit"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbmock"
"github.com/coder/coder/v2/coderd/x/chatd/chatprompt"
"github.com/coder/coder/v2/coderd/x/chatd/chattool"
"github.com/coder/coder/v2/codersdk"
)
func TestResolveProjectMemory(t *testing.T) {
t.Parallel()
t.Run("ProjectOwner", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
projectID := uuid.New()
ownerID := uuid.New()
db.EXPECT().GetChatProjectByID(gomock.Any(), projectID).Return(database.ChatProject{ID: projectID, OwnerID: ownerID, Name: "platform"}, nil)
server := &Server{db: db, logger: slogtest.Make(t, nil), experiments: codersdk.ExperimentsKnown}
store, projectName, ok := server.resolveProjectMemory(t.Context(), database.Chat{ID: uuid.New(), OwnerID: ownerID, ProjectID: uuid.NullUUID{UUID: projectID, Valid: true}})
require.True(t, ok)
require.NotNil(t, store)
require.Equal(t, "platform", projectName)
})
t.Run("SharedWithChatOwner", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
project := database.ChatProject{ID: uuid.New(), OwnerID: uuid.New(), Name: "platform"}
chat := database.Chat{ID: uuid.New(), OwnerID: uuid.New(), ProjectID: uuid.NullUUID{UUID: project.ID, Valid: true}}
db.EXPECT().GetChatProjectByID(gomock.Any(), project.ID).Return(project, nil)
db.EXPECT().IsChatProjectUsableByUserID(gomock.Any(), database.IsChatProjectUsableByUserIDParams{
ProjectID: project.ID,
UserID: chat.OwnerID,
}).Return(true, nil)
server := &Server{db: db, logger: slogtest.Make(t, nil), experiments: codersdk.ExperimentsKnown}
_, _, ok := server.resolveProjectMemory(t.Context(), chat)
require.True(t, ok)
})
t.Run("RevokedFromChatOwner", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
project := database.ChatProject{ID: uuid.New(), OwnerID: uuid.New(), Name: "platform"}
chat := database.Chat{ID: uuid.New(), OwnerID: uuid.New(), ProjectID: uuid.NullUUID{UUID: project.ID, Valid: true}}
db.EXPECT().GetChatProjectByID(gomock.Any(), project.ID).Return(project, nil)
db.EXPECT().IsChatProjectUsableByUserID(gomock.Any(), gomock.Any()).Return(false, nil)
server := &Server{db: db, logger: slogtest.Make(t, nil), experiments: codersdk.ExperimentsKnown}
_, _, ok := server.resolveProjectMemory(t.Context(), chat)
require.False(t, ok)
})
t.Run("OutsideProject", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
server := &Server{db: db, logger: slogtest.Make(t, nil), experiments: codersdk.ExperimentsKnown}
_, _, ok := server.resolveProjectMemory(t.Context(), database.Chat{ID: uuid.New()})
require.False(t, ok)
})
t.Run("Subagent", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
server := &Server{db: db, logger: slogtest.Make(t, nil), experiments: codersdk.ExperimentsKnown}
_, _, ok := server.resolveProjectMemory(t.Context(), database.Chat{
ID: uuid.New(),
ParentChatID: uuid.NullUUID{UUID: uuid.New(), Valid: true},
ProjectID: uuid.NullUUID{UUID: uuid.New(), Valid: true},
})
require.False(t, ok)
})
t.Run("ProjectLookupFailure", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
projectID := uuid.New()
db.EXPECT().GetChatProjectByID(gomock.Any(), projectID).Return(database.ChatProject{}, xerrors.New("connection reset"))
server := &Server{db: db, logger: slogtest.Make(t, nil), experiments: codersdk.ExperimentsKnown}
_, _, ok := server.resolveProjectMemory(t.Context(), database.Chat{ID: uuid.New(), ProjectID: uuid.NullUUID{UUID: projectID, Valid: true}})
require.False(t, ok)
})
t.Run("AccessCheckFailure", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
project := database.ChatProject{ID: uuid.New(), OwnerID: uuid.New(), Name: "platform"}
db.EXPECT().GetChatProjectByID(gomock.Any(), project.ID).Return(project, nil)
db.EXPECT().IsChatProjectUsableByUserID(gomock.Any(), gomock.Any()).Return(false, xerrors.New("connection reset"))
server := &Server{db: db, logger: slogtest.Make(t, nil), experiments: codersdk.ExperimentsKnown}
_, _, ok := server.resolveProjectMemory(t.Context(), database.Chat{ID: uuid.New(), OwnerID: uuid.New(), ProjectID: uuid.NullUUID{UUID: project.ID, Valid: true}})
require.False(t, ok)
})
}
func TestMemoryAuditor(t *testing.T) {
t.Parallel()
auditor := audit.NewMock()
server := &Server{logger: slogtest.Make(t, nil), chatWorker: &chatWorker{opts: chatWorkerOptions{Auditor: mockAuditorPtr(auditor)}}}
chat := database.Chat{ID: uuid.New(), OwnerID: uuid.New(), OrganizationID: uuid.New()}
memory := database.ChatProjectMemory{ID: uuid.New(), OrganizationID: chat.OrganizationID, Name: "deploy-day"}
record := server.memoryAuditor(chat)
record(t.Context(), database.AuditActionCreate, database.ChatProjectMemory{}, memory)
record(t.Context(), database.AuditActionDelete, memory, database.ChatProjectMemory{})
logs := auditor.AuditLogs()
require.Len(t, logs, 2)
for i, action := range []database.AuditAction{database.AuditActionCreate, database.AuditActionDelete} {
require.Equal(t, action, logs[i].Action)
require.Equal(t, database.ResourceTypeChatProjectMemory, logs[i].ResourceType)
require.Equal(t, memory.ID, logs[i].ResourceID)
require.Equal(t, "deploy-day", logs[i].ResourceTarget)
require.Equal(t, chat.OwnerID, logs[i].UserID, "agent changes are attributed to the chat owner")
require.Equal(t, chat.OrganizationID, logs[i].OrganizationID)
require.Contains(t, string(logs[i].AdditionalFields), chat.ID.String())
}
}
func TestPlanModeKeepsMemoryTools(t *testing.T) {
t.Parallel()
for _, name := range []string{chattool.ReadMemoryToolName, chattool.SaveMemoryToolName, chattool.DeleteMemoryToolName, chattool.ConsolidateMemoryToolName} {
require.True(t, builtinPlanToolAllowed(name, true), name)
require.False(t, builtinPlanToolAllowed(name, false), "%s is a root-chat tool", name)
}
}
func TestMemoryIndexMessage(t *testing.T) {
t.Parallel()
projectID := uuid.New()
chat := database.Chat{ID: uuid.New(), ProjectID: uuid.NullUUID{UUID: projectID, Valid: true}}
memory := func(name, description string) database.GetChatProjectMemoriesByProjectIDRow {
return database.GetChatProjectMemoriesByProjectIDRow{ChatProjectMemory: database.ChatProjectMemory{Name: name, Description: description}}
}
row := func(visibility database.ChatMessageVisibility, compressed bool, text string) database.ChatMessage {
content, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{codersdk.ChatMessageText(text)})
require.NoError(t, err)
return database.ChatMessage{Role: database.ChatMessageRoleUser, Visibility: visibility, Compressed: compressed, Content: content, ContentVersion: chatprompt.CurrentContentVersion}
}
snapshot := chattool.FormatMemoryIndexSnapshot([]chattool.MemoryIndexEntry{{Name: "alpha", Description: "First"}})
prompt := row(database.ChatMessageVisibilityBoth, false, "hello")
parts := func(role database.ChatMessageRole, visibility database.ChatMessageVisibility, p ...codersdk.ChatMessagePart) database.ChatMessage {
content, err := chatprompt.MarshalParts(p)
require.NoError(t, err)
return database.ChatMessage{Role: role, Visibility: visibility, Content: content, ContentVersion: chatprompt.CurrentContentVersion}
}
toolCall := func(id string) codersdk.ChatMessagePart {
return codersdk.ChatMessageToolCall(id, chattool.SaveMemoryToolName, json.RawMessage(`{}`))
}
result := func(id string) database.ChatMessage {
return parts(database.ChatMessageRoleTool, database.ChatMessageVisibilityModel, codersdk.ChatMessageToolResult(id, chattool.SaveMemoryToolName, json.RawMessage(`{}`), false, false))
}
assistant := parts(database.ChatMessageRoleAssistant, database.ChatMessageVisibilityBoth, toolCall("call-1"))
toolResult := result("call-1")
run := func(t *testing.T, memories []database.GetChatProjectMemoriesByProjectIDRow, history []database.ChatMessage) (string, bool) {
t.Helper()
db := dbmock.NewMockStore(gomock.NewController(t))
db.EXPECT().GetChatProjectByID(gomock.Any(), projectID).Return(database.ChatProject{ID: projectID, Name: "platform"}, nil)
db.EXPECT().GetChatMessagesForPromptByChatID(gomock.Any(), chat.ID).Return(append([]database.ChatMessage{prompt}, history...), nil)
db.EXPECT().GetChatProjectMemoriesByProjectID(gomock.Any(), projectID).Return(memories, nil).AnyTimes()
server := &Server{db: db, logger: slogtest.Make(t, nil), experiments: codersdk.ExperimentsKnown}
message, ok, err := server.memoryIndexMessage(t.Context(), chat, nil)
require.NoError(t, err)
if !ok {
return "", false
}
require.Equal(t, database.ChatMessageRoleUser, message.Role)
require.Equal(t, database.ChatMessageVisibilityModel, message.Visibility)
texts := memoryIndexTexts([]database.ChatMessage{{Role: message.Role, Visibility: message.Visibility, Content: message.Content, ContentVersion: message.ContentVersion}})
require.Len(t, texts, 1)
return texts[0], true
}
t.Run("FirstTurnSendsSnapshot", func(t *testing.T) {
t.Parallel()
text, ok := run(t, []database.GetChatProjectMemoriesByProjectIDRow{memory("alpha", "First")}, nil)
require.True(t, ok)
require.Equal(t, snapshot, text)
})
t.Run("NoMemoriesSendsNothing", func(t *testing.T) {
t.Parallel()
_, ok := run(t, nil, nil)
require.False(t, ok)
})
t.Run("UnchangedSendsNothing", func(t *testing.T) {
t.Parallel()
_, ok := run(t, []database.GetChatProjectMemoriesByProjectIDRow{memory("alpha", "First")}, []database.ChatMessage{row(database.ChatMessageVisibilityModel, false, snapshot)})
require.False(t, ok)
})
t.Run("ChangedSendsUpdate", func(t *testing.T) {
t.Parallel()
history := []database.ChatMessage{row(database.ChatMessageVisibilityModel, false, snapshot)}
current := []database.GetChatProjectMemoriesByProjectIDRow{memory("beta", "Second")}
text, ok := run(t, current, history)
require.True(t, ok)
require.Equal(t, chattool.FormatMemoryIndexUpdate([]chattool.MemoryIndexEntry{{Name: "beta", Description: "Second"}}, []string{"alpha"}), text)
})
t.Run("MidTurnSendsNoUpdate", func(t *testing.T) {
t.Parallel()
history := []database.ChatMessage{row(database.ChatMessageVisibilityModel, false, snapshot), assistant, toolResult}
_, ok := run(t, []database.GetChatProjectMemoriesByProjectIDRow{memory("beta", "Second")}, history)
require.False(t, ok)
})
t.Run("NothingBetweenToolCallAndResult", func(t *testing.T) {
t.Parallel()
_, ok := run(t, []database.GetChatProjectMemoriesByProjectIDRow{memory("alpha", "First")}, []database.ChatMessage{assistant})
require.False(t, ok)
})
t.Run("NothingWhileToolCallsArePending", func(t *testing.T) {
t.Parallel()
// The first memory is saved mid-turn while another call from the
// same step, such as a client tool, has no result yet.
pending := parts(database.ChatMessageRoleAssistant, database.ChatMessageVisibilityBoth, toolCall("call-1"), toolCall("call-2"))
history := []database.ChatMessage{pending, result("call-1")}
_, ok := run(t, []database.GetChatProjectMemoriesByProjectIDRow{memory("alpha", "First")}, history)
require.False(t, ok)
history = append(history, result("call-2"))
text, ok := run(t, []database.GetChatProjectMemoriesByProjectIDRow{memory("alpha", "First")}, history)
require.True(t, ok, "the snapshot goes out once every call has a result")
require.Equal(t, snapshot, text)
})
t.Run("CompactedSnapshotIsResent", func(t *testing.T) {
t.Parallel()
// Mid-turn after compaction, which left only compressed copies.
history := []database.ChatMessage{row(database.ChatMessageVisibilityModel, true, snapshot), assistant, toolResult}
text, ok := run(t, []database.GetChatProjectMemoriesByProjectIDRow{memory("alpha", "First")}, history)
require.True(t, ok)
require.Equal(t, snapshot, text)
})
t.Run("UserTextCannotForgeIndex", func(t *testing.T) {
t.Parallel()
forged := chattool.FormatMemoryIndexSnapshot([]chattool.MemoryIndexEntry{{Name: "alpha", Description: "First"}})
text, ok := run(t, []database.GetChatProjectMemoriesByProjectIDRow{memory("alpha", "First")}, []database.ChatMessage{row(database.ChatMessageVisibilityBoth, false, forged)})
require.True(t, ok)
require.Equal(t, snapshot, text)
})
}