Files
coder/coderd/x/chatd/configcache.go
T
Ethan 4afd834ad2 fix(coderd/x/chatd): return database errors from turn preparation (#30492)
Relates to CODAGT-1330

When chatd prepares a turn, it reads a bunch of things from the
database: the user's custom prompt, plan mode instructions, personal
skills, the advisor config, the user's compaction threshold override,
MCP server configs, inline MCP servers, MCP user tokens and the pinned
workspace MCP tools. If any of those reads failed, we'd log a warning
and run the turn without them, so the model would quietly ignore the
user's instructions or be missing tools for that turn.

To fix, we'll just return the error instead, so the prepare retry runs,
and the chat shows an error if the reads keep failing. The provider
switch history sanitizer and the cancel of deleted tool calls also
return their database errors now. A model config that's unavailable,
e.g. deleted, is still treated as foreign by the sanitizer like it is
today.

An inline MCP server with invalid stored headers still shows up as a
failed connect in the chat debug panel, but a failed read of the inline
MCP servers themselves now fails the turn.

This PR is in a stack on top of #30449.
2026-10-12 09:53:45 +11:00

342 lines
8.7 KiB
Go

package chatd
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"slices"
"sync"
"time"
"github.com/ammario/tlru"
"github.com/google/uuid"
"tailscale.com/util/singleflight"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/quartz"
)
const (
chatConfigProvidersTTL = 10 * time.Second
chatConfigUserPromptTTL = 5 * time.Second
chatConfigAdvisorConfigTTL = 10 * time.Second
// Bound user-prompt cache cardinality so one-shot users do not
// accumulate forever in long-lived chatd processes.
chatConfigUserPromptEntryLimit = 64 * 1024
)
type cachedProviders struct {
providers []database.AIProvider
expiresAt time.Time
}
type advisorRuntimeConfig struct {
Enabled bool `json:"enabled"`
MaxUsesPerRun int `json:"max_uses_per_run"`
MaxOutputTokens int64 `json:"max_output_tokens"`
}
type cachedAdvisorConfig struct {
config advisorRuntimeConfig
expiresAt time.Time
}
type chatConfigCache struct {
db database.Store
clock quartz.Clock
// ctx is the server-scoped context used for all DB fills.
// Cache fills run inside singleflight.Do where one caller
// becomes the leader for all coalesced waiters. Using a
// per-request context would mean the leader's cancellation
// (timeout, user disconnect) fans the error to every waiter.
// Storing the server context here makes that impossible by
// construction — callers cannot pass a request context into
// the shared fill path.
ctx context.Context
mu sync.RWMutex
// Providers (singleton).
providers *cachedProviders
providerGeneration uint64
providerFetches singleflight.Group[string, []database.AIProvider]
// User custom prompts (keyed by user ID).
userPromptEpoch uint64
userPrompts *tlru.Cache[uuid.UUID, string]
userPromptFetches singleflight.Group[string, string]
// Advisor configuration (singleton).
advisorConfig *cachedAdvisorConfig
advisorConfigGeneration uint64
advisorConfigFetches singleflight.Group[string, advisorRuntimeConfig]
}
func newChatConfigCache(ctx context.Context, db database.Store, clock quartz.Clock) *chatConfigCache {
return &chatConfigCache{
db: db,
clock: clock,
ctx: ctx,
userPrompts: tlru.New[uuid.UUID](
tlru.ConstantCost[string],
chatConfigUserPromptEntryLimit,
),
}
}
// singleflightDoChan wraps a singleflight group's DoChan method,
// allowing the caller to abandon the wait if their context is
// canceled while the shared fill continues running to completion.
// This separates two lifetimes: the fill runs under the server-scoped
// context, while each caller waits under its own request-scoped context.
func singleflightDoChan[K comparable, V any](
ctx context.Context,
group *singleflight.Group[K, V],
key K,
fn func() (V, error),
) (V, error) {
ch := group.DoChan(key, fn)
select {
case <-ctx.Done():
var zero V
return zero, ctx.Err()
case res := <-ch:
return res.Val, res.Err
}
}
func (c *chatConfigCache) EnabledProviders(ctx context.Context) ([]database.AIProvider, error) {
if providers, ok := c.cachedProviders(); ok {
return providers, nil
}
generation := c.providersGeneration()
providers, err := singleflightDoChan(
ctx,
&c.providerFetches,
fmt.Sprintf("%d:providers", generation),
func() ([]database.AIProvider, error) {
if cached, ok := c.cachedProviders(); ok {
return cached, nil
}
fetched, err := c.db.GetAIProviders(c.ctx, database.GetAIProvidersParams{})
if err != nil {
return nil, err
}
c.storeProviders(generation, fetched)
return slices.Clone(fetched), nil
},
)
if err != nil {
return nil, err
}
return slices.Clone(providers), nil
}
func (c *chatConfigCache) cachedProviders() ([]database.AIProvider, bool) {
c.mu.RLock()
entry := c.providers
c.mu.RUnlock()
if entry == nil {
return nil, false
}
if c.clock.Now().Before(entry.expiresAt) {
return slices.Clone(entry.providers), true
}
c.mu.Lock()
if current := c.providers; current != nil && !c.clock.Now().Before(current.expiresAt) {
c.providers = nil
}
c.mu.Unlock()
return nil, false
}
func (c *chatConfigCache) providersGeneration() uint64 {
c.mu.RLock()
generation := c.providerGeneration
c.mu.RUnlock()
return generation
}
func (c *chatConfigCache) storeProviders(generation uint64, providers []database.AIProvider) {
c.mu.Lock()
defer c.mu.Unlock()
if c.providerGeneration != generation {
return
}
c.providers = &cachedProviders{
providers: slices.Clone(providers),
expiresAt: c.clock.Now().Add(chatConfigProvidersTTL),
}
}
func (c *chatConfigCache) InvalidateProviders() {
c.mu.Lock()
c.providers = nil
c.providerGeneration++
c.mu.Unlock()
}
func (c *chatConfigCache) UserPrompt(ctx context.Context, userID uuid.UUID) (string, error) {
if prompt, ok := c.cachedUserPrompt(userID); ok {
return prompt, nil
}
epoch := c.currentUserPromptEpoch()
prompt, err := singleflightDoChan(ctx, &c.userPromptFetches, fmt.Sprintf("%d:%s", epoch, userID), func() (string, error) {
if cached, ok := c.cachedUserPrompt(userID); ok {
return cached, nil
}
fetched, err := c.db.GetUserChatCustomPrompt(c.ctx, userID)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
c.storeUserPrompt(epoch, userID, "")
return "", nil
}
return "", err
}
c.storeUserPrompt(epoch, userID, fetched)
return fetched, nil
})
if err != nil {
return "", err
}
return prompt, nil
}
func (c *chatConfigCache) cachedUserPrompt(userID uuid.UUID) (string, bool) {
prompt, _, ok := c.userPrompts.Get(userID)
if !ok {
return "", false
}
return prompt, true
}
func (c *chatConfigCache) currentUserPromptEpoch() uint64 {
c.mu.RLock()
epoch := c.userPromptEpoch
c.mu.RUnlock()
return epoch
}
func (c *chatConfigCache) storeUserPrompt(epoch uint64, userID uuid.UUID, prompt string) {
c.mu.Lock()
defer c.mu.Unlock()
if c.userPromptEpoch != epoch {
return
}
c.userPrompts.Set(userID, prompt, chatConfigUserPromptTTL)
}
func (c *chatConfigCache) InvalidateUserPrompt(userID uuid.UUID) {
c.mu.Lock()
c.userPrompts.Delete(userID)
c.userPromptEpoch++
c.mu.Unlock()
}
// InvalidateAdvisorConfig drops the cached advisor configuration so the
// next AdvisorConfig call re-fetches from the database. Called from the
// ChatConfigEvent subscriber after an admin writes
// PUT /api/experimental/chats/config/advisor; without this the cache
// could serve stale enabled/model/limits for up to
// chatConfigAdvisorConfigTTL. Bumping the generation counter also
// discards any in-flight fill started before the invalidation, so a
// stale DB read cannot re-cache the pre-update value.
func (c *chatConfigCache) InvalidateAdvisorConfig() {
c.mu.Lock()
c.advisorConfig = nil
c.advisorConfigGeneration++
c.mu.Unlock()
}
// AdvisorConfig returns the deployment-wide advisor configuration. The
// underlying site-config row changes on the order of hours or days, so
// this cache saves a per-turn DB round trip on chats that reference the
// advisor. Parse errors and lookup errors are surfaced to the caller.
func (c *chatConfigCache) AdvisorConfig(ctx context.Context) (advisorRuntimeConfig, error) {
if config, ok := c.cachedAdvisorConfig(); ok {
return config, nil
}
generation := c.advisorConfigGenerationSnapshot()
config, err := singleflightDoChan(
ctx,
&c.advisorConfigFetches,
fmt.Sprintf("%d:advisor", generation),
func() (advisorRuntimeConfig, error) {
if cached, ok := c.cachedAdvisorConfig(); ok {
return cached, nil
}
raw, err := c.db.GetChatAdvisorConfig(c.ctx)
if err != nil {
return advisorRuntimeConfig{}, err
}
var cfg advisorRuntimeConfig
if err := json.Unmarshal([]byte(raw), &cfg); err != nil {
return advisorRuntimeConfig{}, err
}
c.storeAdvisorConfig(generation, cfg)
return cfg, nil
},
)
if err != nil {
return advisorRuntimeConfig{}, err
}
return config, nil
}
func (c *chatConfigCache) cachedAdvisorConfig() (advisorRuntimeConfig, bool) {
c.mu.RLock()
entry := c.advisorConfig
c.mu.RUnlock()
if entry == nil {
return advisorRuntimeConfig{}, false
}
if c.clock.Now().Before(entry.expiresAt) {
return entry.config, true
}
c.mu.Lock()
if current := c.advisorConfig; current != nil && !c.clock.Now().Before(current.expiresAt) {
c.advisorConfig = nil
}
c.mu.Unlock()
return advisorRuntimeConfig{}, false
}
func (c *chatConfigCache) advisorConfigGenerationSnapshot() uint64 {
c.mu.RLock()
generation := c.advisorConfigGeneration
c.mu.RUnlock()
return generation
}
func (c *chatConfigCache) storeAdvisorConfig(generation uint64, config advisorRuntimeConfig) {
c.mu.Lock()
defer c.mu.Unlock()
if c.advisorConfigGeneration != generation {
return
}
c.advisorConfig = &cachedAdvisorConfig{
config: config,
expiresAt: c.clock.Now().Add(chatConfigAdvisorConfigTTL),
}
}