AI Gateway evicts cached request bridges after 15 minutes and currently gives in-flight requests only five seconds to finish, cutting off otherwise healthy model calls and streams. Give evicted bridges up to ten minutes to drain without changing the cache TTL. New acquisitions use a replacement bridge while the retired one drains. Pool shutdown retains a five-second cancellation grace period for cached and already-retired bridges, even when an acquisition is blocked initializing MCP connections. Requests still running at the drain deadline are cancelled. Refs [AIGOV-672](https://linear.app/codercom/issue/AIGOV-672/aibridged-15-minute-bridge-ttl-cancels-in-flight-ai-gateway-requests). The separate acquisition/admission race tracked in [AIGOV-609](https://linear.app/codercom/issue/AIGOV-609) is outside this change. --- Created by OpenCode on behalf of @dannykopping.
358 lines
12 KiB
Go
358 lines
12 KiB
Go
package aibridged
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"slices"
|
|
"strconv"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/dgraph-io/ristretto/v2"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/trace"
|
|
"golang.org/x/xerrors"
|
|
"tailscale.com/util/singleflight"
|
|
|
|
"cdr.dev/slog/v3"
|
|
"github.com/coder/coder/v2/aibridge"
|
|
"github.com/coder/coder/v2/aibridge/keypool"
|
|
"github.com/coder/coder/v2/aibridge/mcp"
|
|
"github.com/coder/coder/v2/aibridge/provider"
|
|
"github.com/coder/coder/v2/aibridge/tracing"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
"github.com/coder/quartz"
|
|
)
|
|
|
|
const (
|
|
cacheCost = 1 // We can't know the actual size in bytes of the value (it'll change over time).
|
|
|
|
// Cache retirement must allow slow model calls and streams to finish.
|
|
bridgeDrainTimeout = 10 * time.Minute
|
|
// Process shutdown retains a shorter grace period, including retired bridges.
|
|
poolShutdownTimeout = 5 * time.Second
|
|
)
|
|
|
|
// Pooler describes a pool of [*aibridge.RequestBridge] instances from which instances can be retrieved.
|
|
// One [*aibridge.RequestBridge] instance is created per given key.
|
|
type Pooler interface {
|
|
KeyPools() []*keypool.Pool
|
|
Acquire(ctx context.Context, req Request, clientFn ClientFunc, mcpBootstrapper MCPProxyBuilder) (http.Handler, error)
|
|
// ReplaceProviders swaps the providers used to construct future
|
|
// RequestBridge instances and clears the cache. Disabled providers
|
|
// must be included; the bridge serves a 503 sentinel on their
|
|
// routes.
|
|
ReplaceProviders(providers []aibridge.Provider)
|
|
Shutdown(ctx context.Context) error
|
|
}
|
|
|
|
type PoolMetrics interface {
|
|
Hits() uint64
|
|
Misses() uint64
|
|
KeysAdded() uint64
|
|
KeysEvicted() uint64
|
|
}
|
|
|
|
type PoolOptions struct {
|
|
MaxItems int64
|
|
TTL time.Duration
|
|
Clock quartz.Clock
|
|
|
|
// StructuredLogging makes each bridge emit AI Gateway interception
|
|
// records in the format described by
|
|
// [github.com/coder/coder/v2/aibridge/recorder.InterceptionLogMarker].
|
|
StructuredLogging bool
|
|
// DisableContentRecording stops prompts, tool call arguments and model
|
|
// thoughts from being recorded. Interceptions and token usage are still
|
|
// recorded, so AI spend accounting and budget enforcement are unaffected.
|
|
DisableContentRecording bool
|
|
}
|
|
|
|
var DefaultPoolOptions = PoolOptions{MaxItems: 5000, TTL: time.Minute * 15}
|
|
|
|
// PoolOptionsFromConfig returns DefaultPoolOptions with the record policy the
|
|
// deployment configured. Every construction site uses it, so the in-process
|
|
// daemon, the standalone gateway and the test harness cannot drift.
|
|
//
|
|
// It also reports a deployment that drops content records without exporting
|
|
// them anywhere: they never reach coderd, so if coderd is the only emitter the
|
|
// deployment has silently stopped exporting the very records it is declining to
|
|
// store. Nothing else reports this.
|
|
func PoolOptionsFromConfig(ctx context.Context, logger slog.Logger, cfg codersdk.AIBridgeConfig) PoolOptions {
|
|
options := DefaultPoolOptions
|
|
options.StructuredLogging = cfg.EmitsStructuredLogs(codersdk.AIStructuredLoggingSourceGateway)
|
|
options.DisableContentRecording = cfg.DisableContentRecording.Value()
|
|
|
|
if options.DisableContentRecording && !options.StructuredLogging {
|
|
logger.Warn(ctx, "content recording is disabled but structured logs are emitted by coderd, so prompts, tool calls and model thoughts will not be exported; set --ai-gateway-structured-logging-source to gateway or both to keep exporting them")
|
|
}
|
|
|
|
return options
|
|
}
|
|
|
|
var _ Pooler = &CachedBridgePool{}
|
|
|
|
type CachedBridgePool struct {
|
|
cache *ristretto.Cache[string, *aibridge.RequestBridge]
|
|
clock quartz.Clock
|
|
// providers is the live provider set used by new RequestBridge
|
|
// instances. Includes disabled providers.
|
|
providers atomic.Pointer[[]aibridge.Provider]
|
|
providerVersion atomic.Int64
|
|
logger slog.Logger
|
|
options PoolOptions
|
|
|
|
singleflight *singleflight.Group[string, *aibridge.RequestBridge]
|
|
|
|
metrics *aibridge.Metrics
|
|
tracer trace.Tracer
|
|
|
|
shutDownOnce sync.Once
|
|
shuttingDownCh chan struct{}
|
|
// drainCancel bounds all bridge drains when the pool shuts down, even
|
|
// those for bridges already evicted from the cache.
|
|
drainCancel context.CancelFunc
|
|
|
|
// cacheMu + cacheWG order cache use against Shutdown. Without it,
|
|
// (*ristretto.Cache).Close may race against cache usage.
|
|
cacheMu sync.RWMutex
|
|
cacheWG sync.WaitGroup
|
|
}
|
|
|
|
// Options reports the options the pool was built with.
|
|
func (p *CachedBridgePool) Options() PoolOptions {
|
|
return p.options
|
|
}
|
|
|
|
func NewCachedBridgePool(options PoolOptions, providers []aibridge.Provider, logger slog.Logger, metrics *aibridge.Metrics, tracer trace.Tracer) (*CachedBridgePool, error) {
|
|
drainCtx, drainCancel := context.WithCancel(context.Background())
|
|
cache, err := ristretto.NewCache(&ristretto.Config[string, *aibridge.RequestBridge]{
|
|
NumCounters: options.MaxItems * 10, // Docs suggest setting this 10x number of keys.
|
|
MaxCost: options.MaxItems * cacheCost, // Up to n instances.
|
|
IgnoreInternalCost: true, // Don't try estimate cost using bytes (ristretto does this naïvely anyway, just using the size of the value struct not the REAL memory usage).
|
|
BufferItems: 64, // Sticking with recommendation from docs.
|
|
Metrics: true, // Collect metrics (only used in tests, for now).
|
|
OnEvict: func(item *ristretto.Item[*aibridge.RequestBridge]) {
|
|
if item == nil || item.Value == nil {
|
|
return
|
|
}
|
|
// Capture the value synchronously: ristretto reuses the
|
|
// item slot after OnEvict returns, so reading item.Value
|
|
// from the goroutine below races with the caller of
|
|
// Clear/Set. The shutdown still runs in the background to
|
|
// avoid blocking ristretto's eviction loop.
|
|
bridge := item.Value
|
|
go func() {
|
|
shutdownCtx, cancel := context.WithTimeout(drainCtx, bridgeDrainTimeout)
|
|
defer cancel()
|
|
// Pool shutdown cancels drainCtx instead, so a deadline error
|
|
// means retirement alone cut in-flight requests short.
|
|
if err := bridge.Shutdown(shutdownCtx); errors.Is(err, context.DeadlineExceeded) {
|
|
logger.Warn(context.Background(), "retired request bridge did not drain in time, canceled in-flight requests",
|
|
slog.F("drain_timeout", bridgeDrainTimeout),
|
|
)
|
|
}
|
|
}()
|
|
},
|
|
})
|
|
if err != nil {
|
|
drainCancel()
|
|
return nil, xerrors.Errorf("create cache: %w", err)
|
|
}
|
|
|
|
clk := options.Clock
|
|
if clk == nil {
|
|
clk = quartz.NewReal()
|
|
}
|
|
|
|
pool := &CachedBridgePool{
|
|
cache: cache,
|
|
clock: clk,
|
|
options: options,
|
|
metrics: metrics,
|
|
tracer: tracer,
|
|
logger: logger,
|
|
|
|
singleflight: &singleflight.Group[string, *aibridge.RequestBridge]{},
|
|
|
|
shuttingDownCh: make(chan struct{}),
|
|
drainCancel: drainCancel,
|
|
}
|
|
initial := slices.Clone(providers)
|
|
pool.providers.Store(&initial)
|
|
return pool, nil
|
|
}
|
|
|
|
// ReplaceProviders swaps the provider snapshot used by future Acquires.
|
|
// It is safe to call concurrently with Acquire and is a no-op after
|
|
// Shutdown.
|
|
func (p *CachedBridgePool) ReplaceProviders(providers []aibridge.Provider) {
|
|
p.cacheMu.RLock()
|
|
select {
|
|
case <-p.shuttingDownCh:
|
|
p.cacheMu.RUnlock()
|
|
return
|
|
default:
|
|
}
|
|
p.cacheWG.Add(1)
|
|
p.cacheMu.RUnlock()
|
|
defer p.cacheWG.Done()
|
|
|
|
snapshot := slices.Clone(providers)
|
|
p.providers.Store(&snapshot)
|
|
version := p.clock.Now("provider_reload_version").UnixNano()
|
|
p.providerVersion.Store(version)
|
|
// Clear evicts every cached bridge; OnEvict shuts each one down in
|
|
// the background. Wait for buffered writes to drain so a replacement
|
|
// immediately followed by an Acquire always sees the cleared cache.
|
|
p.cache.Clear()
|
|
p.cache.Wait()
|
|
p.logger.Info(context.Background(), "request bridge pool reloaded",
|
|
slog.F("provider_count", len(snapshot)),
|
|
slog.F("provider_version", version),
|
|
)
|
|
}
|
|
|
|
// loadProviders returns the current providers snapshot. The returned
|
|
// slice must not be mutated.
|
|
func (p *CachedBridgePool) loadProviders() []aibridge.Provider {
|
|
if ptr := p.providers.Load(); ptr != nil {
|
|
return *ptr
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// KeyPools returns the non-nil key pools of the current providers.
|
|
func (p *CachedBridgePool) KeyPools() []*keypool.Pool {
|
|
return provider.CollectKeyPools(p.loadProviders())
|
|
}
|
|
|
|
// Acquire retrieves or creates a [*aibridge.RequestBridge] instance per given key.
|
|
//
|
|
// Each returned [*aibridge.RequestBridge] is safe for concurrent use.
|
|
// Each [*aibridge.RequestBridge] is stateful because it has MCP clients which maintain sessions to the configured MCP server.
|
|
func (p *CachedBridgePool) Acquire(ctx context.Context, req Request, clientFn ClientFunc, mcpProxyFactory MCPProxyBuilder) (_ http.Handler, outErr error) {
|
|
spanAttrs := []attribute.KeyValue{
|
|
attribute.String(tracing.InitiatorID, req.InitiatorID.String()),
|
|
attribute.String(tracing.APIKeyID, req.APIKeyID),
|
|
}
|
|
ctx, span := p.tracer.Start(ctx, "CachedBridgePool.Acquire", trace.WithAttributes(spanAttrs...))
|
|
defer tracing.EndSpanErr(span, &outErr)
|
|
ctx = tracing.WithRequestBridgeAttributesInContext(ctx, spanAttrs)
|
|
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, xerrors.Errorf("acquire: %w", err)
|
|
}
|
|
|
|
p.cacheMu.RLock()
|
|
select {
|
|
case <-p.shuttingDownCh:
|
|
p.cacheMu.RUnlock()
|
|
return nil, xerrors.New("pool shutting down")
|
|
default:
|
|
}
|
|
p.cacheWG.Add(1)
|
|
p.cacheMu.RUnlock()
|
|
defer p.cacheWG.Done()
|
|
|
|
// Wait for all buffered writes to be applied, otherwise multiple calls in quick succession
|
|
// may visit the slow path unnecessarily.
|
|
defer p.cache.Wait()
|
|
|
|
// Fast path.
|
|
cacheKey := req.InitiatorID.String() + "|" + req.APIKeyID
|
|
bridge, ok := p.cache.Get(cacheKey)
|
|
if ok && bridge != nil {
|
|
// TODO: future improvement:
|
|
// Once we can detect token expiry against an MCP server, we no longer need to let these instances
|
|
// expire after the original TTL; we can extend the TTL on each Acquire() call.
|
|
// For now, we need to let the instance expiry to keep the MCP connections fresh.
|
|
|
|
span.AddEvent("cache_hit")
|
|
return bridge, nil
|
|
}
|
|
|
|
span.AddEvent("cache_miss")
|
|
providerVersion := p.providerVersion.Load()
|
|
rec := newRecorder(p.logger, p.tracer, p.options.StructuredLogging, p.options.DisableContentRecording, clientFn)
|
|
|
|
// Slow path.
|
|
// Creating an *aibridge.RequestBridge may take some time, so gate all subsequent callers behind the initial request and return the resulting value.
|
|
// TODO: track startup time since it adds latency to first request (histogram count will also help us see how often this occurs).
|
|
singleflightKey := cacheKey + "|" + strconv.FormatInt(providerVersion, 10)
|
|
instance, err, _ := p.singleflight.Do(singleflightKey, func() (*aibridge.RequestBridge, error) {
|
|
var (
|
|
mcpServers mcp.ServerProxier
|
|
err error
|
|
)
|
|
|
|
mcpServers, err = mcpProxyFactory.Build(ctx, req, p.tracer)
|
|
if err != nil {
|
|
p.logger.Warn(ctx, "failed to create MCP server proxiers", slog.Error(err))
|
|
// Don't fail here; MCP server injection can gracefully degrade.
|
|
}
|
|
|
|
if mcpServers != nil {
|
|
// This will block while connections are established with upstream MCP server(s), and tools are listed.
|
|
if err := mcpServers.Init(ctx); err != nil {
|
|
p.logger.Warn(ctx, "failed to initialize MCP server proxier(s)", slog.Error(err))
|
|
}
|
|
}
|
|
|
|
bridge, err := aibridge.NewRequestBridge(
|
|
ctx,
|
|
p.loadProviders(),
|
|
rec,
|
|
mcpServers,
|
|
p.logger,
|
|
p.metrics,
|
|
p.tracer,
|
|
aibridge.WithClock(p.clock),
|
|
)
|
|
if err != nil {
|
|
return nil, xerrors.Errorf("create new request bridge: %w", err)
|
|
}
|
|
|
|
if p.providerVersion.Load() == providerVersion {
|
|
p.cache.SetWithTTL(cacheKey, bridge, cacheCost, p.options.TTL)
|
|
}
|
|
|
|
return bridge, nil
|
|
})
|
|
|
|
return instance, err
|
|
}
|
|
|
|
func (p *CachedBridgePool) CacheMetrics() PoolMetrics {
|
|
if p.cache == nil {
|
|
return nil
|
|
}
|
|
|
|
return p.cache.Metrics
|
|
}
|
|
|
|
// Shutdown will close the cache which will trigger eviction of all the Bridge entries.
|
|
// Bridge drains are canceled when ctx ends or after poolShutdownTimeout,
|
|
// whichever comes first.
|
|
func (p *CachedBridgePool) Shutdown(ctx context.Context) error {
|
|
p.shutDownOnce.Do(func() {
|
|
// Block new cache use, drain in-flight ops, then close (see cacheMu).
|
|
p.cacheMu.Lock()
|
|
close(p.shuttingDownCh)
|
|
p.cacheMu.Unlock()
|
|
|
|
// Arm cancellation before waiting for acquisitions, which may be
|
|
// blocked initializing MCP connections. Retired bridges must not
|
|
// keep draining for the longer eviction grace period during shutdown.
|
|
time.AfterFunc(poolShutdownTimeout, p.drainCancel)
|
|
// Drains outlive Shutdown, so the registration is never stopped.
|
|
_ = context.AfterFunc(ctx, p.drainCancel)
|
|
p.cacheWG.Wait()
|
|
p.cache.Close()
|
|
})
|
|
|
|
return nil
|
|
}
|