feat: make the Coder Agents stream silence timeout configurable (#29394)
The `chatloop` stream silence guard cancels and retries a model attempt after `10m` of silence, and it was hard-coded. It's armed before the request opens, so it also bounds time to first token, and slow local models (e.g. `llama-server` on a large prompt) can spend longer than that on prompt processing alone, meaning every attempt just gets cancelled and retried. This PR adds `--chat-stream-silence-timeout` / `CODER_CHAT_STREAM_SILENCE_TIMEOUT` (default `10m`) and passes it to both the main loop and the advisor. Setting it to `0` disables the guard entirely, and anything above `24h` is rejected, which also keeps the watchdog arithmetic below from overflowing. Whilst doing this I ran into the runner's `15m` task watchdog, which kept counting during an open stream. That silently cancelled any healthy stream longer than `15m`, and would've stopped a silence timeout above `15m` from ever firing. To fix, the silence guard now kicks the watchdog out to `max(15m, silence + 5m)` whenever it arms or sees a part. Silent streams still fail through the guard, healthy streams can run for as long as they like, and hang detection is unchanged everywhere else. With the guard disabled there's nothing to kick against, so arming it just stops the watchdog for the rest of the attempt. That does mean the DB write after the stream isn't bounded either when `0` is set; I think that's fine for an opt-in setting, and a user can still stop the chat from the UI if it wedges there. I did consider removing the guard entirely, but the provider HTTP clients have no timeout, so a provider that holds the connection open would only ever be caught as a generic task hang, which is what #25782 was fixing.
This commit is contained in:
1 parent
cbaeb099a4
commit
77479408b4
20 files changed
+233
-43
No files matched your search
+5
@@ -222,6 +222,11 @@ Configure the background chat processing daemon.
|
||||
Force chat debug logging on for every chat, bypassing the runtime
|
||||
admin and user opt-in settings.
|
||||
|
||||
--chat-stream-silence-timeout duration, $CODER_CHAT_STREAM_SILENCE_TIMEOUT (default: 10m0s)
|
||||
Maximum time to wait for the next streamed part from the chat model
|
||||
before the attempt is canceled and retried. This also bounds the time
|
||||
to first token. Set to 0 to disable. Must be no more than 24h.
|
||||
|
||||
CLIENT OPTIONS:
|
||||
These options change the behavior of how clients interact with the Coder.
|
||||
Clients include the Coder CLI, Coder Desktop, IDE extensions, and the web UI.
|
||||
|
||||
+5
@@ -855,6 +855,11 @@ chat:
|
||||
# has no effect. This option will be removed in a future release.
|
||||
# (default: true, type: bool)
|
||||
aiGatewayRoutingEnabled: true
|
||||
# Maximum time to wait for the next streamed part from the chat model before the
|
||||
# attempt is canceled and retried. This also bounds the time to first token. Set
|
||||
# to 0 to disable. Must be no more than 24h.
|
||||
# (default: 10m0s, type: duration)
|
||||
streamSilenceTimeout: 10m0s
|
||||
aibridge:
|
||||
# Deprecated: use --ai-gateway-enabled or CODER_AI_GATEWAY_ENABLED instead.
|
||||
# Whether to start an in-memory aibridged instance.
|
||||
|
||||
Generated
+3
@@ -19764,6 +19764,9 @@ const docTemplate = `{
|
||||
},
|
||||
"hook_url": {
|
||||
"$ref": "#/definitions/serpent.URL"
|
||||
},
|
||||
"stream_silence_timeout": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
Generated
+3
@@ -17774,6 +17774,9 @@
|
||||
},
|
||||
"hook_url": {
|
||||
"$ref": "#/definitions/serpent.URL"
|
||||
},
|
||||
"stream_silence_timeout": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -101,6 +101,7 @@ import (
|
||||
"github.com/coder/coder/v2/coderd/wsbuildorchestrator"
|
||||
"github.com/coder/coder/v2/coderd/x/agenthooks/dispatch"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/mcpclient"
|
||||
"github.com/coder/coder/v2/coderd/x/gitsync"
|
||||
@@ -890,6 +891,10 @@ func New(options *Options) *API {
|
||||
if maxChatsPerAcquire < math.MinInt32 {
|
||||
maxChatsPerAcquire = math.MinInt32
|
||||
}
|
||||
streamSilenceTimeout := options.DeploymentValues.AI.Chat.StreamSilenceTimeout.Value()
|
||||
if streamSilenceTimeout == 0 {
|
||||
streamSilenceTimeout = chatloop.StreamSilenceTimeoutDisabled
|
||||
}
|
||||
|
||||
var oidcMCPSrc mcpclient.UserOIDCTokenSource
|
||||
if options.OIDCConfig != nil {
|
||||
@@ -946,6 +951,7 @@ func New(options *Options) *API {
|
||||
AllowBYOKSet: true,
|
||||
AIBridgeTransportFactory: &api.AIBridgeTransportFactory,
|
||||
AlwaysEnableDebugLogs: options.DeploymentValues.AI.Chat.DebugLoggingEnabled.Value(),
|
||||
StreamSilenceTimeout: streamSilenceTimeout,
|
||||
Experiments: experiments,
|
||||
AgentConn: api.agentProvider.AgentConn,
|
||||
AgentInactiveDisconnectTimeout: api.AgentInactiveDisconnectTimeout,
|
||||
|
||||
@@ -52,9 +52,10 @@ func (rt *Runtime) RunAdvisor(
|
||||
resetProviderOptionsForNestedCall(nestedCall.ProviderOptions)
|
||||
|
||||
assistantOpts := chatloop.GenerateAssistantOptions{
|
||||
Model: rt.cfg.Model,
|
||||
Messages: BuildAdvisorMessages(question, conversationSnapshot),
|
||||
CallTemplate: nestedCall,
|
||||
Model: rt.cfg.Model,
|
||||
Messages: BuildAdvisorMessages(question, conversationSnapshot),
|
||||
CallTemplate: nestedCall,
|
||||
StreamSilenceTimeout: rt.cfg.StreamSilenceTimeout,
|
||||
}
|
||||
if opts != nil && opts.OnAdviceDelta != nil {
|
||||
assistantOpts.PublishMessagePart = func(role codersdk.ChatMessageRole, part codersdk.ChatMessagePart) {
|
||||
|
||||
@@ -2,6 +2,7 @@ package chatadvisor
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"charm.land/fantasy"
|
||||
fantasyopenai "charm.land/fantasy/providers/openai"
|
||||
@@ -12,9 +13,10 @@ import (
|
||||
type RuntimeConfig struct {
|
||||
Model fantasy.LanguageModel
|
||||
// CallTemplate's provider options are cloned for each nested call.
|
||||
CallTemplate fantasy.Call
|
||||
MaxUsesPerRun int
|
||||
MaxOutputTokens int64
|
||||
CallTemplate fantasy.Call
|
||||
MaxUsesPerRun int
|
||||
MaxOutputTokens int64
|
||||
StreamSilenceTimeout time.Duration
|
||||
}
|
||||
|
||||
// Runtime executes nested, tool-less advisor runs against the configured
|
||||
|
||||
+13
-4
@@ -210,6 +210,7 @@ type Server struct {
|
||||
|
||||
// Configuration
|
||||
inFlightChatStaleAfter time.Duration
|
||||
streamSilenceTimeout time.Duration
|
||||
}
|
||||
|
||||
func (p *Server) loadAdvisorConfig(ctx context.Context, logger slog.Logger) advisorRuntimeConfig {
|
||||
@@ -399,10 +400,11 @@ func (p *Server) newAdvisorRuntime(
|
||||
}
|
||||
|
||||
rt, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{
|
||||
Model: advisor.model.LanguageModel(),
|
||||
CallTemplate: advisor.newCall(),
|
||||
MaxUsesPerRun: maxUsesPerRun,
|
||||
MaxOutputTokens: maxOutputTokens,
|
||||
Model: advisor.model.LanguageModel(),
|
||||
CallTemplate: advisor.newCall(),
|
||||
MaxUsesPerRun: maxUsesPerRun,
|
||||
MaxOutputTokens: maxOutputTokens,
|
||||
StreamSilenceTimeout: p.streamSilenceTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warn(
|
||||
@@ -2970,6 +2972,7 @@ type Config struct {
|
||||
MaxChatsPerAcquire int32
|
||||
InFlightChatStaleAfter time.Duration
|
||||
ChatHeartbeatInterval time.Duration
|
||||
StreamSilenceTimeout time.Duration
|
||||
AgentConn AgentConnFunc
|
||||
AgentInactiveDisconnectTimeout time.Duration
|
||||
CreateWorkspace chattool.CreateWorkspaceFn
|
||||
@@ -3026,6 +3029,11 @@ func New(ps pubsub.Pubsub, cfg Config) *Server {
|
||||
chatHeartbeatInterval = DefaultChatHeartbeatInterval
|
||||
}
|
||||
|
||||
streamSilenceTimeout := cfg.StreamSilenceTimeout
|
||||
if streamSilenceTimeout == 0 {
|
||||
streamSilenceTimeout = chatloop.DefaultStreamSilenceTimeout
|
||||
}
|
||||
|
||||
clk := cfg.Clock
|
||||
if clk == nil {
|
||||
clk = quartz.NewReal()
|
||||
@@ -3095,6 +3103,7 @@ func New(ps pubsub.Pubsub, cfg Config) *Server {
|
||||
aibridgeTransportFactory: cfg.AIBridgeTransportFactory,
|
||||
experiments: cfg.Experiments,
|
||||
inFlightChatStaleAfter: inFlightChatStaleAfter,
|
||||
streamSilenceTimeout: streamSilenceTimeout,
|
||||
usageTracker: cfg.UsageTracker,
|
||||
clock: clk,
|
||||
recordingSem: make(chan struct{}, maxConcurrentRecordingUploads),
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/coder/coder/v2/coderd/database/dbgen"
|
||||
"github.com/coder/coder/v2/coderd/database/dbtestutil"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
"github.com/coder/coder/v2/testutil"
|
||||
@@ -136,6 +137,9 @@ func TestActiveServer_RetryStreamSilenceTimeoutAndClassification(t *testing.T) {
|
||||
t.Run("silent stream generation retry recovers", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Longer than the 15m task watchdog.
|
||||
const streamSilenceTimeout = 20 * time.Minute
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
db, ps := dbtestutil.NewDB(t)
|
||||
reg := prometheus.NewRegistry()
|
||||
@@ -164,12 +168,14 @@ func TestActiveServer_RetryStreamSilenceTimeoutAndClassification(t *testing.T) {
|
||||
cfg.PrometheusRegistry = reg
|
||||
cfg.PendingChatAcquireInterval = 30 * time.Minute
|
||||
cfg.ChatHeartbeatInterval = 30 * time.Minute
|
||||
cfg.StreamSilenceTimeout = streamSilenceTimeout
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(factory)
|
||||
})
|
||||
|
||||
chat := createChatThroughServer(ctx, t, db, server, org.ID, user.ID, model.ID, "hello")
|
||||
firstGuard := streamGuardTrap.MustWait(ctx)
|
||||
firstGuard.MustRelease(ctx)
|
||||
require.Equal(t, streamSilenceTimeout, firstGuard.Duration)
|
||||
waitUntilProviderCall(ctx, t, &calls, 1)
|
||||
advanceMockClockBy(ctx, t, clock, firstGuard.Duration)
|
||||
retryTimer := retryTrap.MustWait(ctx)
|
||||
@@ -195,6 +201,47 @@ func TestActiveServer_RetryStreamSilenceTimeoutAndClassification(t *testing.T) {
|
||||
"kind": string(codersdk.ChatErrorKindStreamSilenceTimeout),
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("disabled stream silence timeout never cancels a silent stream", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
db, ps := dbtestutil.NewDB(t)
|
||||
clock := quartz.NewMock(t).WithLogger(quartz.NoOpLogger)
|
||||
var calls atomic.Int32
|
||||
unblock := make(chan struct{})
|
||||
openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
|
||||
if !req.Stream {
|
||||
return chattest.OpenAINonStreamingResponse("title")
|
||||
}
|
||||
calls.Add(1)
|
||||
select {
|
||||
case <-unblock:
|
||||
case <-req.Request.Context().Done():
|
||||
}
|
||||
return chattest.OpenAIStreamingResponse(openAITextChunksWithStop("done")...)
|
||||
})
|
||||
user, org, model := seedChatDependenciesWithProvider(t, db, "openai", openAIURL)
|
||||
factory := chattest.NewMockAIBridgeTransport(t, openAIURL)
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
cfg.Clock = clock
|
||||
cfg.PendingChatAcquireInterval = 30 * time.Minute
|
||||
cfg.ChatHeartbeatInterval = 30 * time.Minute
|
||||
cfg.StreamSilenceTimeout = chatloop.StreamSilenceTimeoutDisabled
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(factory)
|
||||
})
|
||||
|
||||
chat := createChatThroughServer(ctx, t, db, server, org.ID, user.ID, model.ID, "hello")
|
||||
waitUntilProviderCall(ctx, t, &calls, 1)
|
||||
// Past the 15m task watchdog.
|
||||
advanceMockClockBy(ctx, t, clock, 16*time.Minute)
|
||||
close(unblock)
|
||||
waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
|
||||
require.Equal(t, int32(1), calls.Load())
|
||||
messages, err := db.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ChatID: chat.ID})
|
||||
require.NoError(t, err)
|
||||
requireTextPart(t, messages[len(messages)-1], "done")
|
||||
})
|
||||
}
|
||||
|
||||
func retryEntriesWithMessage(sink *testutil.FakeSink, message string) []slog.SinkEntry {
|
||||
|
||||
@@ -34,11 +34,13 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
// defaultStreamSilenceTimeout bounds how long an individual
|
||||
// DefaultStreamSilenceTimeout bounds how long an individual
|
||||
// model attempt may go without receiving a stream part before
|
||||
// the attempt is canceled and retried.
|
||||
defaultStreamSilenceTimeout = 10 * time.Minute
|
||||
streamSilenceGuardTimerTag = "streamSilenceGuard"
|
||||
DefaultStreamSilenceTimeout = 10 * time.Minute
|
||||
// StreamSilenceTimeoutDisabled turns the silence guard off.
|
||||
StreamSilenceTimeoutDisabled time.Duration = -1
|
||||
streamSilenceGuardTimerTag = "streamSilenceGuard"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -277,8 +279,8 @@ func GenerateAssistant(ctx context.Context, opts GenerateAssistantOptions) (Assi
|
||||
if opts.Model == nil {
|
||||
return AssistantOutcome{}, xerrors.New("chat model is required")
|
||||
}
|
||||
if opts.StreamSilenceTimeout <= 0 {
|
||||
opts.StreamSilenceTimeout = defaultStreamSilenceTimeout
|
||||
if opts.StreamSilenceTimeout == 0 {
|
||||
opts.StreamSilenceTimeout = DefaultStreamSilenceTimeout
|
||||
}
|
||||
if opts.Clock == nil {
|
||||
opts.Clock = quartz.NewReal()
|
||||
@@ -691,6 +693,9 @@ func newStreamSilenceGuard(
|
||||
cancel: cancel,
|
||||
timeout: timeout,
|
||||
}
|
||||
if timeout < 0 {
|
||||
return guard
|
||||
}
|
||||
guard.timer = clock.AfterFunc(
|
||||
timeout,
|
||||
guard.onTimeout,
|
||||
@@ -719,14 +724,14 @@ func (g *streamSilenceGuard) onTimeout() {
|
||||
func (g *streamSilenceGuard) Reset() {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.settled {
|
||||
if g.settled || g.timer == nil {
|
||||
return
|
||||
}
|
||||
g.timer.Reset(g.timeout, streamSilenceGuardTimerTag)
|
||||
}
|
||||
|
||||
func (g *streamSilenceGuard) Disarm() {
|
||||
if !g.settle() {
|
||||
if !g.settle() || g.timer == nil {
|
||||
return
|
||||
}
|
||||
g.timer.Stop()
|
||||
@@ -750,6 +755,14 @@ func classifyStreamSilenceTimeout(
|
||||
})
|
||||
}
|
||||
|
||||
type streamWatchdogKey struct{}
|
||||
|
||||
// WithStreamWatchdog returns a context whose guarded streams call kick
|
||||
// with the silence timeout whenever the guard arms or resets.
|
||||
func WithStreamWatchdog(ctx context.Context, kick func(silence time.Duration)) context.Context {
|
||||
return context.WithValue(ctx, streamWatchdogKey{}, kick)
|
||||
}
|
||||
|
||||
func guardedStream(
|
||||
parent context.Context,
|
||||
provider, model string,
|
||||
@@ -759,7 +772,12 @@ func guardedStream(
|
||||
metrics *Metrics,
|
||||
) (guardedAttempt, error) {
|
||||
attemptCtx, cancelAttempt := context.WithCancelCause(parent)
|
||||
kick, _ := parent.Value(streamWatchdogKey{}).(func(time.Duration))
|
||||
if kick == nil {
|
||||
kick = func(time.Duration) {}
|
||||
}
|
||||
guard := newStreamSilenceGuard(clock, timeout, cancelAttempt)
|
||||
kick(timeout)
|
||||
var releaseOnce sync.Once
|
||||
release := func() {
|
||||
releaseOnce.Do(func() {
|
||||
@@ -786,6 +804,7 @@ func guardedStream(
|
||||
stream: fantasy.StreamResponse(func(yield func(fantasy.StreamPart) bool) {
|
||||
for part := range stream {
|
||||
guard.Reset()
|
||||
kick(timeout)
|
||||
recordTTFT()
|
||||
if !yield(part) {
|
||||
return
|
||||
|
||||
@@ -743,6 +743,7 @@ func (s *taskStarter) generateAssistant(
|
||||
CallTemplate: prepared.CallTemplate,
|
||||
PublishMessagePart: attempt.publish,
|
||||
OnModelStreamStart: attempt.startModelInvocation,
|
||||
StreamSilenceTimeout: s.server.streamSilenceTimeout,
|
||||
Logger: s.opts.Logger,
|
||||
Clock: s.opts.Clock,
|
||||
Metrics: s.server.metrics,
|
||||
|
||||
+13
-5
@@ -25,10 +25,8 @@ import (
|
||||
|
||||
const (
|
||||
postCommitWatchPublishTimeout = 10 * time.Second
|
||||
// defaultTaskTimeout must exceed chatloop's stream-silence guard so
|
||||
// silent provider streams fail through chat-specific retry handling
|
||||
// before the runner retries the whole task.
|
||||
defaultTaskTimeout = 15 * time.Minute
|
||||
defaultTaskTimeout = 15 * time.Minute
|
||||
taskTimeoutMargin = 5 * time.Minute
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -156,9 +154,19 @@ func runTaskWithRetry(
|
||||
|
||||
func taskAttemptContext(ctx context.Context, clock quartz.Clock, kind taskKind) (context.Context, func()) {
|
||||
attemptCtx, cancelCause := context.WithCancelCause(ctx)
|
||||
tag := "task-timeout-" + string(kind)
|
||||
timer := clock.AfterFunc(defaultTaskTimeout, func() {
|
||||
cancelCause(errTaskTimeout)
|
||||
}, "chatworker", "task-timeout-"+string(kind))
|
||||
}, "chatworker", tag)
|
||||
// A silent stream must fail through the silence guard rather than the
|
||||
// watchdog, and a healthy long stream must not be cut off.
|
||||
attemptCtx = chatloop.WithStreamWatchdog(attemptCtx, func(silence time.Duration) {
|
||||
if silence < 0 {
|
||||
timer.Stop()
|
||||
return
|
||||
}
|
||||
timer.Reset(max(defaultTaskTimeout, silence+taskTimeoutMargin), "chatworker", tag)
|
||||
})
|
||||
return attemptCtx, func() {
|
||||
timer.Stop()
|
||||
cancelCause(nil)
|
||||
|
||||
+23
-7
@@ -4394,6 +4394,17 @@ Write out the current server config as YAML to stdout.`,
|
||||
YAML: "aiGatewayRoutingEnabled",
|
||||
Hidden: true,
|
||||
},
|
||||
{
|
||||
Name: "Chat: Stream Silence Timeout",
|
||||
Description: "Maximum time to wait for the next streamed part from the chat model before the attempt is canceled and retried. This also bounds the time to first token. Set to 0 to disable. Must be no more than 24h.",
|
||||
Flag: "chat-stream-silence-timeout",
|
||||
Env: "CODER_CHAT_STREAM_SILENCE_TIMEOUT",
|
||||
Value: &c.AI.Chat.StreamSilenceTimeout,
|
||||
Default: (10 * time.Minute).String(),
|
||||
Group: &deploymentGroupChat,
|
||||
YAML: "streamSilenceTimeout",
|
||||
Annotations: serpent.Annotations{}.Mark(annotationFormatDuration, "true"),
|
||||
},
|
||||
// AI Bridge Options (deprecated in favor of AI Gateway options)
|
||||
{
|
||||
Name: "AI Bridge Enabled",
|
||||
@@ -4872,13 +4883,14 @@ type AIBridgeProxyConfig struct {
|
||||
}
|
||||
|
||||
type ChatConfig struct {
|
||||
AcquireBatchSize serpent.Int64 `json:"acquire_batch_size" typescript:",notnull"`
|
||||
DebugLoggingEnabled serpent.Bool `json:"debug_logging_enabled" typescript:",notnull"`
|
||||
HookURL serpent.URL `json:"hook_url" typescript:",notnull"`
|
||||
HookSecret serpent.String `json:"hook_secret" typescript:",notnull"`
|
||||
HookTimeout serpent.Duration `json:"hook_timeout" typescript:",notnull"`
|
||||
HookEnabled serpent.Bool `json:"hook_enabled" typescript:",notnull"`
|
||||
HookAllowInsecure serpent.Bool `json:"hook_allow_insecure" typescript:",notnull"`
|
||||
AcquireBatchSize serpent.Int64 `json:"acquire_batch_size" typescript:",notnull"`
|
||||
DebugLoggingEnabled serpent.Bool `json:"debug_logging_enabled" typescript:",notnull"`
|
||||
HookURL serpent.URL `json:"hook_url" typescript:",notnull"`
|
||||
HookSecret serpent.String `json:"hook_secret" typescript:",notnull"`
|
||||
HookTimeout serpent.Duration `json:"hook_timeout" typescript:",notnull"`
|
||||
HookEnabled serpent.Bool `json:"hook_enabled" typescript:",notnull"`
|
||||
HookAllowInsecure serpent.Bool `json:"hook_allow_insecure" typescript:",notnull"`
|
||||
StreamSilenceTimeout serpent.Duration `json:"stream_silence_timeout" typescript:",notnull"`
|
||||
// Deprecated: AI Gateway routing is now the only routing path. Setting this
|
||||
// value has no effect. This option will be removed in a future release.
|
||||
AIGatewayRoutingEnabled serpent.Bool `json:"ai_gateway_routing_enabled" typescript:",notnull" swaggerignore:"true"`
|
||||
@@ -4994,6 +5006,10 @@ func (c *DeploymentValues) Validate() error {
|
||||
}
|
||||
}
|
||||
|
||||
if timeout := c.AI.Chat.StreamSilenceTimeout.Value(); timeout < 0 || timeout > 24*time.Hour {
|
||||
return xerrors.Errorf("chat stream silence timeout (%s) must be between 0 and 24h; set --chat-stream-silence-timeout to a valid duration", timeout)
|
||||
}
|
||||
|
||||
// Gated on the builder being enabled and run here rather than as a per-option
|
||||
// serpent validator, which only fires in Set and so misses the YAML path.
|
||||
if !c.TemplateBuilder.Disabled.Value() {
|
||||
|
||||
@@ -946,6 +946,39 @@ func TestDeploymentValues_Validate_ChatHooks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeploymentValues_Validate_ChatStreamSilenceTimeout(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
timeout time.Duration
|
||||
wantErr string
|
||||
}{
|
||||
{name: "Disabled", timeout: 0},
|
||||
{name: "Negative", timeout: -time.Second, wantErr: "chat stream silence timeout"},
|
||||
{name: "Maximum", timeout: 24 * time.Hour},
|
||||
{name: "AboveMaximum", timeout: 24*time.Hour + time.Second, wantErr: "chat stream silence timeout"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dv := &codersdk.DeploymentValues{}
|
||||
dv.Sessions.DefaultDuration = serpent.Duration(time.Hour)
|
||||
dv.Sessions.RefreshDefaultDuration = serpent.Duration(48 * time.Hour)
|
||||
dv.AI.Chat.StreamSilenceTimeout = serpent.Duration(tt.timeout)
|
||||
|
||||
err := dv.Validate()
|
||||
if tt.wantErr == "" {
|
||||
require.NoError(t, err)
|
||||
return
|
||||
}
|
||||
require.ErrorContains(t, err, tt.wantErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeploymentValues_DurationFormatNanoseconds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -418,6 +418,15 @@ Force chat debug logging on for every chat, bypassing the runtime admin and user
|
||||
- YAML key: `chat.debugLoggingEnabled`
|
||||
- Default value: `false`
|
||||
|
||||
### Stream silence timeout
|
||||
|
||||
Maximum time to wait for the next streamed part from the chat model before the attempt is canceled and retried. This also bounds the time to first token. Set to 0 to disable. Must be no more than 24h.
|
||||
|
||||
- Environment variable: `CODER_CHAT_STREAM_SILENCE_TIMEOUT`
|
||||
- CLI flag: [`--chat-stream-silence-timeout`](../../reference/cli/server.md#--chat-stream-silence-timeout)
|
||||
- YAML key: `chat.streamSilenceTimeout`
|
||||
- Default value: `10m0s`
|
||||
|
||||
## Client
|
||||
|
||||
These options change the behavior of how clients interact with the Coder. Clients include the Coder CLI, Coder Desktop, IDE extensions, and the web UI.
|
||||
|
||||
Generated
+2
-1
@@ -229,7 +229,8 @@ curl -X GET http://coder-server:8080/api/v2/deployment/config \
|
||||
"rawQuery": "string",
|
||||
"scheme": "string",
|
||||
"user": {}
|
||||
}
|
||||
},
|
||||
"stream_silence_timeout": 0
|
||||
}
|
||||
},
|
||||
"allow_workspace_renames": true,
|
||||
|
||||
Generated
+18
-13
@@ -1026,7 +1026,8 @@ title: Schemas
|
||||
"rawQuery": "string",
|
||||
"scheme": "string",
|
||||
"user": {}
|
||||
}
|
||||
},
|
||||
"stream_silence_timeout": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -2515,21 +2516,23 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in
|
||||
"rawQuery": "string",
|
||||
"scheme": "string",
|
||||
"user": {}
|
||||
}
|
||||
},
|
||||
"stream_silence_timeout": 0
|
||||
}
|
||||
```
|
||||
|
||||
### Properties
|
||||
|
||||
| Name | Type | Required | Restrictions | Description |
|
||||
|-------------------------|----------------------------|----------|--------------|-------------|
|
||||
| `acquire_batch_size` | integer | false | | |
|
||||
| `debug_logging_enabled` | boolean | false | | |
|
||||
| `hook_allow_insecure` | boolean | false | | |
|
||||
| `hook_enabled` | boolean | false | | |
|
||||
| `hook_secret` | string | false | | |
|
||||
| `hook_timeout` | integer | false | | |
|
||||
| `hook_url` | [serpent.URL](#serpenturl) | false | | |
|
||||
| Name | Type | Required | Restrictions | Description |
|
||||
|--------------------------|----------------------------|----------|--------------|-------------|
|
||||
| `acquire_batch_size` | integer | false | | |
|
||||
| `debug_logging_enabled` | boolean | false | | |
|
||||
| `hook_allow_insecure` | boolean | false | | |
|
||||
| `hook_enabled` | boolean | false | | |
|
||||
| `hook_secret` | string | false | | |
|
||||
| `hook_timeout` | integer | false | | |
|
||||
| `hook_url` | [serpent.URL](#serpenturl) | false | | |
|
||||
| `stream_silence_timeout` | integer | false | | |
|
||||
|
||||
## codersdk.ChatContext
|
||||
|
||||
@@ -7311,7 +7314,8 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o
|
||||
"rawQuery": "string",
|
||||
"scheme": "string",
|
||||
"user": {}
|
||||
}
|
||||
},
|
||||
"stream_silence_timeout": 0
|
||||
}
|
||||
},
|
||||
"allow_workspace_renames": true,
|
||||
@@ -7922,7 +7926,8 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o
|
||||
"rawQuery": "string",
|
||||
"scheme": "string",
|
||||
"user": {}
|
||||
}
|
||||
},
|
||||
"stream_silence_timeout": 0
|
||||
}
|
||||
},
|
||||
"allow_workspace_renames": true,
|
||||
|
||||
Generated
+11
@@ -1790,6 +1790,17 @@ How often to reconcile workspace prebuilds state.
|
||||
|
||||
Force chat debug logging on for every chat, bypassing the runtime admin and user opt-in settings.
|
||||
|
||||
### --chat-stream-silence-timeout
|
||||
|
||||
| | |
|
||||
|-------------|-------------------------------------------------|
|
||||
| Type | <code>duration</code> |
|
||||
| Environment | <code>$CODER_CHAT_STREAM_SILENCE_TIMEOUT</code> |
|
||||
| YAML | <code>chat.streamSilenceTimeout</code> |
|
||||
| Default | <code>10m0s</code> |
|
||||
|
||||
Maximum time to wait for the next streamed part from the chat model before the attempt is canceled and retried. This also bounds the time to first token. Set to 0 to disable. Must be no more than 24h.
|
||||
|
||||
### --ai-gateway-enabled
|
||||
|
||||
| | |
|
||||
|
||||
@@ -223,6 +223,11 @@ Configure the background chat processing daemon.
|
||||
Force chat debug logging on for every chat, bypassing the runtime
|
||||
admin and user opt-in settings.
|
||||
|
||||
--chat-stream-silence-timeout duration, $CODER_CHAT_STREAM_SILENCE_TIMEOUT (default: 10m0s)
|
||||
Maximum time to wait for the next streamed part from the chat model
|
||||
before the attempt is canceled and retried. This also bounds the time
|
||||
to first token. Set to 0 to disable. Must be no more than 24h.
|
||||
|
||||
CLIENT OPTIONS:
|
||||
These options change the behavior of how clients interact with the Coder.
|
||||
Clients include the Coder CLI, Coder Desktop, IDE extensions, and the web UI.
|
||||
|
||||
Generated
+1
@@ -2054,6 +2054,7 @@ export interface ChatConfig {
|
||||
readonly hook_timeout: number;
|
||||
readonly hook_enabled: boolean;
|
||||
readonly hook_allow_insecure: boolean;
|
||||
readonly stream_silence_timeout: number;
|
||||
/**
|
||||
* @deprecated AI Gateway routing is now the only routing path. Setting this
|
||||
* value has no effect. This option will be removed in a future release.
|
||||
|
||||
Reference in new issue
Block a user