Files
coder/coderd/workspaceagentsrpc_internal_test.go
T
Faur Ioan-AurelandClaude Opus 5.5 1a65385a4b fix: log workspace shutdown reasons for SSH connections (#30515)
Workspace stops and deletes closed active SSH connections with generic
shutdown or process-exit reasons. This obscured the workspace action
that ended a connection and made related logs harder to correlate.

The agent’s SSH tracking and logging now identify connections closed by
the SSH server as server_shutdown, while preserving earlier client or
network disconnects. The agent includes the client session ID when
available for ordinary sessions and connections used only for port
forwarding.

Coderd now enriches shutdown reasons when it processes connection
reports. It translates server_shutdown into workspace_stopped when the
latest build is a running or successful stop or delete requested before
the connection closed. Coderd persists the enriched reason and logs the
build reason, transition, and build ID alongside the client session ID.
Unavailable or unrelated build state retains the generic shutdown
reason.

The CLI now logs shutdown context once per stop or delete build through
its existing workspace watcher. This includes the client session ID when
available and preserves the existing graceful SSH shutdown behavior.

Regarding the PTY connections, when the agent shut down, the
reconnecting PTY server closed each PTY
and its attached connections, and the connection handler then returned
without an error. The server therefore logged and reported those
connections as graceful closes. Its fallback for a forced shutdown
reported a free-form reason that coderd did not recognize, so coderd
never refined either case into a workspace shutdown.

We change the reconnecting PTY server so that it reports any connection
that ends after the agent begins shutting down as a server shutdown
initiated by the agent, and its log keeps the client session ID. The
forced shutdown fallback now reports the same standard reason, which
lets coderd translate it into a workspace stop or delete in the
connection log, as it already does for SSH. A client that closes its
connection before the shutdown still keeps its graceful reason.

Regression tests cover shutdown attribution, fallback behavior, CLI
logging, and PostgreSQL persistence with and without a client session
ID. The test logging helper also synchronizes with cleanup to prevent
logging after a test completes

- resolves
https://linear.app/codercom/issue/CLIENT-665/close-workspace-tunnel-with-a-reason

<!--

If you have used AI to produce some or all of this PR, please ensure you
have read our [AI Contribution
guidelines](https://github.com/coder/coder/blob/main/contributing/AI_CONTRIBUTING.md)
before submitting.

-->

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 23:06:41 +03:00

627 lines
19 KiB
Go

package coderd
import (
"context"
"database/sql"
"fmt"
"sync"
"testing"
"time"
"github.com/google/uuid"
"github.com/prometheus/client_golang/prometheus"
promtest "github.com/prometheus/client_golang/prometheus/testutil"
dto "github.com/prometheus/client_model/go"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"cdr.dev/slog/v3"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbmock"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/coderd/wspubsub"
"github.com/coder/coder/v2/testutil"
"github.com/coder/websocket"
)
func TestCheckBuildIsLatestDuringShutdown(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
transition database.WorkspaceTransition
startStatus database.ProvisionerJobStatus
shutdownStatus database.ProvisionerJobStatus
wantError bool
}{
{name: "Stopping", transition: database.WorkspaceTransitionStop, startStatus: database.ProvisionerJobStatusSucceeded, shutdownStatus: database.ProvisionerJobStatusRunning},
{name: "StopPending", transition: database.WorkspaceTransitionStop, startStatus: database.ProvisionerJobStatusSucceeded, shutdownStatus: database.ProvisionerJobStatusPending},
{name: "Deleting", transition: database.WorkspaceTransitionDelete, startStatus: database.ProvisionerJobStatusSucceeded, shutdownStatus: database.ProvisionerJobStatusRunning},
{name: "DeletePending", transition: database.WorkspaceTransitionDelete, startStatus: database.ProvisionerJobStatusSucceeded, shutdownStatus: database.ProvisionerJobStatusPending},
{name: "Stopped", transition: database.WorkspaceTransitionStop, shutdownStatus: database.ProvisionerJobStatusSucceeded, wantError: true},
{name: "Deleted", transition: database.WorkspaceTransitionDelete, shutdownStatus: database.ProvisionerJobStatusSucceeded, wantError: true},
{name: "Canceling", transition: database.WorkspaceTransitionStop, shutdownStatus: database.ProvisionerJobStatusCanceling, wantError: true},
{name: "Canceled", transition: database.WorkspaceTransitionStop, shutdownStatus: database.ProvisionerJobStatusCanceled, wantError: true},
{name: "Failed", transition: database.WorkspaceTransitionStop, shutdownStatus: database.ProvisionerJobStatusFailed, wantError: true},
{name: "FailedStart", transition: database.WorkspaceTransitionStop, startStatus: database.ProvisionerJobStatusFailed, shutdownStatus: database.ProvisionerJobStatusRunning, wantError: true},
{name: "CanceledStart", transition: database.WorkspaceTransitionDelete, startStatus: database.ProvisionerJobStatusCanceled, shutdownStatus: database.ProvisionerJobStatusRunning, wantError: true},
{name: "IncompleteStart", transition: database.WorkspaceTransitionStop, startStatus: database.ProvisionerJobStatusRunning, shutdownStatus: database.ProvisionerJobStatusPending, wantError: true},
{name: "Restarted", transition: database.WorkspaceTransitionStart, wantError: true},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
build := database.WorkspaceBuild{ID: uuid.New(), WorkspaceID: uuid.New(), BuildNumber: 1, Transition: database.WorkspaceTransitionStart, JobID: uuid.New()}
latest := database.WorkspaceBuild{ID: uuid.New(), WorkspaceID: build.WorkspaceID, BuildNumber: 2, Transition: tc.transition, JobID: uuid.New()}
db.EXPECT().GetLatestWorkspaceBuildByWorkspaceID(gomock.Any(), build.WorkspaceID).Return(latest, nil)
if tc.transition != database.WorkspaceTransitionStart {
db.EXPECT().GetProvisionerJobByID(gomock.Any(), latest.JobID).Return(database.ProvisionerJob{JobStatus: tc.shutdownStatus}, nil)
if tc.shutdownStatus == database.ProvisionerJobStatusPending || tc.shutdownStatus == database.ProvisionerJobStatusRunning {
db.EXPECT().GetProvisionerJobByID(gomock.Any(), build.JobID).Return(database.ProvisionerJob{JobStatus: tc.startStatus}, nil)
}
}
err := checkBuildIsLatest(testutil.Context(t, testutil.WaitLong), db, build)
if tc.wantError {
require.ErrorContains(t, err, "build is outdated")
} else {
require.NoError(t, err)
}
})
}
}
func TestAgentConnectionMonitor_ContextCancel(t *testing.T) {
t.Parallel()
now := dbtime.Now()
agentID := uuid.UUID{1}
replicaID := uuid.UUID{2}
testCases := []struct {
name string
agent database.WorkspaceAgent
initialMatcher connectionUpdateMatcher
}{
{
name: "no disconnected at",
agent: database.WorkspaceAgent{
ID: agentID,
FirstConnectedAt: sql.NullTime{
Time: now.Add(-time.Minute),
Valid: true,
},
},
initialMatcher: connectionUpdate(agentID, replicaID),
},
{
name: "disconnected at",
agent: database.WorkspaceAgent{
ID: agentID,
FirstConnectedAt: sql.NullTime{
Time: now.Add(-time.Minute),
Valid: true,
},
DisconnectedAt: sql.NullTime{
Time: now.Add(-2 * time.Minute),
Valid: true,
},
},
initialMatcher: connectionUpdate(agentID, replicaID, withDisconnectedAt(now.Add(-2*time.Minute))),
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
fConn := &fakePingerCloser{}
ctrl := gomock.NewController(t)
mDB := dbmock.NewMockStore(ctrl)
fUpdater := &fakeUpdater{}
logger := testutil.Logger(t)
build := database.WorkspaceBuild{
ID: uuid.New(),
WorkspaceID: uuid.New(),
}
uut := &agentConnectionMonitor{
apiCtx: ctx,
workspaceAgent: tc.agent,
workspaceBuild: build,
conn: fConn,
db: mDB,
replicaID: replicaID,
updater: fUpdater,
logger: logger,
pingPeriod: testutil.IntervalFast,
disconnectTimeout: testutil.WaitShort,
}
uut.init()
connected := mDB.EXPECT().UpdateWorkspaceAgentConnectionByID(
gomock.Any(),
tc.initialMatcher,
).
AnyTimes().
Return(nil)
mDB.EXPECT().UpdateWorkspaceAgentConnectionByID(
gomock.Any(),
connectionUpdate(agentID, replicaID, withDisconnectedNotBefore(now)),
).
After(connected).
Times(1).
Return(nil)
mDB.EXPECT().GetLatestWorkspaceBuildByWorkspaceID(gomock.Any(), build.WorkspaceID).
AnyTimes().
Return(database.WorkspaceBuild{ID: build.ID}, nil)
closeCtx, cancel := context.WithCancel(ctx)
defer cancel()
done := make(chan struct{})
go func() {
uut.monitor(closeCtx)
close(done)
}()
// wait a couple intervals, but not long enough for a disconnect
time.Sleep(3 * testutil.IntervalFast)
fConn.requireNotClosed(t)
fUpdater.requireEventuallySomeUpdates(t, build.WorkspaceID)
n := fUpdater.getUpdates()
cancel()
fConn.requireEventuallyClosed(t, websocket.StatusGoingAway, "canceled")
// make sure we got at least one additional update on close
_ = testutil.TryReceive(ctx, t, done)
m := fUpdater.getUpdates()
require.Greater(t, m, n)
})
}
}
func TestAgentConnectionMonitor_PingTimeout(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
now := dbtime.Now()
fConn := &fakePingerCloser{}
ctrl := gomock.NewController(t)
mDB := dbmock.NewMockStore(ctrl)
fUpdater := &fakeUpdater{}
logger := testutil.Logger(t)
agent := database.WorkspaceAgent{
ID: uuid.New(),
FirstConnectedAt: sql.NullTime{
Time: now.Add(-time.Minute),
Valid: true,
},
}
build := database.WorkspaceBuild{
ID: uuid.New(),
WorkspaceID: uuid.New(),
}
replicaID := uuid.New()
uut := &agentConnectionMonitor{
apiCtx: ctx,
workspaceAgent: agent,
workspaceBuild: build,
conn: fConn,
db: mDB,
replicaID: replicaID,
updater: fUpdater,
logger: logger,
pingPeriod: testutil.IntervalFast,
disconnectTimeout: testutil.WaitShort,
}
uut.init()
// set the last ping to the past, so we go thru the timeout
uut.lastPing.Store(new(now.Add(-time.Hour)))
connected := mDB.EXPECT().UpdateWorkspaceAgentConnectionByID(
gomock.Any(),
connectionUpdate(agent.ID, replicaID),
).
AnyTimes().
Return(nil)
mDB.EXPECT().UpdateWorkspaceAgentConnectionByID(
gomock.Any(),
connectionUpdate(agent.ID, replicaID, withDisconnectedNotBefore(now)),
).
After(connected).
Times(1).
Return(nil)
mDB.EXPECT().GetLatestWorkspaceBuildByWorkspaceID(gomock.Any(), build.WorkspaceID).
AnyTimes().
Return(database.WorkspaceBuild{ID: build.ID}, nil)
done := make(chan struct{})
go func() {
uut.monitor(ctx)
close(done)
}()
fConn.requireEventuallyClosed(t, websocket.StatusGoingAway, "ping timeout")
fUpdater.requireEventuallySomeUpdates(t, build.WorkspaceID)
_ = testutil.TryReceive(ctx, t, done) // ensure monitor() exits before mDB assertions are checked.
}
func TestAgentConnectionMonitor_BuildOutdated(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
now := dbtime.Now()
fConn := &fakePingerCloser{}
ctrl := gomock.NewController(t)
mDB := dbmock.NewMockStore(ctrl)
fUpdater := &fakeUpdater{}
logger := testutil.Logger(t)
agent := database.WorkspaceAgent{
ID: uuid.New(),
FirstConnectedAt: sql.NullTime{
Time: now.Add(-time.Minute),
Valid: true,
},
}
build := database.WorkspaceBuild{
ID: uuid.New(),
WorkspaceID: uuid.New(),
}
replicaID := uuid.New()
uut := &agentConnectionMonitor{
apiCtx: ctx,
workspaceAgent: agent,
workspaceBuild: build,
conn: fConn,
db: mDB,
replicaID: replicaID,
updater: fUpdater,
logger: logger,
pingPeriod: testutil.IntervalFast,
disconnectTimeout: testutil.WaitShort,
}
uut.init()
connected := mDB.EXPECT().UpdateWorkspaceAgentConnectionByID(
gomock.Any(),
connectionUpdate(agent.ID, replicaID),
).
AnyTimes().
Return(nil)
mDB.EXPECT().UpdateWorkspaceAgentConnectionByID(
gomock.Any(),
connectionUpdate(agent.ID, replicaID, withDisconnectedNotBefore(now)),
).
After(connected).
Times(1).
Return(nil)
// return a new buildID each time, meaning the connection is outdated
mDB.EXPECT().GetLatestWorkspaceBuildByWorkspaceID(gomock.Any(), build.WorkspaceID).
AnyTimes().
Return(database.WorkspaceBuild{ID: uuid.New()}, nil)
done := make(chan struct{})
go func() {
uut.monitor(ctx)
close(done)
}()
fConn.requireEventuallyClosed(t, websocket.StatusGoingAway, "build is outdated")
fUpdater.requireEventuallySomeUpdates(t, build.WorkspaceID)
_ = testutil.TryReceive(ctx, t, done) // ensure monitor() exits before mDB assertions are checked.
}
func TestAgentConnectionMonitor_SendPings(t *testing.T) {
t.Parallel()
testCtx := testutil.Context(t, testutil.WaitShort)
ctx, cancel := context.WithCancel(testCtx)
t.Cleanup(cancel)
fConn := &fakePingerCloser{}
uut := &agentConnectionMonitor{
pingPeriod: testutil.IntervalFast,
conn: fConn,
}
done := make(chan struct{})
go func() {
uut.sendPings(ctx)
close(done)
}()
fConn.requireEventuallyHasPing(t)
cancel()
_ = testutil.TryReceive(testCtx, t, done)
lastPing := uut.lastPing.Load()
require.NotNil(t, lastPing)
}
func TestAgentConnectionMonitor_StartClose(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
fConn := &fakePingerCloser{}
now := dbtime.Now()
ctrl := gomock.NewController(t)
mDB := dbmock.NewMockStore(ctrl)
fUpdater := &fakeUpdater{}
logger := testutil.Logger(t)
agent := database.WorkspaceAgent{
ID: uuid.New(),
FirstConnectedAt: sql.NullTime{
Time: now.Add(-time.Minute),
Valid: true,
},
}
build := database.WorkspaceBuild{
ID: uuid.New(),
WorkspaceID: uuid.New(),
}
replicaID := uuid.New()
uut := &agentConnectionMonitor{
apiCtx: ctx,
workspaceAgent: agent,
workspaceBuild: build,
conn: fConn,
db: mDB,
replicaID: replicaID,
updater: fUpdater,
logger: logger,
pingPeriod: testutil.IntervalFast,
disconnectTimeout: testutil.WaitShort,
}
connected := mDB.EXPECT().UpdateWorkspaceAgentConnectionByID(
gomock.Any(),
connectionUpdate(agent.ID, replicaID),
).
AnyTimes().
Return(nil)
mDB.EXPECT().UpdateWorkspaceAgentConnectionByID(
gomock.Any(),
connectionUpdate(agent.ID, replicaID, withDisconnectedNotBefore(now)),
).
After(connected).
Times(1).
Return(nil)
mDB.EXPECT().GetLatestWorkspaceBuildByWorkspaceID(gomock.Any(), build.WorkspaceID).
AnyTimes().
Return(database.WorkspaceBuild{ID: build.ID}, nil)
uut.start(ctx)
closed := make(chan struct{})
go func() {
uut.close()
close(closed)
}()
_ = testutil.TryReceive(ctx, t, closed)
}
func TestAgentConnectionMonitor_FirstConnectionMetric(t *testing.T) {
t.Parallel()
const metricName = "coderd_agents_first_connection_seconds"
t.Run("records metric on first connection", func(t *testing.T) {
t.Parallel()
reg := prometheus.NewRegistry()
logger := testutil.Logger(t)
metrics := NewWorkspaceAgentRPCMetrics(reg, logger)
createdAt := dbtime.Now().Add(-30 * time.Second)
uut := &agentConnectionMonitor{
workspace: database.Workspace{
TemplateName: "my-template",
},
workspaceAgent: database.WorkspaceAgent{
Name: "main",
CreatedAt: createdAt,
// FirstConnectedAt is zero-value (not valid),
// so init() treats this as the first connection.
},
metrics: metrics,
}
uut.init()
require.Equal(t, 1,
promtest.CollectAndCount(metrics.FirstConnectionDuration, metricName))
// Verify the observed sum reflects the duration since CreatedAt.
// testutil has no helper for reading histogram sums, so extract
// the sample via dto.Metric directly.
var observed dto.Metric
require.NoError(t, metrics.FirstConnectionDuration.
WithLabelValues("my-template", "main").(prometheus.Histogram).
Write(&observed))
require.EqualValues(t, 1, observed.GetHistogram().GetSampleCount())
require.GreaterOrEqual(t, observed.GetHistogram().GetSampleSum(), float64(30))
})
t.Run("skips metric and logs warning if duration is negative", func(t *testing.T) {
t.Parallel()
reg := prometheus.NewRegistry()
sink := testutil.NewFakeSink(t)
metrics := NewWorkspaceAgentRPCMetrics(reg, sink.Logger())
// Set CreatedAt in the future so the duration is negative,
// simulating clock skew.
uut := &agentConnectionMonitor{
workspace: database.Workspace{
TemplateName: "my-template",
},
workspaceAgent: database.WorkspaceAgent{
Name: "main",
CreatedAt: dbtime.Now().Add(time.Minute),
},
metrics: metrics,
}
uut.init()
// The negative-duration path skips the observation, so the
// histogram should have no recorded label combinations.
require.Equal(t, 0,
promtest.CollectAndCount(metrics.FirstConnectionDuration, metricName))
// Verify that a warning was logged.
warnings := sink.Entries(func(e slog.SinkEntry) bool {
return e.Level == slog.LevelWarn
})
require.Len(t, warnings, 1)
require.Contains(t, warnings[0].Message, "negative agent first connection duration")
})
}
type fakePingerCloser struct {
sync.Mutex
pings []time.Time
code websocket.StatusCode
reason string
closed bool
}
func (f *fakePingerCloser) Ping(context.Context) error {
f.Lock()
defer f.Unlock()
f.pings = append(f.pings, time.Now())
return nil
}
func (f *fakePingerCloser) Close(code websocket.StatusCode, reason string) error {
f.Lock()
defer f.Unlock()
if f.closed {
return nil
}
f.closed = true
f.code = code
f.reason = reason
return nil
}
func (f *fakePingerCloser) requireNotClosed(t *testing.T) {
f.Lock()
defer f.Unlock()
require.False(t, f.closed)
}
func (f *fakePingerCloser) requireEventuallyClosed(t *testing.T, code websocket.StatusCode, reason string) {
require.Eventually(t, func() bool {
f.Lock()
defer f.Unlock()
return f.closed
}, testutil.WaitShort, testutil.IntervalFast)
f.Lock()
defer f.Unlock()
require.Equal(t, code, f.code)
require.Equal(t, reason, f.reason)
}
func (f *fakePingerCloser) requireEventuallyHasPing(t *testing.T) {
require.Eventually(t, func() bool {
f.Lock()
defer f.Unlock()
return len(f.pings) > 0
}, testutil.WaitShort, testutil.IntervalFast)
}
type fakeUpdater struct {
sync.Mutex
updates []uuid.UUID
}
func (f *fakeUpdater) publishWorkspaceUpdate(_ context.Context, _ uuid.UUID, event wspubsub.WorkspaceEvent) {
f.Lock()
defer f.Unlock()
f.updates = append(f.updates, event.WorkspaceID)
}
func (f *fakeUpdater) requireEventuallySomeUpdates(t *testing.T, workspaceID uuid.UUID) {
require.Eventually(t, func() bool {
f.Lock()
defer f.Unlock()
return len(f.updates) >= 1
}, testutil.WaitShort, testutil.IntervalFast)
f.Lock()
defer f.Unlock()
for _, u := range f.updates {
require.Equal(t, workspaceID, u)
}
}
func (f *fakeUpdater) getUpdates() int {
f.Lock()
defer f.Unlock()
return len(f.updates)
}
type connectionUpdateMatcher struct {
agentID uuid.UUID
replicaID uuid.UUID
disconnectedAt sql.NullTime
disconnectedNotBefore sql.NullTime
}
type connectionUpdateMatcherOption func(m connectionUpdateMatcher) connectionUpdateMatcher
func connectionUpdate(id, replica uuid.UUID, opts ...connectionUpdateMatcherOption) connectionUpdateMatcher {
m := connectionUpdateMatcher{
agentID: id,
replicaID: replica,
}
for _, opt := range opts {
m = opt(m)
}
return m
}
func withDisconnectedNotBefore(t time.Time) connectionUpdateMatcherOption {
return func(m connectionUpdateMatcher) connectionUpdateMatcher {
m.disconnectedNotBefore = sql.NullTime{
Valid: true,
Time: t,
}
return m
}
}
func withDisconnectedAt(t time.Time) connectionUpdateMatcherOption {
return func(m connectionUpdateMatcher) connectionUpdateMatcher {
m.disconnectedAt = sql.NullTime{
Valid: true,
Time: t,
}
return m
}
}
func (m connectionUpdateMatcher) Matches(x interface{}) bool {
args, ok := x.(database.UpdateWorkspaceAgentConnectionByIDParams)
if !ok {
return false
}
if args.ID != m.agentID {
return false
}
if !args.LastConnectedReplicaID.Valid {
return false
}
if args.LastConnectedReplicaID.UUID != m.replicaID {
return false
}
if m.disconnectedNotBefore.Valid {
if !args.DisconnectedAt.Valid {
return false
}
if args.DisconnectedAt.Time.Before(m.disconnectedNotBefore.Time) {
return false
}
// disconnectedNotBefore takes precedence over disconnectedAt
} else if args.DisconnectedAt != m.disconnectedAt {
return false
}
return true
}
func (m connectionUpdateMatcher) String() string {
return fmt.Sprintf("{agent=%s, replica=%s, disconnectedAt=%v, disconnectedNotBefore=%v}",
m.agentID.String(), m.replicaID.String(), m.disconnectedAt, m.disconnectedNotBefore)
}
func (connectionUpdateMatcher) Got(x interface{}) string {
args, ok := x.(database.UpdateWorkspaceAgentConnectionByIDParams)
if !ok {
return fmt.Sprintf("type=%T", x)
}
return fmt.Sprintf("{agent=%s, replica=%s, disconnectedAt=%v}",
args.ID, args.LastConnectedReplicaID.UUID, args.DisconnectedAt)
}