Files
coder/support/support_test.go
Samuel Volin a1751d1887 fix: timeout agent dial in coder support bundle so unreachable agents wont hang (#31697)
if `coder support bundle <workspace>` is run when the workspace agent
had a record but was unreachable, this would cause the command to hang.

This command uploads a number of diagnostics from both the server and
agent, including `pprof`. If the agent is unreachable, this should still
be able to write out as much of the bundle as possible without timing
out.

The agent dial now has a configurable time limit
(`Deps.AgentDialTimeout`, default 30s). On timeout the bundle logs a
warning, skips the sections that need a live agent connection (ping,
peer diagnostics, magicsock, manifest, agent logs over tailnet,
listening ports, workspace files, agent pprof), and still writes out the
bundle. This includes agent logs fetched through the API. The CLI prints
a warning and records it in `cli_logs.txt`; exit status stays 0.

Other changes:
- Each request over an established agent connection has a 10s limit, and
workspace file collection has a 60s limit, so an agent that drops
mid-collection can't stall the bundle either.
- Agent pprof is skipped when the first dial already failed, instead of
waiting through a second dial. Server pprof collection is unchanged.
- `support.AgentInfo` now takes the dial timeout and also reports
whether the agent was reachable.

<details>
<summary>Verification and decision log</summary>

**Reproduction.** An end-to-end CLI test seeds a workspace whose build
has an agent record but never starts an agent process, then runs `coder
support bundle <ws> --yes` with a 90s outer deadline.

| Tree | Result |
|---|---|
| `main` | Hung until the 90s deadline. |
| This branch | Returned in 30.0s with exit 0 and wrote a 33-file zip.
`logs.txt` contains `agent unreachable, skipping agent-connected
sections` and `cli_logs.txt` contains `could not connect to workspace
agent`. |

`TestRun/AgentUnreachable` in `support/support_test.go` covers both
`--pprof` off and on.

**Decisions.**
- The hang was in `DialAgent` itself (`workspacesdk.go`,
`AwaitReachable(dialCtx)`), not in `Ping`, so the time limit goes on the
context passed to `DialAgent`. That function keeps the connection alive
on a separate background context, so cancelling the dial context after a
successful dial doesn't close the connection.
- The regression test uses a real agent record with no agent process
rather than the random agent ID the ticket suggested. A random ID fails
fast during agent lookup and never reaches the code that hangs.
- Limits follow the existing 10s and 45s timeouts used elsewhere in the
bundle collector, plus the ticket's 30s dial target and its worst case
of about 60s for an unreachable agent.

</details>
2026-11-16 13:04:58 -07:00

394 lines
17 KiB
Go

package support_test
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
"cdr.dev/slog/v3"
"cdr.dev/slog/v3/sloggers/sloghuman"
"cdr.dev/slog/v3/sloggers/slogtest"
"github.com/coder/coder/v2/agent"
"github.com/coder/coder/v2/agent/agentfiles"
"github.com/coder/coder/v2/agent/agenttest"
"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbfake"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/codersdk/workspacesdk"
"github.com/coder/coder/v2/support"
"github.com/coder/coder/v2/testutil"
"github.com/coder/serpent"
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m, testutil.GoleakOptions...)
}
func TestRun(t *testing.T) {
t.Parallel()
t.Run("OK", func(t *testing.T) {
t.Parallel()
cfg := coderdtest.DeploymentValues(t)
promPort := testutil.RandomPort(t)
cfg.Prometheus.Enable = serpent.Bool(true)
cfg.Prometheus.Address.Host = "127.0.0.1"
cfg.Prometheus.Address.Port = fmt.Sprintf("%d", promPort)
cfg.Experiments = []string{"foo"}
ctx := testutil.Context(t, testutil.WaitLong)
client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
DeploymentValues: cfg,
Logger: new(slog.Make(sloghuman.Sink(io.Discard))),
})
admin := coderdtest.CreateFirstUser(t, client)
ws, agt := setupWorkspaceAndAgent(ctx, t, client, db, admin)
bun, err := support.Run(ctx, &support.Deps{
Client: client,
Log: testutil.Logger(t).Named("bundle"),
WorkspaceID: ws.ID,
AgentID: agt.ID,
})
require.NoError(t, err)
assertNotNilNotEmpty(t, bun, "bundle should be present")
assertNotNilNotEmpty(t, bun.Deployment.BuildInfo, "deployment build info should be present")
assertNotNilNotEmpty(t, bun.Deployment.Config, "deployment config should be present")
assertNotNilNotEmpty(t, bun.Deployment.Config.Options, "deployment config should be present")
assertSanitizedDeploymentConfig(t, bun.Deployment.Config)
assertNotNilNotEmpty(t, bun.Deployment.HealthReport, "deployment health report should be present")
assertNotNilNotEmpty(t, bun.Deployment.Experiments, "deployment experiments should be present")
require.NotNil(t, bun.Deployment.Licenses, "license status should be present")
assertNotNilNotEmpty(t, bun.Network.ConnectionInfo, "agent connection info should be present")
assertNotNilNotEmpty(t, bun.Network.CoordinatorDebug, "network coordinator debug should be present")
assertNotNilNotEmpty(t, bun.Network.Netcheck, "network netcheck should be present")
assertNotNilNotEmpty(t, bun.Network.TailnetDebug, "network tailnet debug should be present")
assertNotNilNotEmpty(t, bun.Network.Interfaces, "network interfaces health should be present")
assertNotNilNotEmpty(t, bun.Workspace.Workspace, "workspace should be present")
assertSanitizedWorkspace(t, bun.Workspace.Workspace)
assertNotNilNotEmpty(t, bun.Workspace.BuildLogs, "workspace build logs should be present")
assertNotNilNotEmpty(t, bun.Workspace.Template, "workspace template should be present")
assertNotNilNotEmpty(t, bun.Workspace.TemplateVersion, "workspace template version should be present")
assertNotNilNotEmpty(t, bun.Workspace.TemplateFileBase64, "workspace template file should be present")
require.NotNil(t, bun.Workspace.Parameters, "workspace parameters should be present")
assertNotNilNotEmpty(t, bun.Agent.Agent, "agent should be present")
assertSanitizedEnv(t, bun.Agent.Agent.EnvironmentVariables)
assertNotNilNotEmpty(t, bun.Agent.ListeningPorts, "agent listening ports should be present")
assertNotNilNotEmpty(t, bun.Agent.Logs, "agent logs should be present")
assertNotNilNotEmpty(t, bun.Agent.Manifest, "agent manifest should be present")
assertSanitizedEnv(t, bun.Agent.Manifest.EnvironmentVariables)
assertNotNilNotEmpty(t, bun.Agent.AgentMagicsockHTML, "agent magicsock should be present")
assertNotNilNotEmpty(t, bun.Agent.ClientMagicsockHTML, "client magicsock should be present")
assertNotNilNotEmpty(t, bun.Agent.PeerDiagnostics, "agent peer diagnostics should be present")
assertNotNilNotEmpty(t, bun.Agent.PingResult, "agent ping result should be present")
assertNotNilNotEmpty(t, bun.Agent.Prometheus, "agent prometheus metrics should be present")
assertNotNilNotEmpty(t, bun.Deployment.Prometheus, "deployment prometheus metrics should be present")
assertNotNilNotEmpty(t, bun.Agent.StartupLogs, "agent startup logs should be present")
assertNotNilNotEmpty(t, bun.Logs, "bundle logs should be present")
assert.Nil(t, bun.Pprof.Server, "server pprof should not be collected without CollectPprof")
assert.Nil(t, bun.Pprof.Agent, "agent pprof should not be collected without CollectPprof")
// New: deployment health settings should be present
assertNotNilNotEmpty(t, bun.Deployment.HealthSettings, "deployment health settings should be present")
// New: aggregated workspaces should be present and include created workspace
assert.NotNil(t, bun.Deployment.Workspaces, "deployment workspaces should be present")
assert.GreaterOrEqual(t, bun.Deployment.Workspaces.Count, 1)
for _, aws := range bun.Deployment.Workspaces.Workspaces {
for _, res := range aws.LatestBuild.Resources {
for _, a := range res.Agents {
assertSanitizedEnv(t, a.EnvironmentVariables)
}
}
}
})
t.Run("AgentUnreachable", func(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
collectPprof bool
}{
{name: "NoPprof"},
{name: "Pprof", collectPprof: true},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
Logger: new(slog.Make(sloghuman.Sink(io.Discard))),
})
admin := coderdtest.CreateFirstUser(t, client)
ws, agt, _ := seedWorkspaceWithAgent(ctx, t, client, db, admin)
bun, err := support.Run(ctx, &support.Deps{
Client: client,
Log: slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Named("bundle").Leveled(slog.LevelDebug),
WorkspaceID: ws.ID,
AgentID: agt.ID,
AgentDialTimeout: testutil.WaitShort,
CollectPprof: tc.collectPprof,
})
require.NoError(t, err)
require.NoError(t, ctx.Err(), "Run must finish before the test deadline")
assertNotNilNotEmpty(t, bun.Deployment.BuildInfo, "deployment build info should be present")
assertNotNilNotEmpty(t, bun.Deployment.HealthReport, "deployment health report should be present")
assertNotNilNotEmpty(t, bun.Network.Netcheck, "network netcheck should be present")
assertNotNilNotEmpty(t, bun.Workspace.Workspace, "workspace should be present")
assertNotNilNotEmpty(t, bun.Agent.Agent, "agent should be present")
assertNotNilNotEmpty(t, bun.Agent.StartupLogs, "agent startup logs should be present")
assert.Nil(t, bun.Agent.PingResult, "agent ping result should be absent")
assert.Nil(t, bun.Agent.PeerDiagnostics, "agent peer diagnostics should be absent")
assert.Nil(t, bun.Agent.ListeningPorts, "agent listening ports should be absent")
assert.Empty(t, bun.Agent.Manifest, "agent manifest should be absent")
logs := strings.Join(bun.Logs, "\n")
assert.Contains(t, logs, "agent unreachable, skipping agent-connected sections")
if tc.collectPprof {
assert.Contains(t, logs, "skipping agent pprof: agent unreachable")
assert.Nil(t, bun.Pprof.Agent, "agent pprof should not be collected")
}
})
}
})
t.Run("OK_NoWorkspace", func(t *testing.T) {
t.Parallel()
cfg := coderdtest.DeploymentValues(t)
cfg.Experiments = []string{"foo"}
ctx := testutil.Context(t, testutil.WaitLong)
client := coderdtest.New(t, &coderdtest.Options{
DeploymentValues: cfg,
Logger: new(slog.Make(sloghuman.Sink(io.Discard))),
})
_ = coderdtest.CreateFirstUser(t, client)
bun, err := support.Run(ctx, &support.Deps{
Client: client,
Log: slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Named("bundle").Leveled(slog.LevelDebug),
})
require.NoError(t, err)
assertNotNilNotEmpty(t, bun, "bundle should be present")
assertNotNilNotEmpty(t, bun.Deployment.BuildInfo, "deployment build info should be present")
assertNotNilNotEmpty(t, bun.Deployment.Config, "deployment config should be present")
assertNotNilNotEmpty(t, bun.Deployment.Config.Options, "deployment config should be present")
assertSanitizedDeploymentConfig(t, bun.Deployment.Config)
assertNotNilNotEmpty(t, bun.Deployment.HealthReport, "deployment health report should be present")
assertNotNilNotEmpty(t, bun.Deployment.Experiments, "deployment experiments should be present")
assertNotNilNotEmpty(t, bun.Network.ConnectionInfo, "agent connection info should be present")
assertNotNilNotEmpty(t, bun.Network.CoordinatorDebug, "network coordinator debug should be present")
assertNotNilNotEmpty(t, bun.Network.Netcheck, "network netcheck should be present")
assertNotNilNotEmpty(t, bun.Network.TailnetDebug, "network tailnet debug should be present")
assertNotNilNotEmpty(t, bun.Network.Interfaces, "network interfaces health should be present")
assert.Empty(t, bun.Workspace.Workspace, "did not expect workspace to be present")
assert.Empty(t, bun.Agent, "did not expect agent to be present")
assertNotNilNotEmpty(t, bun.Logs, "bundle logs should be present")
assert.Nil(t, bun.Pprof.Server, "server pprof should not be collected without CollectPprof")
assert.Nil(t, bun.Pprof.Agent, "agent pprof should not be collected without CollectPprof")
// New: health settings should be present even without workspace context
assertNotNilNotEmpty(t, bun.Deployment.HealthSettings, "deployment health settings should be present")
// New: aggregated workspaces struct should exist (may be empty)
assert.NotNil(t, bun.Deployment.Workspaces)
})
t.Run("NoAuth", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
client := coderdtest.New(t, &coderdtest.Options{
Logger: new(slog.Make(sloghuman.Sink(io.Discard))),
})
bun, err := support.Run(ctx, &support.Deps{
Client: client,
Log: slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Named("bundle").Leveled(slog.LevelDebug),
})
var sdkErr *codersdk.Error
require.Nil(t, bun)
require.ErrorAs(t, err, &sdkErr)
require.Equal(t, http.StatusUnauthorized, sdkErr.StatusCode())
})
t.Run("MemberNoWorkspace", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
client := coderdtest.New(t, &coderdtest.Options{
Logger: new(slog.Make(sloghuman.Sink(io.Discard))),
})
admin := coderdtest.CreateFirstUser(t, client)
memberClient, _ := coderdtest.CreateAnotherUser(t, client, admin.OrganizationID)
bun, err := support.Run(ctx, &support.Deps{
Client: memberClient,
Log: slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Named("bundle").Leveled(slog.LevelDebug),
})
require.NoError(t, err)
require.NotNil(t, bun)
// Member should still get build info.
assertNotNilNotEmpty(t, bun.Deployment.BuildInfo, "deployment build info should be present")
// Experiments may be empty if none are configured, but should not error.
require.NotNil(t, bun.Deployment.Experiments, "deployment experiments should not be nil")
// Member should NOT get admin-only deployment information.
assert.Nil(t, bun.Deployment.Config, "member should not see deployment config")
assert.Nil(t, bun.Deployment.HealthReport, "member should not see health report")
// Member should still get network info from client-side checks.
assertNotNilNotEmpty(t, bun.Network.ConnectionInfo, "agent connection info should be present")
assertNotNilNotEmpty(t, bun.Network.Interfaces, "network interfaces should be present")
assertNotNilNotEmpty(t, bun.Network.Netcheck, "network netcheck should be present")
// No workspace specified.
assert.Empty(t, bun.Workspace.Workspace, "did not expect workspace to be present")
assert.Empty(t, bun.Agent, "did not expect agent to be present")
assertNotNilNotEmpty(t, bun.Logs, "bundle logs should be present")
})
}
func TestRunCollectsWorkspaceFiles(t *testing.T) {
// The resolved dir matches the agent's canonicalized manifest paths
// (the macOS temp dir is a symlink).
home := testutil.TempDirResolved(t)
t.Setenv("HOME", home)
t.Setenv("USERPROFILE", home)
require.NoError(t, os.WriteFile(filepath.Join(home, "workspace-service.log"), []byte("workspace service log"), 0o600))
cfg := coderdtest.DeploymentValues(t)
ctx := testutil.Context(t, testutil.WaitLong)
client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
DeploymentValues: cfg,
Logger: new(slog.Make(sloghuman.Sink(io.Discard))),
})
admin := coderdtest.CreateFirstUser(t, client)
ws, agt := setupWorkspaceAndAgent(ctx, t, client, db, admin)
bun, err := support.Run(ctx, &support.Deps{
Client: client,
Log: testutil.Logger(t).Named("bundle"),
WorkspaceID: ws.ID,
AgentID: agt.ID,
WorkspaceFilePatterns: []string{"$HOME/workspace-service.log"},
})
require.NoError(t, err)
assertWorkspaceFilesArchive(t, bun.Agent.WorkspaceFilesArchive, agentfiles.BundleFilesArchivePath(filepath.Join(home, "workspace-service.log")), "workspace service log")
}
func assertSanitizedDeploymentConfig(t *testing.T, dc *codersdk.DeploymentConfig) {
t.Helper()
for _, opt := range dc.Options {
if opt.Annotations.IsSet("secret") {
assert.Empty(t, opt.Value.String())
}
}
}
func assertSanitizedWorkspace(t *testing.T, ws codersdk.Workspace) {
t.Helper()
for _, res := range ws.LatestBuild.Resources {
for _, agt := range res.Agents {
assertSanitizedEnv(t, agt.EnvironmentVariables)
}
}
}
func assertSanitizedEnv(t *testing.T, env map[string]string) {
t.Helper()
for k, v := range env {
assert.Equal(t, "***REDACTED***", v, "environment variable %q not sanitized", k)
}
}
func setupWorkspaceAndAgent(ctx context.Context, t *testing.T, client *codersdk.Client, db database.Store, user codersdk.CreateFirstUserResponse) (codersdk.Workspace, codersdk.WorkspaceAgent) {
ws, agt, agentToken := seedWorkspaceWithAgent(ctx, t, client, db, user)
tempDir := t.TempDir()
logPath := filepath.Join(tempDir, "coder-agent.log")
require.NoError(t, os.WriteFile(logPath, []byte("hello from the agent"), 0o600))
_ = agenttest.New(t, client.URL, agentToken, func(o *agent.Options) {
o.LogDir = tempDir
})
coderdtest.NewWorkspaceAgentWaiter(t, client, ws.ID).Wait()
return ws, agt
}
// seedWorkspaceWithAgent creates a workspace whose build has an agent row and
// logs, without starting an agent process.
func seedWorkspaceWithAgent(ctx context.Context, t *testing.T, client *codersdk.Client, db database.Store, user codersdk.CreateFirstUserResponse) (codersdk.Workspace, codersdk.WorkspaceAgent, string) {
t.Helper()
// This is a valid zip file
zipBytes := make([]byte, 22)
zipBytes[0] = 80
zipBytes[1] = 75
zipBytes[2] = 0o5
zipBytes[3] = 0o6
uploadRes, err := client.Upload(ctx, codersdk.ContentTypeZip, bytes.NewReader(zipBytes))
require.NoError(t, err)
tv := dbfake.TemplateVersion(t, db).
FileID(uploadRes.ID).
Seed(database.TemplateVersion{
OrganizationID: user.OrganizationID,
CreatedBy: user.UserID,
}).
Do()
wbr := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
OrganizationID: user.OrganizationID,
OwnerID: user.UserID,
TemplateID: tv.Template.ID,
}).Resource().WithAgent().Do()
ws, err := client.Workspace(ctx, wbr.Workspace.ID)
require.NoError(t, err)
agt := ws.LatestBuild.Resources[0].Agents[0]
// Insert a provisioner job log
_, err = db.InsertProvisionerJobLogs(ctx, database.InsertProvisionerJobLogsParams{
JobID: wbr.Build.JobID,
CreatedAt: []time.Time{dbtime.Now()},
Source: []database.LogSource{database.LogSourceProvisionerDaemon},
Level: []database.LogLevel{database.LogLevelInfo},
Stage: []string{"The World"},
Output: []string{"Players"},
})
require.NoError(t, err)
// Insert an agent log
_, err = db.InsertWorkspaceAgentLogs(ctx, database.InsertWorkspaceAgentLogsParams{
AgentID: agt.ID,
CreatedAt: dbtime.Now(),
Output: []string{"Bond, James Bond"},
Level: []database.LogLevel{database.LogLevelInfo},
LogSourceID: wbr.Build.JobID,
OutputLength: 0o7,
})
require.NoError(t, err)
return ws, agt, wbr.AgentToken
}
func assertWorkspaceFilesArchive(t *testing.T, data []byte, wantEntry string, wantContent string) {
t.Helper()
require.NotEmpty(t, data)
entries := testutil.ReadTar(t, data)
require.Equal(t, wantContent, string(entries[wantEntry]))
var manifest workspacesdk.BundleFilesManifest
require.NoError(t, json.Unmarshal(entries["manifest.json"], &manifest))
require.Len(t, manifest.Files, 1)
require.Equal(t, wantEntry, manifest.Files[0].ArchivePath)
}
func assertNotNilNotEmpty[T any](t *testing.T, v T, msg string) {
t.Helper()
if assert.NotNil(t, v, msg+" but was nil") {
assert.NotEmpty(t, v, msg+" but was empty")
}
}