2022-03-28 18:19:28 -06:00
package cliui
import (
"context"
2024-02-27 22:04:46 +04:00
"fmt"
2022-03-30 17:59:54 -05:00
"io"
2024-02-27 22:04:46 +04:00
"strconv"
"strings"
2022-03-28 18:19:28 -06:00
"time"
2023-06-28 11:54:13 +03:00
"github.com/google/uuid"
2022-03-28 18:19:28 -06:00
"golang.org/x/xerrors"
2024-08-28 15:39:01 +10:00
"tailscale.com/tailcfg"
2022-03-28 18:19:28 -06:00
2023-08-18 13:55:43 -05:00
"github.com/coder/coder/v2/codersdk"
2024-08-28 15:39:01 +10:00
"github.com/coder/coder/v2/codersdk/healthsdk"
"github.com/coder/coder/v2/codersdk/workspacesdk"
2024-02-27 22:04:46 +04:00
"github.com/coder/coder/v2/tailnet"
2022-03-28 18:19:28 -06:00
)
2023-06-28 11:54:13 +03:00
var errAgentShuttingDown = xerrors . New ( "agent is shutting down" )
2023-01-27 19:05:40 +02:00
2022-03-28 18:19:28 -06:00
type AgentOptions struct {
FetchInterval time . Duration
2023-07-12 10:21:54 -05:00
Fetch func ( ctx context . Context , agentID uuid . UUID ) ( codersdk . WorkspaceAgent , error )
2023-07-28 09:57:23 -06:00
FetchLogs func ( ctx context . Context , agentID uuid . UUID , after int64 , follow bool ) ( <- chan [] codersdk . WorkspaceAgentLog , io . Closer , error )
2023-06-08 16:52:44 +03:00
Wait bool // If true, wait for the agent to be ready (startup script).
2024-09-18 21:47:53 +10:00
DocsURL string
2022-03-28 18:19:28 -06:00
}
// Agent displays a spinning indicator that waits for a workspace agent to connect.
2023-07-12 10:21:54 -05:00
func Agent ( ctx context . Context , writer io . Writer , agentID uuid . UUID , opts AgentOptions ) error {
ctx , cancel := context . WithCancel ( ctx )
defer cancel ()
2022-03-28 18:19:28 -06:00
if opts . FetchInterval == 0 {
opts . FetchInterval = 500 * time . Millisecond
}
2023-06-28 11:54:13 +03:00
if opts . FetchLogs == nil {
2023-07-28 09:57:23 -06:00
opts . FetchLogs = func ( _ context . Context , _ uuid . UUID , _ int64 , _ bool ) ( <- chan [] codersdk . WorkspaceAgentLog , io . Closer , error ) {
c := make ( chan [] codersdk . WorkspaceAgentLog )
2023-06-28 11:54:13 +03:00
close ( c )
return c , closeFunc ( func () error { return nil }), nil
}
2022-03-28 18:19:28 -06:00
}
2023-06-28 11:54:13 +03:00
type fetchAgent struct {
agent codersdk . WorkspaceAgent
err error
}
fetchedAgent := make ( chan fetchAgent , 1 )
go func () {
t := time . NewTimer ( 0 )
defer t . Stop ()
for {
select {
case <- ctx . Done ():
return
case <- t . C :
2023-07-12 10:21:54 -05:00
agent , err := opts . Fetch ( ctx , agentID )
2023-06-28 11:54:13 +03:00
select {
case <- fetchedAgent :
default :
}
if err != nil {
fetchedAgent <- fetchAgent { err : xerrors . Errorf ( "fetch workspace agent: %w" , err )}
return
}
fetchedAgent <- fetchAgent { agent : agent }
t . Reset ( opts . FetchInterval )
}
}
}()
fetch := func () ( codersdk . WorkspaceAgent , error ) {
select {
case <- ctx . Done ():
return codersdk . WorkspaceAgent {}, ctx . Err ()
case f := <- fetchedAgent :
if f . err != nil {
return codersdk . WorkspaceAgent {}, f . err
}
return f . agent , nil
}
}
agent , err := fetch ()
2022-03-28 18:19:28 -06:00
if err != nil {
return xerrors . Errorf ( "fetch: %w" , err )
}
2023-09-25 16:47:17 -05:00
logSources := map [ uuid . UUID ] codersdk . WorkspaceAgentLogSource {}
for _ , source := range agent . LogSources {
logSources [ source . ID ] = source
}
2022-11-09 17:27:05 +02:00
2023-06-28 11:54:13 +03:00
sw := & stageWriter { w : writer }
2022-11-09 17:27:05 +02:00
2023-06-28 11:54:13 +03:00
showStartupLogs := false
2022-03-28 18:19:28 -06:00
for {
2023-06-28 11:54:13 +03:00
// It doesn't matter if we're connected or not, if the agent is
// shutting down, we don't know if it's coming back.
if agent . LifecycleState . ShuttingDown () {
return errAgentShuttingDown
2022-03-28 18:19:28 -06:00
}
2023-06-28 11:54:13 +03:00
2022-11-09 17:27:05 +02:00
switch agent . Status {
2023-06-28 11:54:13 +03:00
case codersdk . WorkspaceAgentConnecting , codersdk . WorkspaceAgentTimeout :
// Since we were waiting for the agent to connect, also show
// startup logs if applicable.
showStartupLogs = true
stage := "Waiting for the workspace agent to connect"
sw . Start ( stage )
for agent . Status == codersdk . WorkspaceAgentConnecting {
if agent , err = fetch (); err != nil {
return xerrors . Errorf ( "fetch: %w" , err )
}
}
if agent . Status == codersdk . WorkspaceAgentTimeout {
now := time . Now ()
sw . Log ( now , codersdk . LogLevelInfo , "The workspace agent is having trouble connecting, wait for it to connect or restart your workspace." )
2025-02-03 20:26:20 +05:00
sw . Log ( now , codersdk . LogLevelInfo , troubleshootingMessage ( agent , fmt . Sprintf ( "%s/admin/templates/troubleshooting#agent-connection-issues" , opts . DocsURL )))
2023-06-28 11:54:13 +03:00
for agent . Status == codersdk . WorkspaceAgentTimeout {
if agent , err = fetch (); err != nil {
return xerrors . Errorf ( "fetch: %w" , err )
2023-01-27 19:05:40 +02:00
}
}
}
2023-06-28 11:54:13 +03:00
sw . Complete ( stage , agent . FirstConnectedAt . Sub ( agent . CreatedAt ))
2022-11-09 17:27:05 +02:00
2023-06-28 11:54:13 +03:00
case codersdk . WorkspaceAgentConnected :
if ! showStartupLogs && agent . LifecycleState == codersdk . WorkspaceAgentLifecycleReady {
// The workspace is ready, there's nothing to do but connect.
return nil
2023-01-27 19:05:40 +02:00
}
2023-09-25 16:47:17 -05:00
stage := "Running workspace agent startup scripts"
2024-07-10 15:39:43 +03:00
follow := opts . Wait && agent . LifecycleState . Starting ()
2023-06-28 11:54:13 +03:00
if ! follow {
stage += " (non-blocking)"
}
sw . Start ( stage )
2024-07-09 14:32:52 +03:00
if follow {
sw . Log ( time . Time {}, codersdk . LogLevelInfo , "==> ℹ ︎ To connect immediately, reconnect with --wait=no or CODER_SSH_WAIT=no, see --help for more information." )
}
2023-01-27 19:05:40 +02:00
2023-06-28 11:54:13 +03:00
err = func () error { // Use func because of defer in for loop.
logStream , logsCloser , err := opts . FetchLogs ( ctx , agent . ID , 0 , follow )
if err != nil {
return xerrors . Errorf ( "fetch workspace agent startup logs: %w" , err )
}
defer logsCloser . Close ()
2023-07-28 09:57:23 -06:00
var lastLog codersdk . WorkspaceAgentLog
2023-07-18 09:57:29 -06:00
fetchedAgentWhileFollowing := fetchedAgent
if ! follow {
fetchedAgentWhileFollowing = nil
}
2023-06-28 11:54:13 +03:00
for {
// This select is essentially and inline `fetch()`.
select {
case <- ctx . Done ():
return ctx . Err ()
2023-07-18 09:57:29 -06:00
case f := <- fetchedAgentWhileFollowing :
2023-06-28 11:54:13 +03:00
if f . err != nil {
return xerrors . Errorf ( "fetch: %w" , f . err )
}
agent = f . agent
2023-07-18 09:57:29 -06:00
// If the agent is no longer starting, stop following
// logs because FetchLogs will keep streaming forever.
// We do one last non-follow request to ensure we have
// fetched all logs.
if ! agent . LifecycleState . Starting () {
_ = logsCloser . Close ()
fetchedAgentWhileFollowing = nil
logStream , logsCloser , err = opts . FetchLogs ( ctx , agent . ID , lastLog . ID , false )
if err != nil {
return xerrors . Errorf ( "fetch workspace agent startup logs: %w" , err )
}
// Logs are already primed, so we can call close.
_ = logsCloser . Close ()
}
2023-06-28 11:54:13 +03:00
case logs , ok := <- logStream :
if ! ok {
return nil
}
for _ , log := range logs {
2023-09-25 16:47:17 -05:00
source , hasSource := logSources [ log . SourceID ]
output := log . Output
if hasSource && source . DisplayName != "" {
output = source . DisplayName + ": " + output
}
sw . Log ( log . CreatedAt , log . Level , output )
2023-07-18 09:57:29 -06:00
lastLog = log
2023-06-28 11:54:13 +03:00
}
}
}
}()
if err != nil {
return err
}
for follow && agent . LifecycleState . Starting () {
if agent , err = fetch (); err != nil {
return xerrors . Errorf ( "fetch: %w" , err )
}
}
2023-01-27 19:05:40 +02:00
2023-03-06 21:34:00 +02:00
switch agent . LifecycleState {
2023-06-28 11:54:13 +03:00
case codersdk . WorkspaceAgentLifecycleReady :
2024-01-04 15:51:48 -05:00
sw . Complete ( stage , safeDuration ( sw , agent . ReadyAt , agent . StartedAt ))
2023-09-25 16:47:17 -05:00
case codersdk . WorkspaceAgentLifecycleStartTimeout :
2024-07-09 21:55:16 +03:00
// Backwards compatibility: Avoid printing warning if
// coderd is old and doesn't set ReadyAt for timeouts.
if agent . ReadyAt == nil {
sw . Fail ( stage , 0 )
} else {
sw . Fail ( stage , safeDuration ( sw , agent . ReadyAt , agent . StartedAt ))
}
2023-09-25 16:47:17 -05:00
sw . Log ( time . Time {}, codersdk . LogLevelWarn , "Warning: A startup script timed out and your workspace may be incomplete." )
2023-06-28 11:54:13 +03:00
case codersdk . WorkspaceAgentLifecycleStartError :
2024-01-04 15:51:48 -05:00
sw . Fail ( stage , safeDuration ( sw , agent . ReadyAt , agent . StartedAt ))
2023-06-28 11:54:13 +03:00
// Use zero time (omitted) to separate these from the startup logs.
2023-09-25 16:47:17 -05:00
sw . Log ( time . Time {}, codersdk . LogLevelWarn , "Warning: A startup script exited with an error and your workspace may be incomplete." )
2025-02-03 20:26:20 +05:00
sw . Log ( time . Time {}, codersdk . LogLevelWarn , troubleshootingMessage ( agent , fmt . Sprintf ( "%s/admin/templates/troubleshooting#startup-script-exited-with-an-error" , opts . DocsURL )))
2023-06-28 11:54:13 +03:00
default :
switch {
case agent . LifecycleState . Starting ():
// Use zero time (omitted) to separate these from the startup logs.
2023-09-25 16:47:17 -05:00
sw . Log ( time . Time {}, codersdk . LogLevelWarn , "Notice: The startup scripts are still running and your workspace may be incomplete." )
2025-02-03 20:26:20 +05:00
sw . Log ( time . Time {}, codersdk . LogLevelWarn , troubleshootingMessage ( agent , fmt . Sprintf ( "%s/admin/templates/troubleshooting#your-workspace-may-be-incomplete" , opts . DocsURL )))
2023-06-28 11:54:13 +03:00
// Note: We don't complete or fail the stage here, it's
// intentionally left open to indicate this stage didn't
// complete.
case agent . LifecycleState . ShuttingDown ():
// We no longer know if the startup script failed or not,
// but we need to tell the user something.
2024-01-04 15:51:48 -05:00
sw . Complete ( stage , safeDuration ( sw , agent . ReadyAt , agent . StartedAt ))
2023-06-28 11:54:13 +03:00
return errAgentShuttingDown
}
2023-03-06 21:34:00 +02:00
}
2023-06-28 11:54:13 +03:00
return nil
case codersdk . WorkspaceAgentDisconnected :
// If the agent was still starting during disconnect, we'll
// show startup logs.
showStartupLogs = agent . LifecycleState . Starting ()
stage := "The workspace agent lost connection"
sw . Start ( stage )
sw . Log ( time . Now (), codersdk . LogLevelWarn , "Wait for it to reconnect or restart your workspace." )
2025-02-03 20:26:20 +05:00
sw . Log ( time . Now (), codersdk . LogLevelWarn , troubleshootingMessage ( agent , fmt . Sprintf ( "%s/admin/templates/troubleshooting#agent-connection-issues" , opts . DocsURL )))
2023-09-27 19:10:15 +03:00
2024-01-04 15:51:48 -05:00
disconnectedAt := agent . DisconnectedAt
2023-06-28 11:54:13 +03:00
for agent . Status == codersdk . WorkspaceAgentDisconnected {
if agent , err = fetch (); err != nil {
return xerrors . Errorf ( "fetch: %w" , err )
}
}
2024-01-04 15:51:48 -05:00
sw . Complete ( stage , safeDuration ( sw , agent . LastConnectedAt , disconnectedAt ))
2023-01-27 19:05:40 +02:00
}
2022-11-09 17:27:05 +02:00
}
2023-06-28 11:54:13 +03:00
}
func troubleshootingMessage ( agent codersdk . WorkspaceAgent , url string ) string {
m := "For more information and troubleshooting, see " + url
if agent . TroubleshootingURL != "" {
m += " and " + agent . TroubleshootingURL
}
2023-01-27 19:05:40 +02:00
return m
2022-11-09 17:27:05 +02:00
}
2023-06-28 11:54:13 +03:00
2024-01-04 15:51:48 -05:00
// safeDuration returns a-b. If a or b is nil, it returns 0.
// This is because we often dereference a time pointer, which can
// cause a panic. These dereferences are used to calculate durations,
// which are not critical, and therefor should not break things
// when it fails.
// A panic has been observed in a test.
func safeDuration ( sw * stageWriter , a , b * time . Time ) time . Duration {
if a == nil || b == nil {
if sw != nil {
// Ideally the message includes which fields are <nil>, but you can
// use the surrounding log lines to figure that out. And passing more
// params makes this unwieldy.
sw . Log ( time . Now (), codersdk . LogLevelWarn , "Warning: Failed to calculate duration from a time being <nil>." )
}
return 0
}
return a . Sub ( * b )
}
2023-06-28 11:54:13 +03:00
type closeFunc func () error
func ( c closeFunc ) Close () error {
return c ()
}
2024-02-27 22:04:46 +04:00
func PeerDiagnostics ( w io . Writer , d tailnet . PeerDiagnostics ) {
if d . PreferredDERP > 0 {
rn , ok := d . DERPRegionNames [ d . PreferredDERP ]
if ! ok {
rn = "unknown"
}
_ , _ = fmt . Fprintf ( w , "✔ preferred DERP region: %d (%s)\n" , d . PreferredDERP , rn )
} else {
_ , _ = fmt . Fprint ( w , "✘ not connected to DERP\n" )
}
if d . SentNode {
2024-09-05 13:32:08 +10:00
_ , _ = fmt . Fprint ( w , "✔ sent local data to Coder networking coordinator\n" )
2024-02-27 22:04:46 +04:00
} else {
_ , _ = fmt . Fprint ( w , "✘ have not sent local data to Coder networking coordinator\n" )
}
if d . ReceivedNode != nil {
dp := d . ReceivedNode . DERP
dn := ""
// should be 127.3.3.40:N where N is the DERP region
ap := strings . Split ( dp , ":" )
if len ( ap ) == 2 {
dp = ap [ 1 ]
di , err := strconv . Atoi ( dp )
if err == nil {
var ok bool
dn , ok = d . DERPRegionNames [ di ]
if ok {
dn = fmt . Sprintf ( "(%s)" , dn )
} else {
dn = "(unknown)"
}
}
}
_ , _ = fmt . Fprintf ( w ,
"✔ received remote agent data from Coder networking coordinator\n preferred DERP region: %s %s\n endpoints: %s\n" ,
dp , dn , strings . Join ( d . ReceivedNode . Endpoints , ", " ))
} else {
_ , _ = fmt . Fprint ( w , "✘ have not received remote agent data from Coder networking coordinator\n" )
}
if ! d . LastWireguardHandshake . IsZero () {
ago := time . Since ( d . LastWireguardHandshake )
symbol := "✔"
// wireguard is supposed to refresh handshake on 5 minute intervals
if ago > 5 * time . Minute {
symbol = "⚠"
}
_ , _ = fmt . Fprintf ( w , "%s Wireguard handshake %s ago\n" , symbol , ago . Round ( time . Second ))
} else {
_ , _ = fmt . Fprint ( w , "✘ Wireguard is not connected\n" )
}
}
2024-08-28 15:39:01 +10:00
type ConnDiags struct {
2024-09-18 21:47:53 +10:00
ConnInfo workspacesdk . AgentConnectionInfo
PingP2P bool
DisableDirect bool
LocalNetInfo * tailcfg . NetInfo
LocalInterfaces * healthsdk . InterfacesReport
AgentNetcheck * healthsdk . AgentNetcheckReport
ClientIPIsAWS bool
AgentIPIsAWS bool
Verbose bool
TroubleshootingURL string
2024-08-28 15:39:01 +10:00
}
2024-08-29 21:41:12 +10:00
func ( d ConnDiags ) Write ( w io . Writer ) {
_ , _ = fmt . Fprintln ( w , "" )
general , client , agent := d . splitDiagnostics ()
for _ , msg := range general {
_ , _ = fmt . Fprintln ( w , msg )
}
2024-09-25 13:24:23 +10:00
if len ( general ) > 0 {
_ , _ = fmt . Fprintln ( w , "" )
}
2024-08-29 21:41:12 +10:00
if len ( client ) > 0 {
_ , _ = fmt . Fprint ( w , "Possible client-side issues with direct connection:\n\n" )
for _ , msg := range client {
_ , _ = fmt . Fprintf ( w , " - %s\n\n" , msg )
}
}
if len ( agent ) > 0 {
_ , _ = fmt . Fprint ( w , "Possible agent-side issues with direct connections:\n\n" )
for _ , msg := range agent {
_ , _ = fmt . Fprintf ( w , " - %s\n\n" , msg )
}
}
}
func ( d ConnDiags ) splitDiagnostics () ( general , client , agent [] string ) {
2024-08-28 15:39:01 +10:00
if d . AgentNetcheck != nil {
for _ , msg := range d . AgentNetcheck . Interfaces . Warnings {
2024-08-29 21:41:12 +10:00
agent = append ( agent , msg . Message )
2024-08-28 15:39:01 +10:00
}
2024-09-05 13:32:08 +10:00
if len ( d . AgentNetcheck . Interfaces . Warnings ) > 0 {
2024-09-18 21:47:53 +10:00
agent [ len ( agent ) - 1 ] += fmt . Sprintf ( "\n%s#low-mtu" , d . TroubleshootingURL )
2024-09-05 13:32:08 +10:00
}
2024-08-28 15:39:01 +10:00
}
if d . LocalInterfaces != nil {
for _ , msg := range d . LocalInterfaces . Warnings {
2024-08-29 21:41:12 +10:00
client = append ( client , msg . Message )
2024-08-28 15:39:01 +10:00
}
2024-09-05 13:32:08 +10:00
if len ( d . LocalInterfaces . Warnings ) > 0 {
2024-09-18 21:47:53 +10:00
client [ len ( client ) - 1 ] += fmt . Sprintf ( "\n%s#low-mtu" , d . TroubleshootingURL )
2024-09-05 13:32:08 +10:00
}
2024-08-28 15:39:01 +10:00
}
2024-08-29 21:41:12 +10:00
if d . PingP2P && ! d . Verbose {
return general , client , agent
2024-08-28 15:39:01 +10:00
}
if d . DisableDirect {
2024-11-12 17:33:35 +11:00
general = append ( general , "❗ Direct connections are disabled locally, by `--disable-direct-connections` or `CODER_DISABLE_DIRECT_CONNECTIONS`.\n" +
" They may still be established over a private network." )
2024-08-29 21:41:12 +10:00
if ! d . Verbose {
return general , client , agent
}
2024-08-28 15:39:01 +10:00
}
2024-08-29 21:41:12 +10:00
if d . ConnInfo . DisableDirectConnections {
2024-09-18 21:47:53 +10:00
general = append ( general ,
fmt . Sprintf ( "❗ Your Coder administrator has blocked direct connections\n %s#disabled-deployment-wide" , d . TroubleshootingURL ))
2024-08-29 21:41:12 +10:00
if ! d . Verbose {
return general , client , agent
}
2024-08-28 15:39:01 +10:00
}
2024-08-29 21:41:12 +10:00
if ! d . ConnInfo . DERPMap . HasSTUN () {
2024-09-18 21:47:53 +10:00
general = append ( general ,
fmt . Sprintf ( "❗ The DERP map is not configured to use STUN\n %s#no-stun-servers" , d . TroubleshootingURL ))
2024-08-29 21:41:12 +10:00
} else if d . LocalNetInfo != nil && ! d . LocalNetInfo . UDP {
2024-09-18 21:47:53 +10:00
client = append ( client ,
fmt . Sprintf ( "Client could not connect to STUN over UDP\n %s#udp-blocked" , d . TroubleshootingURL ))
2024-08-28 15:39:01 +10:00
}
if d . LocalNetInfo != nil && d . LocalNetInfo . MappingVariesByDestIP . EqualBool ( true ) {
2024-09-18 21:47:53 +10:00
client = append ( client ,
fmt . Sprintf ( "Client is potentially behind a hard NAT, as multiple endpoints were retrieved from different STUN servers\n %s#endpoint-dependent-nat-hard-nat" , d . TroubleshootingURL ))
2024-08-28 15:39:01 +10:00
}
2024-08-29 21:41:12 +10:00
if d . AgentNetcheck != nil && d . AgentNetcheck . NetInfo != nil {
if d . AgentNetcheck . NetInfo . MappingVariesByDestIP . EqualBool ( true ) {
2024-09-18 21:47:53 +10:00
agent = append ( agent ,
fmt . Sprintf ( "Agent is potentially behind a hard NAT, as multiple endpoints were retrieved from different STUN servers\n %s#endpoint-dependent-nat-hard-nat" , d . TroubleshootingURL ))
2024-08-29 21:41:12 +10:00
}
if ! d . AgentNetcheck . NetInfo . UDP {
2024-09-18 21:47:53 +10:00
agent = append ( agent ,
fmt . Sprintf ( "Agent could not connect to STUN over UDP\n %s#udp-blocked" , d . TroubleshootingURL ))
2024-08-29 21:41:12 +10:00
}
2024-08-28 15:39:01 +10:00
}
2024-08-29 21:41:12 +10:00
if d . ClientIPIsAWS {
2024-09-18 21:47:53 +10:00
client = append ( client ,
fmt . Sprintf ( "Client IP address is within an AWS range (AWS uses hard NAT)\n %s#endpoint-dependent-nat-hard-nat" , d . TroubleshootingURL ))
2024-08-29 21:41:12 +10:00
}
if d . AgentIPIsAWS {
2024-09-18 21:47:53 +10:00
agent = append ( agent ,
fmt . Sprintf ( "Agent IP address is within an AWS range (AWS uses hard NAT)\n %s#endpoint-dependent-nat-hard-nat" , d . TroubleshootingURL ))
2024-08-29 21:41:12 +10:00
}
2024-09-25 13:24:23 +10:00
2024-08-29 21:41:12 +10:00
return general , client , agent
2024-08-28 15:39:01 +10:00
}