package pubsub import ( "context" "database/sql" "database/sql/driver" "errors" "io" "net" "sync" "sync/atomic" "time" "github.com/lib/pq" "golang.org/x/xerrors" "cdr.dev/slog/v3" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/healthcheck/health" "github.com/coder/coder/v2/codersdk/healthsdk" ) // Listener represents a pubsub handler. type Listener func(ctx context.Context, message []byte) // ListenerWithErr represents a pubsub handler that can also receive error // indications type ListenerWithErr func(ctx context.Context, message []byte, err error) // ErrDroppedMessages is sent to ListenerWithErr if messages are dropped or // might have been dropped. var ErrDroppedMessages = xerrors.New("dropped messages") // latencyMeasureInterval is how often the background loop runs a latency // measurement. It is intentionally shorter than a typical metrics scrape // interval (the coder/observability stack scrapes every 15s) so every scrape // observes a fresh measurement. const latencyMeasureInterval = time.Second * 10 // latencyMeasureTimeout bounds a single latency measurement. It is shorter // than latencyMeasureInterval so a hung probe cannot bleed into the next // interval; a timed-out probe is recorded as a measurement error. const latencyMeasureTimeout = time.Second * 5 type Subscriber interface { Subscribe(event string, listener Listener) (cancel func(), err error) SubscribeWithErr(event string, listener ListenerWithErr) (cancel func(), err error) } type Publisher interface { Publish(event string, message []byte) error } // Pubsub is a generic interface for broadcasting and receiving messages. // Implementors should assume high-availability with the backing implementation. type Pubsub interface { Subscriber Publisher Close() error } // ReportingPubsub is a Pubsub that can report on its own health. type ReportingPubsub interface { Pubsub // HealthReport returns the health of the pubsub backend. HealthReport(ctx context.Context) healthsdk.PubsubReport } // msgOrErr either contains a message or an error type msgOrErr struct { msg []byte err error } // MsgQueue implements a fixed length queue with the ability to replace elements // after they are queued (but before they are dequeued). // // The purpose of this data structure is to build something that works a bit // like a golang channel, but if the queue is full, then we can replace the // last element with an error so that the subscriber can get notified that some // messages were dropped, all without blocking. type MsgQueue struct { ctx context.Context cond *sync.Cond q [BufferSize]msgOrErr front int size int closed bool l Listener le ListenerWithErr } func NewMsgQueue(ctx context.Context, l Listener, le ListenerWithErr) *MsgQueue { if l == nil && le == nil { panic("l or le must be non-nil") } q := &MsgQueue{ ctx: ctx, cond: sync.NewCond(&sync.Mutex{}), l: l, le: le, } go q.run() return q } func (q *MsgQueue) run() { for { // wait until there is something on the queue or we are closed q.cond.L.Lock() for q.size == 0 && !q.closed { q.cond.Wait() } if q.closed { q.cond.L.Unlock() return } item := q.q[q.front] q.front = (q.front + 1) % BufferSize q.size-- q.cond.L.Unlock() // process item without holding lock if item.err == nil { // real message if q.l != nil { q.l(q.ctx, item.msg) continue } if q.le != nil { q.le(q.ctx, item.msg, nil) continue } // unhittable continue } // if the listener wants errors, send it. if q.le != nil { q.le(q.ctx, nil, item.err) } } } func (q *MsgQueue) Enqueue(msg []byte) { q.cond.L.Lock() defer q.cond.L.Unlock() if q.size == BufferSize { // queue is full, so we're going to drop the msg we got called with. // We also need to record that messages are being dropped, which we // do at the last message in the queue. This potentially makes us // lose 2 messages instead of one, but it's more important at this // point to warn the subscriber that they're losing messages so they // can do something about it. back := (q.front + BufferSize - 1) % BufferSize q.q[back].msg = nil q.q[back].err = ErrDroppedMessages return } // queue is not full, insert the message next := (q.front + q.size) % BufferSize q.q[next].msg = msg q.q[next].err = nil q.size++ q.cond.Broadcast() } func (q *MsgQueue) Close() { q.cond.L.Lock() defer q.cond.L.Unlock() defer q.cond.Broadcast() q.closed = true } // Dropped records an error in the queue that messages might have been Dropped func (q *MsgQueue) Dropped() { q.cond.L.Lock() defer q.cond.L.Unlock() if q.size == BufferSize { // queue is full, but we need to record that messages are being dropped, // which we do at the last message in the queue. This potentially drops // another message, but it's more important for the subscriber to know. back := (q.front + BufferSize - 1) % BufferSize q.q[back].msg = nil q.q[back].err = ErrDroppedMessages return } // queue is not full, insert the error next := (q.front + q.size) % BufferSize q.q[next].msg = nil q.q[next].err = ErrDroppedMessages q.size++ q.cond.Broadcast() } // pqListener is an interface that represents a *pq.Listener for testing type pqListener interface { io.Closer Listen(string) error Unlisten(string) error NotifyChan() <-chan *pq.Notification } type pqListenerShim struct { *pq.Listener } func (l pqListenerShim) NotifyChan() <-chan *pq.Notification { return l.Notify } type queueSet struct { m map[*MsgQueue]struct{} // unlistenInProgress will be non-nil if another goroutine is unlistening for the event this // queueSet corresponds to. If non-nil, that goroutine will close the channel when it is done. unlistenInProgress chan struct{} } func newQueueSet() *queueSet { return &queueSet{ m: make(map[*MsgQueue]struct{}), } } // PGPubsub is a pubsub implementation using PostgreSQL. type PGPubsub struct { logger slog.Logger listenDone chan struct{} pgListener pqListener db *sql.DB qMu sync.Mutex queues map[string]*queueSet // making the close state its own mutex domain simplifies closing logic so // that we don't have to hold the qMu --- which could block processing // notifications while the pqListener is closing. closeMu sync.Mutex closedListener bool closeListenerErr error // connected tracks whether pgListener currently has a connection to // PostgreSQL, as reported by its event callback. connected atomic.Bool metrics *BackendMetrics } // BufferSize is the maximum number of unhandled messages we will buffer // for a subscriber before dropping messages. const BufferSize = 2048 // Subscribe calls the listener when an event matching the name is received. func (p *PGPubsub) Subscribe(event string, listener Listener) (cancel func(), err error) { return p.subscribeQueue(event, NewMsgQueue(context.Background(), listener, nil)) } func (p *PGPubsub) SubscribeWithErr(event string, listener ListenerWithErr) (cancel func(), err error) { return p.subscribeQueue(event, NewMsgQueue(context.Background(), nil, listener)) } func (p *PGPubsub) subscribeQueue(event string, newQ *MsgQueue) (cancel func(), err error) { defer func() { if err != nil { // if we hit an error, we need to close the queue so we don't // leak its goroutine. newQ.Close() p.metrics.RecordSubscribeFailure(event) } else { p.metrics.RecordSubscribeSuccess(event) } }() var ( unlistenInProgress <-chan struct{} // MUST hold the p.qMu lock to manipulate this! qs *queueSet ) func() { p.qMu.Lock() defer p.qMu.Unlock() var ok bool if qs, ok = p.queues[event]; !ok { qs = newQueueSet() p.queues[event] = qs p.metrics.AddEvent(event) } qs.m[newQ] = struct{}{} p.metrics.AddSubscriber(event) unlistenInProgress = qs.unlistenInProgress }() // NOTE there cannot be any `return` statements between here and the next +-+, otherwise the // assumptions the defer makes could be violated if unlistenInProgress != nil { // We have to wait here because we don't want our `Listen` call to happen before the other // goroutine calls `Unlisten`. That would result in this subscription not getting any // events. c.f. https://github.com/coder/coder/issues/15312 p.logger.Debug(context.Background(), "waiting for Unlisten in progress", slog.F("event", event)) <-unlistenInProgress p.logger.Debug(context.Background(), "unlistening complete", slog.F("event", event)) } // +-+ (see above) defer func() { if err != nil { p.qMu.Lock() defer p.qMu.Unlock() delete(qs.m, newQ) p.metrics.RemoveSubscriber(event) if len(qs.m) == 0 { // we know that newQ was in the queueSet since we last unlocked, so there cannot // have been any _new_ goroutines trying to Unlisten(). Therefore, if the queueSet // is now empty, it's safe to delete. delete(p.queues, event) p.metrics.RemoveEvent(event) } } }() // The pgListener waits for the response to `LISTEN` on a mainloop that also dispatches // notifies. We need to avoid holding the mutex while this happens, since holding the mutex // blocks reading notifications and can deadlock the pgListener. // c.f. https://github.com/coder/coder/issues/11950 err = p.pgListener.Listen(event) if err == nil { p.logger.Debug(context.Background(), "started listening to event channel", slog.F("event", event)) } if errors.Is(err, pq.ErrChannelAlreadyOpen) { // It's ok if it's already open! err = nil } if err != nil { return nil, xerrors.Errorf("listen: %w", err) } return func() { var unlistening chan struct{} func() { p.qMu.Lock() defer p.qMu.Unlock() newQ.Close() qSet, ok := p.queues[event] if !ok { p.logger.Critical(context.Background(), "event was removed before cancel", slog.F("event", event)) return } delete(qSet.m, newQ) p.metrics.RemoveSubscriber(event) if len(qSet.m) == 0 { unlistening = make(chan struct{}) qSet.unlistenInProgress = unlistening } }() // as above, we must not hold the lock while calling into pgListener if unlistening != nil { uErr := p.pgListener.Unlisten(event) close(unlistening) // we can now delete the queueSet if it is empty. func() { p.qMu.Lock() defer p.qMu.Unlock() qSet, ok := p.queues[event] if ok && len(qSet.m) == 0 { p.logger.Debug(context.Background(), "removing queueSet", slog.F("event", event)) delete(p.queues, event) p.metrics.RemoveEvent(event) } }() p.closeMu.Lock() defer p.closeMu.Unlock() if uErr != nil && !p.closedListener { p.logger.Warn(context.Background(), "failed to unlisten", slog.Error(uErr), slog.F("event", event)) } else { p.logger.Debug(context.Background(), "stopped listening to event channel", slog.F("event", event)) } } }, nil } func (p *PGPubsub) Publish(event string, message []byte) error { p.logger.Debug(context.Background(), "publish", slog.F("event", event), slog.F("message_len", len(message))) // This is safe because we are calling pq.QuoteLiteral. pg_notify doesn't // support the first parameter being a prepared statement. //nolint:gosec _, err := p.db.ExecContext(context.Background(), `select pg_notify(`+pq.QuoteLiteral(event)+`, $1)`, message) if err != nil { p.metrics.RecordPublishFailure(event) return xerrors.Errorf("exec pg_notify: %w", err) } p.metrics.RecordPublishSuccess(event, len(message)) return nil } // HealthReport implements ReportingPubsub. It warns when the listener is not // connected to PostgreSQL, since no messages are received until it reconnects. func (p *PGPubsub) HealthReport(context.Context) healthsdk.PubsubReport { report := healthsdk.PubsubReport{ BaseReport: healthsdk.BaseReport{ Severity: health.SeverityOK, Warnings: []health.Message{}, }, Backend: BackendPostgres, } if !p.connected.Load() { report.Severity = health.SeverityWarning report.Warnings = append(report.Warnings, health.Messagef(health.CodePubsubNotConnected, "pubsub listener is not connected to PostgreSQL")) } return report } // Close closes the pubsub instance. func (p *PGPubsub) Close() error { p.logger.Info(context.Background(), "pubsub is closing") // Close the listener before waiting for the latency loop. While Postgres // is unreachable, an in-flight probe blocks in pq.Listener.Listen until // the listener reconnects or closes, so waiting first can hang forever. err := p.closeListener() p.metrics.StopLatencyLoop() <-p.listenDone p.logger.Debug(context.Background(), "pubsub closed") return err } // closeListener closes the pgListener, unless it has already been closed. func (p *PGPubsub) closeListener() error { p.closeMu.Lock() defer p.closeMu.Unlock() if p.closedListener { return p.closeListenerErr } p.closedListener = true p.closeListenerErr = p.pgListener.Close() p.connected.Store(false) return p.closeListenerErr } // listen begins receiving messages on the pq listener. func (p *PGPubsub) listen() { defer func() { p.logger.Info(context.Background(), "pubsub listen stopped receiving notify") close(p.listenDone) }() notify := p.pgListener.NotifyChan() for notif := range notify { // A nil notification can be dispatched on reconnect. if notif == nil { p.logger.Debug(context.Background(), "notifying subscribers of a reconnection") p.recordReconnect() continue } p.listenReceive(notif) } } func (p *PGPubsub) listenReceive(notif *pq.Notification) { p.metrics.RecordReceived(notif.Channel, []byte(notif.Extra)) p.qMu.Lock() defer p.qMu.Unlock() qSet, ok := p.queues[notif.Channel] if !ok { return } extra := []byte(notif.Extra) for q := range qSet.m { q.Enqueue(extra) } } func (p *PGPubsub) recordReconnect() { p.qMu.Lock() defer p.qMu.Unlock() for _, qSet := range p.queues { for q := range qSet.m { q.Dropped() } } } // logDialer is a pq.Dialer and pq.DialerContext that logs when it starts // connecting and when the TCP connection is established. type logDialer struct { logger slog.Logger d net.Dialer } var ( _ pq.Dialer = logDialer{} _ pq.DialerContext = logDialer{} ) func (d logDialer) Dial(network, address string) (net.Conn, error) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() return d.DialContext(ctx, network, address) } func (d logDialer) DialTimeout(network, address string, timeout time.Duration) (net.Conn, error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return d.DialContext(ctx, network, address) } func (d logDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { deadline, hasDeadline := ctx.Deadline() timeoutMS := 0 if hasDeadline { timeoutMS = int(time.Until(deadline) / time.Millisecond) } logger := d.logger.With(slog.F("network", network), slog.F("address", address), slog.F("timeout_ms", timeoutMS)) logger.Debug(ctx, "pubsub dialing postgres") start := time.Now() conn, err := d.d.DialContext(ctx, network, address) if err != nil { logger.Error(ctx, "pubsub failed to dial postgres") return nil, err } elapsed := time.Since(start) logger.Debug(ctx, "pubsub postgres TCP connection established", slog.F("elapsed_ms", elapsed.Milliseconds())) return conn, nil } func (p *PGPubsub) startListener(ctx context.Context, connectURL string) error { p.metrics.MarkDisconnected() // Creates a new listener using pq. var ( dialer = logDialer{ logger: p.logger, // pq.defaultDialer uses a zero net.Dialer as well. d: net.Dialer{}, } connector driver.Connector err error ) // Create a custom connector if the database driver supports it. connectorCreator, ok := p.db.Driver().(database.ConnectorCreator) if ok { connector, err = connectorCreator.Connector(connectURL) if err != nil { return xerrors.Errorf("create custom connector: %w", err) } } else { // use the default pq connector otherwise connector, err = pq.NewConnector(connectURL) if err != nil { return xerrors.Errorf("create pq connector: %w", err) } } // Set the dialer if the connector supports it. dc, ok := connector.(database.DialerConnector) if !ok { p.logger.Critical(ctx, "connector does not support setting log dialer, database connection debug logs will be missing") } else { dc.Dialer(dialer) } var ( errCh = make(chan error, 1) sentErrCh = false ) p.pgListener = pqListenerShim{ Listener: pq.NewConnectorListener(connector, connectURL, time.Second, time.Minute, func(t pq.ListenerEventType, err error) { switch t { case pq.ListenerEventConnected: p.logger.Debug(ctx, "pubsub connected to postgres") p.connected.Store(true) p.metrics.MarkConnected() case pq.ListenerEventDisconnected: p.logger.Error(ctx, "pubsub disconnected from postgres", slog.Error(err)) p.connected.Store(false) p.metrics.RecordDisconnect() p.metrics.MarkDisconnected() case pq.ListenerEventReconnected: p.logger.Info(ctx, "pubsub reconnected to postgres") p.connected.Store(true) p.metrics.MarkConnected() case pq.ListenerEventConnectionAttemptFailed: p.logger.Error(ctx, "pubsub failed to connect to postgres", slog.Error(err)) } // This callback gets events whenever the connection state changes. // Only send the first error. if sentErrCh { return } errCh <- err // won't block because we are buffered. sentErrCh = true }), } // We don't respect context cancellation here. There's a bug in the pq library // where if you close the listener before or while the connection is being // established, the connection will be established anyway, and will not be // closed. // https://github.com/lib/pq/issues/1192 if err := <-errCh; err != nil { _ = p.pgListener.Close() return xerrors.Errorf("create pq listener: %w", err) } return nil } // We'll track messages as size "normal" and "colossal", where the // latter are messages larger than 7600 bytes, or 95% of the postgres // notify limit. If we see a lot of colossal packets that's an indication that // we might be trying to send too much data over the pubsub and are in danger of // failing to publish. // // These are exported so other pubsub implementations (e.g. the NATS // pubsub) classify message size identically, keeping the messages_total // "size" label consistent across backends. const ( ColossalThreshold = 7600 MessageSizeNormal = "normal" MessageSizeColossal = "colossal" ) // New creates a new Pubsub implementation using a PostgreSQL connection. The // metrics argument is the shared pubsub metrics; pass nil to disable metrics // (an unregistered set is created internally). func New(startCtx context.Context, logger slog.Logger, db *sql.DB, connectURL string, metrics *Metrics) (*PGPubsub, error) { p := newWithoutListener(logger, db, metrics) if err := p.startListener(startCtx, connectURL); err != nil { return nil, err } go p.listen() // Probe latency out-of-band. It records into no-op instruments when // Prometheus is disabled, but always runs so the loop lifecycle does not // depend on whether metrics are collected. p.metrics.StartLatencyLoop(p) logger.Debug(startCtx, "pubsub has started") return p, nil } // newWithoutListener creates a new PGPubsub without creating the pqListener. func newWithoutListener(logger slog.Logger, db *sql.DB, metrics *Metrics) *PGPubsub { if metrics == nil { metrics = NewMetrics(nil) } return &PGPubsub{ logger: logger, listenDone: make(chan struct{}), db: db, queues: make(map[string]*queueSet), metrics: metrics.ForBackend(logger, BackendPostgres), } }