Fills in the `pubsub` section of the deployment health report on the server. Stacked on #31620. - New `pubsub.ReportingPubsub` interface: `Pubsub` plus `HealthReport(ctx) healthsdk.PubsubReport`. The `Pubsub` interface itself is unchanged. - The PostgreSQL, NATS and in-memory backends implement it. Each reports its backend name (`postgres`, `nats` or `memory`) and its connection state: - **PostgreSQL:** tracks the `pq` listener's connected state from its event callback. When it's not connected, the report has severity `warning` and `EPS01`. - **NATS:** warns with `EPS02` if the embedded server isn't `Running()`, and with `EPS01` for each publish or subscribe connection whose `Status()` isn't `CONNECTED`. - **In-memory:** always `ok`. - `coderd.Options.Pubsub` is now a `ReportingPubsub`. `ReplicaSyncPubsub` stays a plain `Pubsub`. - `coderd/healthcheck` has a new `Pubsub` check, which counts toward `healthy` and the overall severity like the other sections. `formatHealthcheck` applies the dismissed setting for it. - New health codes `EPS01` (not connected) and `EPS02` (embedded NATS server not running), documented in `docs/admin/monitoring/health-check.md`. - Test helpers follow the type change: `coderdtest.Options.Pubsub` and the pubsub returned by `dbtestutil.NewDB` are now `ReportingPubsub`, and a few test wrappers embed `ReportingPubsub` instead of `Pubsub`. <details> <summary>Decision log</summary> - `HealthReport` returns a `healthsdk.PubsubReport` rather than filling one in place. This matches the `healthcheck.Checker` methods. `ctx` is unused for now. - The healthcheck layer sets `Dismissed`, so backends don't need to know about health settings. - Problems are reported as warnings, not errors, so they show on the health page without marking the deployment unhealthy. Both backends reconnect on their own. - Postgres connection state comes from the listener event callback, not from `pq.Listener.Ping`, so the health check doesn't do database I/O. Closing the pubsub also marks it disconnected. - NATS `conn` gains `Status()`, which `*natsgo.Conn` already implements. The pools are immutable after construction, so `HealthReport` reads them without locking. - The docs say the pubsub section appears in the JSON response, because the health UI doesn't show it until a later PR in this stack. - psmock is not regenerated: none of its users pass it to coderd. </details> --- Generated by Coder Agents on behalf of @spikecurtis.
656 lines
19 KiB
Go
656 lines
19 KiB
Go
package pubsub
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"database/sql/driver"
|
|
"errors"
|
|
"io"
|
|
"net"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/lib/pq"
|
|
"golang.org/x/xerrors"
|
|
|
|
"cdr.dev/slog/v3"
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/healthcheck/health"
|
|
"github.com/coder/coder/v2/codersdk/healthsdk"
|
|
)
|
|
|
|
// Listener represents a pubsub handler.
|
|
type Listener func(ctx context.Context, message []byte)
|
|
|
|
// ListenerWithErr represents a pubsub handler that can also receive error
|
|
// indications
|
|
type ListenerWithErr func(ctx context.Context, message []byte, err error)
|
|
|
|
// ErrDroppedMessages is sent to ListenerWithErr if messages are dropped or
|
|
// might have been dropped.
|
|
var ErrDroppedMessages = xerrors.New("dropped messages")
|
|
|
|
// latencyMeasureInterval is how often the background loop runs a latency
|
|
// measurement. It is intentionally shorter than a typical metrics scrape
|
|
// interval (the coder/observability stack scrapes every 15s) so every scrape
|
|
// observes a fresh measurement.
|
|
const latencyMeasureInterval = time.Second * 10
|
|
|
|
// latencyMeasureTimeout bounds a single latency measurement. It is shorter
|
|
// than latencyMeasureInterval so a hung probe cannot bleed into the next
|
|
// interval; a timed-out probe is recorded as a measurement error.
|
|
const latencyMeasureTimeout = time.Second * 5
|
|
|
|
type Subscriber interface {
|
|
Subscribe(event string, listener Listener) (cancel func(), err error)
|
|
SubscribeWithErr(event string, listener ListenerWithErr) (cancel func(), err error)
|
|
}
|
|
|
|
type Publisher interface {
|
|
Publish(event string, message []byte) error
|
|
}
|
|
|
|
// Pubsub is a generic interface for broadcasting and receiving messages.
|
|
// Implementors should assume high-availability with the backing implementation.
|
|
type Pubsub interface {
|
|
Subscriber
|
|
Publisher
|
|
Close() error
|
|
}
|
|
|
|
// ReportingPubsub is a Pubsub that can report on its own health.
|
|
type ReportingPubsub interface {
|
|
Pubsub
|
|
// HealthReport returns the health of the pubsub backend.
|
|
HealthReport(ctx context.Context) healthsdk.PubsubReport
|
|
}
|
|
|
|
// msgOrErr either contains a message or an error
|
|
type msgOrErr struct {
|
|
msg []byte
|
|
err error
|
|
}
|
|
|
|
// MsgQueue implements a fixed length queue with the ability to replace elements
|
|
// after they are queued (but before they are dequeued).
|
|
//
|
|
// The purpose of this data structure is to build something that works a bit
|
|
// like a golang channel, but if the queue is full, then we can replace the
|
|
// last element with an error so that the subscriber can get notified that some
|
|
// messages were dropped, all without blocking.
|
|
type MsgQueue struct {
|
|
ctx context.Context
|
|
cond *sync.Cond
|
|
q [BufferSize]msgOrErr
|
|
front int
|
|
size int
|
|
closed bool
|
|
l Listener
|
|
le ListenerWithErr
|
|
}
|
|
|
|
func NewMsgQueue(ctx context.Context, l Listener, le ListenerWithErr) *MsgQueue {
|
|
if l == nil && le == nil {
|
|
panic("l or le must be non-nil")
|
|
}
|
|
q := &MsgQueue{
|
|
ctx: ctx,
|
|
cond: sync.NewCond(&sync.Mutex{}),
|
|
l: l,
|
|
le: le,
|
|
}
|
|
go q.run()
|
|
return q
|
|
}
|
|
|
|
func (q *MsgQueue) run() {
|
|
for {
|
|
// wait until there is something on the queue or we are closed
|
|
q.cond.L.Lock()
|
|
for q.size == 0 && !q.closed {
|
|
q.cond.Wait()
|
|
}
|
|
if q.closed {
|
|
q.cond.L.Unlock()
|
|
return
|
|
}
|
|
item := q.q[q.front]
|
|
q.front = (q.front + 1) % BufferSize
|
|
q.size--
|
|
q.cond.L.Unlock()
|
|
|
|
// process item without holding lock
|
|
if item.err == nil {
|
|
// real message
|
|
if q.l != nil {
|
|
q.l(q.ctx, item.msg)
|
|
continue
|
|
}
|
|
if q.le != nil {
|
|
q.le(q.ctx, item.msg, nil)
|
|
continue
|
|
}
|
|
// unhittable
|
|
continue
|
|
}
|
|
// if the listener wants errors, send it.
|
|
if q.le != nil {
|
|
q.le(q.ctx, nil, item.err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (q *MsgQueue) Enqueue(msg []byte) {
|
|
q.cond.L.Lock()
|
|
defer q.cond.L.Unlock()
|
|
|
|
if q.size == BufferSize {
|
|
// queue is full, so we're going to drop the msg we got called with.
|
|
// We also need to record that messages are being dropped, which we
|
|
// do at the last message in the queue. This potentially makes us
|
|
// lose 2 messages instead of one, but it's more important at this
|
|
// point to warn the subscriber that they're losing messages so they
|
|
// can do something about it.
|
|
back := (q.front + BufferSize - 1) % BufferSize
|
|
q.q[back].msg = nil
|
|
q.q[back].err = ErrDroppedMessages
|
|
return
|
|
}
|
|
// queue is not full, insert the message
|
|
next := (q.front + q.size) % BufferSize
|
|
q.q[next].msg = msg
|
|
q.q[next].err = nil
|
|
q.size++
|
|
q.cond.Broadcast()
|
|
}
|
|
|
|
func (q *MsgQueue) Close() {
|
|
q.cond.L.Lock()
|
|
defer q.cond.L.Unlock()
|
|
defer q.cond.Broadcast()
|
|
q.closed = true
|
|
}
|
|
|
|
// Dropped records an error in the queue that messages might have been Dropped
|
|
func (q *MsgQueue) Dropped() {
|
|
q.cond.L.Lock()
|
|
defer q.cond.L.Unlock()
|
|
|
|
if q.size == BufferSize {
|
|
// queue is full, but we need to record that messages are being dropped,
|
|
// which we do at the last message in the queue. This potentially drops
|
|
// another message, but it's more important for the subscriber to know.
|
|
back := (q.front + BufferSize - 1) % BufferSize
|
|
q.q[back].msg = nil
|
|
q.q[back].err = ErrDroppedMessages
|
|
return
|
|
}
|
|
// queue is not full, insert the error
|
|
next := (q.front + q.size) % BufferSize
|
|
q.q[next].msg = nil
|
|
q.q[next].err = ErrDroppedMessages
|
|
q.size++
|
|
q.cond.Broadcast()
|
|
}
|
|
|
|
// pqListener is an interface that represents a *pq.Listener for testing
|
|
type pqListener interface {
|
|
io.Closer
|
|
Listen(string) error
|
|
Unlisten(string) error
|
|
NotifyChan() <-chan *pq.Notification
|
|
}
|
|
|
|
type pqListenerShim struct {
|
|
*pq.Listener
|
|
}
|
|
|
|
func (l pqListenerShim) NotifyChan() <-chan *pq.Notification {
|
|
return l.Notify
|
|
}
|
|
|
|
type queueSet struct {
|
|
m map[*MsgQueue]struct{}
|
|
// unlistenInProgress will be non-nil if another goroutine is unlistening for the event this
|
|
// queueSet corresponds to. If non-nil, that goroutine will close the channel when it is done.
|
|
unlistenInProgress chan struct{}
|
|
}
|
|
|
|
func newQueueSet() *queueSet {
|
|
return &queueSet{
|
|
m: make(map[*MsgQueue]struct{}),
|
|
}
|
|
}
|
|
|
|
// PGPubsub is a pubsub implementation using PostgreSQL.
|
|
type PGPubsub struct {
|
|
logger slog.Logger
|
|
listenDone chan struct{}
|
|
pgListener pqListener
|
|
db *sql.DB
|
|
|
|
qMu sync.Mutex
|
|
queues map[string]*queueSet
|
|
|
|
// making the close state its own mutex domain simplifies closing logic so
|
|
// that we don't have to hold the qMu --- which could block processing
|
|
// notifications while the pqListener is closing.
|
|
closeMu sync.Mutex
|
|
closedListener bool
|
|
closeListenerErr error
|
|
|
|
// connected tracks whether pgListener currently has a connection to
|
|
// PostgreSQL, as reported by its event callback.
|
|
connected atomic.Bool
|
|
|
|
metrics *BackendMetrics
|
|
}
|
|
|
|
// BufferSize is the maximum number of unhandled messages we will buffer
|
|
// for a subscriber before dropping messages.
|
|
const BufferSize = 2048
|
|
|
|
// Subscribe calls the listener when an event matching the name is received.
|
|
func (p *PGPubsub) Subscribe(event string, listener Listener) (cancel func(), err error) {
|
|
return p.subscribeQueue(event, NewMsgQueue(context.Background(), listener, nil))
|
|
}
|
|
|
|
func (p *PGPubsub) SubscribeWithErr(event string, listener ListenerWithErr) (cancel func(), err error) {
|
|
return p.subscribeQueue(event, NewMsgQueue(context.Background(), nil, listener))
|
|
}
|
|
|
|
func (p *PGPubsub) subscribeQueue(event string, newQ *MsgQueue) (cancel func(), err error) {
|
|
defer func() {
|
|
if err != nil {
|
|
// if we hit an error, we need to close the queue so we don't
|
|
// leak its goroutine.
|
|
newQ.Close()
|
|
p.metrics.RecordSubscribeFailure(event)
|
|
} else {
|
|
p.metrics.RecordSubscribeSuccess(event)
|
|
}
|
|
}()
|
|
|
|
var (
|
|
unlistenInProgress <-chan struct{}
|
|
// MUST hold the p.qMu lock to manipulate this!
|
|
qs *queueSet
|
|
)
|
|
func() {
|
|
p.qMu.Lock()
|
|
defer p.qMu.Unlock()
|
|
|
|
var ok bool
|
|
if qs, ok = p.queues[event]; !ok {
|
|
qs = newQueueSet()
|
|
p.queues[event] = qs
|
|
p.metrics.AddEvent(event)
|
|
}
|
|
qs.m[newQ] = struct{}{}
|
|
p.metrics.AddSubscriber(event)
|
|
unlistenInProgress = qs.unlistenInProgress
|
|
}()
|
|
// NOTE there cannot be any `return` statements between here and the next +-+, otherwise the
|
|
// assumptions the defer makes could be violated
|
|
if unlistenInProgress != nil {
|
|
// We have to wait here because we don't want our `Listen` call to happen before the other
|
|
// goroutine calls `Unlisten`. That would result in this subscription not getting any
|
|
// events. c.f. https://github.com/coder/coder/issues/15312
|
|
p.logger.Debug(context.Background(), "waiting for Unlisten in progress", slog.F("event", event))
|
|
<-unlistenInProgress
|
|
p.logger.Debug(context.Background(), "unlistening complete", slog.F("event", event))
|
|
}
|
|
// +-+ (see above)
|
|
defer func() {
|
|
if err != nil {
|
|
p.qMu.Lock()
|
|
defer p.qMu.Unlock()
|
|
delete(qs.m, newQ)
|
|
p.metrics.RemoveSubscriber(event)
|
|
if len(qs.m) == 0 {
|
|
// we know that newQ was in the queueSet since we last unlocked, so there cannot
|
|
// have been any _new_ goroutines trying to Unlisten(). Therefore, if the queueSet
|
|
// is now empty, it's safe to delete.
|
|
delete(p.queues, event)
|
|
p.metrics.RemoveEvent(event)
|
|
}
|
|
}
|
|
}()
|
|
|
|
// The pgListener waits for the response to `LISTEN` on a mainloop that also dispatches
|
|
// notifies. We need to avoid holding the mutex while this happens, since holding the mutex
|
|
// blocks reading notifications and can deadlock the pgListener.
|
|
// c.f. https://github.com/coder/coder/issues/11950
|
|
err = p.pgListener.Listen(event)
|
|
if err == nil {
|
|
p.logger.Debug(context.Background(), "started listening to event channel", slog.F("event", event))
|
|
}
|
|
if errors.Is(err, pq.ErrChannelAlreadyOpen) {
|
|
// It's ok if it's already open!
|
|
err = nil
|
|
}
|
|
if err != nil {
|
|
return nil, xerrors.Errorf("listen: %w", err)
|
|
}
|
|
|
|
return func() {
|
|
var unlistening chan struct{}
|
|
func() {
|
|
p.qMu.Lock()
|
|
defer p.qMu.Unlock()
|
|
newQ.Close()
|
|
qSet, ok := p.queues[event]
|
|
if !ok {
|
|
p.logger.Critical(context.Background(), "event was removed before cancel", slog.F("event", event))
|
|
return
|
|
}
|
|
delete(qSet.m, newQ)
|
|
p.metrics.RemoveSubscriber(event)
|
|
if len(qSet.m) == 0 {
|
|
unlistening = make(chan struct{})
|
|
qSet.unlistenInProgress = unlistening
|
|
}
|
|
}()
|
|
|
|
// as above, we must not hold the lock while calling into pgListener
|
|
if unlistening != nil {
|
|
uErr := p.pgListener.Unlisten(event)
|
|
close(unlistening)
|
|
// we can now delete the queueSet if it is empty.
|
|
func() {
|
|
p.qMu.Lock()
|
|
defer p.qMu.Unlock()
|
|
qSet, ok := p.queues[event]
|
|
if ok && len(qSet.m) == 0 {
|
|
p.logger.Debug(context.Background(), "removing queueSet", slog.F("event", event))
|
|
delete(p.queues, event)
|
|
p.metrics.RemoveEvent(event)
|
|
}
|
|
}()
|
|
|
|
p.closeMu.Lock()
|
|
defer p.closeMu.Unlock()
|
|
if uErr != nil && !p.closedListener {
|
|
p.logger.Warn(context.Background(), "failed to unlisten", slog.Error(uErr), slog.F("event", event))
|
|
} else {
|
|
p.logger.Debug(context.Background(), "stopped listening to event channel", slog.F("event", event))
|
|
}
|
|
}
|
|
}, nil
|
|
}
|
|
|
|
func (p *PGPubsub) Publish(event string, message []byte) error {
|
|
p.logger.Debug(context.Background(), "publish", slog.F("event", event), slog.F("message_len", len(message)))
|
|
// This is safe because we are calling pq.QuoteLiteral. pg_notify doesn't
|
|
// support the first parameter being a prepared statement.
|
|
//nolint:gosec
|
|
_, err := p.db.ExecContext(context.Background(), `select pg_notify(`+pq.QuoteLiteral(event)+`, $1)`, message)
|
|
if err != nil {
|
|
p.metrics.RecordPublishFailure(event)
|
|
return xerrors.Errorf("exec pg_notify: %w", err)
|
|
}
|
|
p.metrics.RecordPublishSuccess(event, len(message))
|
|
return nil
|
|
}
|
|
|
|
// HealthReport implements ReportingPubsub. It warns when the listener is not
|
|
// connected to PostgreSQL, since no messages are received until it reconnects.
|
|
func (p *PGPubsub) HealthReport(context.Context) healthsdk.PubsubReport {
|
|
report := healthsdk.PubsubReport{
|
|
BaseReport: healthsdk.BaseReport{
|
|
Severity: health.SeverityOK,
|
|
Warnings: []health.Message{},
|
|
},
|
|
Backend: BackendPostgres,
|
|
}
|
|
if !p.connected.Load() {
|
|
report.Severity = health.SeverityWarning
|
|
report.Warnings = append(report.Warnings,
|
|
health.Messagef(health.CodePubsubNotConnected, "pubsub listener is not connected to PostgreSQL"))
|
|
}
|
|
return report
|
|
}
|
|
|
|
// Close closes the pubsub instance.
|
|
func (p *PGPubsub) Close() error {
|
|
p.logger.Info(context.Background(), "pubsub is closing")
|
|
// Close the listener before waiting for the latency loop. While Postgres
|
|
// is unreachable, an in-flight probe blocks in pq.Listener.Listen until
|
|
// the listener reconnects or closes, so waiting first can hang forever.
|
|
err := p.closeListener()
|
|
p.metrics.StopLatencyLoop()
|
|
<-p.listenDone
|
|
p.logger.Debug(context.Background(), "pubsub closed")
|
|
return err
|
|
}
|
|
|
|
// closeListener closes the pgListener, unless it has already been closed.
|
|
func (p *PGPubsub) closeListener() error {
|
|
p.closeMu.Lock()
|
|
defer p.closeMu.Unlock()
|
|
if p.closedListener {
|
|
return p.closeListenerErr
|
|
}
|
|
p.closedListener = true
|
|
p.closeListenerErr = p.pgListener.Close()
|
|
p.connected.Store(false)
|
|
|
|
return p.closeListenerErr
|
|
}
|
|
|
|
// listen begins receiving messages on the pq listener.
|
|
func (p *PGPubsub) listen() {
|
|
defer func() {
|
|
p.logger.Info(context.Background(), "pubsub listen stopped receiving notify")
|
|
close(p.listenDone)
|
|
}()
|
|
|
|
notify := p.pgListener.NotifyChan()
|
|
for notif := range notify {
|
|
// A nil notification can be dispatched on reconnect.
|
|
if notif == nil {
|
|
p.logger.Debug(context.Background(), "notifying subscribers of a reconnection")
|
|
p.recordReconnect()
|
|
continue
|
|
}
|
|
p.listenReceive(notif)
|
|
}
|
|
}
|
|
|
|
func (p *PGPubsub) listenReceive(notif *pq.Notification) {
|
|
p.metrics.RecordReceived(notif.Channel, []byte(notif.Extra))
|
|
|
|
p.qMu.Lock()
|
|
defer p.qMu.Unlock()
|
|
qSet, ok := p.queues[notif.Channel]
|
|
if !ok {
|
|
return
|
|
}
|
|
extra := []byte(notif.Extra)
|
|
for q := range qSet.m {
|
|
q.Enqueue(extra)
|
|
}
|
|
}
|
|
|
|
func (p *PGPubsub) recordReconnect() {
|
|
p.qMu.Lock()
|
|
defer p.qMu.Unlock()
|
|
for _, qSet := range p.queues {
|
|
for q := range qSet.m {
|
|
q.Dropped()
|
|
}
|
|
}
|
|
}
|
|
|
|
// logDialer is a pq.Dialer and pq.DialerContext that logs when it starts
|
|
// connecting and when the TCP connection is established.
|
|
type logDialer struct {
|
|
logger slog.Logger
|
|
d net.Dialer
|
|
}
|
|
|
|
var (
|
|
_ pq.Dialer = logDialer{}
|
|
_ pq.DialerContext = logDialer{}
|
|
)
|
|
|
|
func (d logDialer) Dial(network, address string) (net.Conn, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
return d.DialContext(ctx, network, address)
|
|
}
|
|
|
|
func (d logDialer) DialTimeout(network, address string, timeout time.Duration) (net.Conn, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
defer cancel()
|
|
return d.DialContext(ctx, network, address)
|
|
}
|
|
|
|
func (d logDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
|
|
deadline, hasDeadline := ctx.Deadline()
|
|
timeoutMS := 0
|
|
if hasDeadline {
|
|
timeoutMS = int(time.Until(deadline) / time.Millisecond)
|
|
}
|
|
|
|
logger := d.logger.With(slog.F("network", network), slog.F("address", address), slog.F("timeout_ms", timeoutMS))
|
|
|
|
logger.Debug(ctx, "pubsub dialing postgres")
|
|
start := time.Now()
|
|
conn, err := d.d.DialContext(ctx, network, address)
|
|
if err != nil {
|
|
logger.Error(ctx, "pubsub failed to dial postgres")
|
|
return nil, err
|
|
}
|
|
elapsed := time.Since(start)
|
|
logger.Debug(ctx, "pubsub postgres TCP connection established", slog.F("elapsed_ms", elapsed.Milliseconds()))
|
|
return conn, nil
|
|
}
|
|
|
|
func (p *PGPubsub) startListener(ctx context.Context, connectURL string) error {
|
|
p.metrics.MarkDisconnected()
|
|
// Creates a new listener using pq.
|
|
var (
|
|
dialer = logDialer{
|
|
logger: p.logger,
|
|
// pq.defaultDialer uses a zero net.Dialer as well.
|
|
d: net.Dialer{},
|
|
}
|
|
connector driver.Connector
|
|
err error
|
|
)
|
|
|
|
// Create a custom connector if the database driver supports it.
|
|
connectorCreator, ok := p.db.Driver().(database.ConnectorCreator)
|
|
if ok {
|
|
connector, err = connectorCreator.Connector(connectURL)
|
|
if err != nil {
|
|
return xerrors.Errorf("create custom connector: %w", err)
|
|
}
|
|
} else {
|
|
// use the default pq connector otherwise
|
|
connector, err = pq.NewConnector(connectURL)
|
|
if err != nil {
|
|
return xerrors.Errorf("create pq connector: %w", err)
|
|
}
|
|
}
|
|
|
|
// Set the dialer if the connector supports it.
|
|
dc, ok := connector.(database.DialerConnector)
|
|
if !ok {
|
|
p.logger.Critical(ctx, "connector does not support setting log dialer, database connection debug logs will be missing")
|
|
} else {
|
|
dc.Dialer(dialer)
|
|
}
|
|
|
|
var (
|
|
errCh = make(chan error, 1)
|
|
sentErrCh = false
|
|
)
|
|
p.pgListener = pqListenerShim{
|
|
Listener: pq.NewConnectorListener(connector, connectURL, time.Second, time.Minute, func(t pq.ListenerEventType, err error) {
|
|
switch t {
|
|
case pq.ListenerEventConnected:
|
|
p.logger.Debug(ctx, "pubsub connected to postgres")
|
|
p.connected.Store(true)
|
|
p.metrics.MarkConnected()
|
|
case pq.ListenerEventDisconnected:
|
|
p.logger.Error(ctx, "pubsub disconnected from postgres", slog.Error(err))
|
|
p.connected.Store(false)
|
|
p.metrics.RecordDisconnect()
|
|
p.metrics.MarkDisconnected()
|
|
case pq.ListenerEventReconnected:
|
|
p.logger.Info(ctx, "pubsub reconnected to postgres")
|
|
p.connected.Store(true)
|
|
p.metrics.MarkConnected()
|
|
case pq.ListenerEventConnectionAttemptFailed:
|
|
p.logger.Error(ctx, "pubsub failed to connect to postgres", slog.Error(err))
|
|
}
|
|
// This callback gets events whenever the connection state changes.
|
|
// Only send the first error.
|
|
if sentErrCh {
|
|
return
|
|
}
|
|
errCh <- err // won't block because we are buffered.
|
|
sentErrCh = true
|
|
}),
|
|
}
|
|
// We don't respect context cancellation here. There's a bug in the pq library
|
|
// where if you close the listener before or while the connection is being
|
|
// established, the connection will be established anyway, and will not be
|
|
// closed.
|
|
// https://github.com/lib/pq/issues/1192
|
|
if err := <-errCh; err != nil {
|
|
_ = p.pgListener.Close()
|
|
return xerrors.Errorf("create pq listener: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// We'll track messages as size "normal" and "colossal", where the
|
|
// latter are messages larger than 7600 bytes, or 95% of the postgres
|
|
// notify limit. If we see a lot of colossal packets that's an indication that
|
|
// we might be trying to send too much data over the pubsub and are in danger of
|
|
// failing to publish.
|
|
//
|
|
// These are exported so other pubsub implementations (e.g. the NATS
|
|
// pubsub) classify message size identically, keeping the messages_total
|
|
// "size" label consistent across backends.
|
|
const (
|
|
ColossalThreshold = 7600
|
|
MessageSizeNormal = "normal"
|
|
MessageSizeColossal = "colossal"
|
|
)
|
|
|
|
// New creates a new Pubsub implementation using a PostgreSQL connection. The
|
|
// metrics argument is the shared pubsub metrics; pass nil to disable metrics
|
|
// (an unregistered set is created internally).
|
|
func New(startCtx context.Context, logger slog.Logger, db *sql.DB, connectURL string, metrics *Metrics) (*PGPubsub, error) {
|
|
p := newWithoutListener(logger, db, metrics)
|
|
if err := p.startListener(startCtx, connectURL); err != nil {
|
|
return nil, err
|
|
}
|
|
go p.listen()
|
|
// Probe latency out-of-band. It records into no-op instruments when
|
|
// Prometheus is disabled, but always runs so the loop lifecycle does not
|
|
// depend on whether metrics are collected.
|
|
p.metrics.StartLatencyLoop(p)
|
|
logger.Debug(startCtx, "pubsub has started")
|
|
return p, nil
|
|
}
|
|
|
|
// newWithoutListener creates a new PGPubsub without creating the pqListener.
|
|
func newWithoutListener(logger slog.Logger, db *sql.DB, metrics *Metrics) *PGPubsub {
|
|
if metrics == nil {
|
|
metrics = NewMetrics(nil)
|
|
}
|
|
return &PGPubsub{
|
|
logger: logger,
|
|
listenDone: make(chan struct{}),
|
|
db: db,
|
|
queues: make(map[string]*queueSet),
|
|
metrics: metrics.ForBackend(logger, BackendPostgres),
|
|
}
|
|
}
|