2026-03-17 11:31:59 +08:00
|
|
|
|
package delayqueue
|
|
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
|
"context"
|
|
|
|
|
|
"encoding/json"
|
|
|
|
|
|
"errors"
|
|
|
|
|
|
"fmt"
|
|
|
|
|
|
"log/slog"
|
|
|
|
|
|
"time"
|
|
|
|
|
|
|
|
|
|
|
|
"github.com/google/uuid"
|
|
|
|
|
|
"github.com/redis/go-redis/v9"
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
// Job 表示一个延迟任务(泛型载荷)
|
|
|
|
|
|
type Job[T any] struct {
|
|
|
|
|
|
ID string
|
|
|
|
|
|
Queue string
|
|
|
|
|
|
Payload T
|
|
|
|
|
|
Attempts int
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Handler 处理任务(泛型载荷)
|
|
|
|
|
|
type Handler[T any] func(ctx context.Context, job *Job[T]) error
|
|
|
|
|
|
|
|
|
|
|
|
// RedisDelayQueue 基于 Redis 的延迟队列(泛型载荷)
|
|
|
|
|
|
type RedisDelayQueue[T any] struct {
|
|
|
|
|
|
rdb *redis.Client
|
|
|
|
|
|
logger *slog.Logger
|
|
|
|
|
|
prefix string
|
|
|
|
|
|
pollInterval time.Duration
|
|
|
|
|
|
batchSize int
|
|
|
|
|
|
requeueDelay time.Duration
|
|
|
|
|
|
maxAttempts int
|
|
|
|
|
|
jobTTL time.Duration
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Option 可选项(泛型载荷)
|
|
|
|
|
|
type Option[T any] func(*RedisDelayQueue[T])
|
|
|
|
|
|
|
|
|
|
|
|
const (
|
|
|
|
|
|
defaultPrefix = "mcai"
|
|
|
|
|
|
defaultPollInterval = 500 * time.Millisecond
|
|
|
|
|
|
defaultBatchSize = 64
|
|
|
|
|
|
defaultRequeueDelay = 5 * time.Second
|
|
|
|
|
|
defaultMaxAttempts = 5
|
|
|
|
|
|
defaultJobTTL = 7 * 24 * time.Hour
|
|
|
|
|
|
)
|
|
|
|
|
|
|
2026-07-14 18:23:24 +08:00
|
|
|
|
// ErrRetryAfterMaxAttempts 让已耗尽普通尝试次数的任务继续保留并重试。
|
|
|
|
|
|
var ErrRetryAfterMaxAttempts = errors.New("retry delay queue job after max attempts")
|
|
|
|
|
|
|
2026-08-05 14:03:57 +08:00
|
|
|
|
var ErrJobRescheduled = errors.New("delay queue job rescheduled")
|
|
|
|
|
|
|
2026-03-17 11:31:59 +08:00
|
|
|
|
// NewRedisDelayQueue 创建延迟队列(泛型)
|
|
|
|
|
|
func NewRedisDelayQueue[T any](rdb *redis.Client, logger *slog.Logger, opts ...Option[T]) *RedisDelayQueue[T] {
|
|
|
|
|
|
q := &RedisDelayQueue[T]{
|
|
|
|
|
|
rdb: rdb,
|
|
|
|
|
|
logger: logger,
|
|
|
|
|
|
prefix: defaultPrefix,
|
|
|
|
|
|
pollInterval: defaultPollInterval,
|
|
|
|
|
|
batchSize: defaultBatchSize,
|
|
|
|
|
|
requeueDelay: defaultRequeueDelay,
|
|
|
|
|
|
maxAttempts: defaultMaxAttempts,
|
|
|
|
|
|
jobTTL: defaultJobTTL,
|
|
|
|
|
|
}
|
|
|
|
|
|
for _, opt := range opts {
|
|
|
|
|
|
opt(q)
|
|
|
|
|
|
}
|
|
|
|
|
|
return q
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func WithPrefix[T any](prefix string) Option[T] {
|
|
|
|
|
|
return func(q *RedisDelayQueue[T]) { q.prefix = prefix }
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func WithPollInterval[T any](d time.Duration) Option[T] {
|
|
|
|
|
|
return func(q *RedisDelayQueue[T]) { q.pollInterval = d }
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func WithBatchSize[T any](n int) Option[T] {
|
|
|
|
|
|
return func(q *RedisDelayQueue[T]) { q.batchSize = n }
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func WithRequeueDelay[T any](d time.Duration) Option[T] {
|
|
|
|
|
|
return func(q *RedisDelayQueue[T]) { q.requeueDelay = d }
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func WithMaxAttempts[T any](n int) Option[T] {
|
|
|
|
|
|
return func(q *RedisDelayQueue[T]) { q.maxAttempts = n }
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func WithJobTTL[T any](d time.Duration) Option[T] {
|
|
|
|
|
|
return func(q *RedisDelayQueue[T]) { q.jobTTL = d }
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-07-14 18:23:24 +08:00
|
|
|
|
// IsFinalAttempt 判断当前处理是否为队列允许的最后一次尝试。
|
|
|
|
|
|
func (q *RedisDelayQueue[T]) IsFinalAttempt(job *Job[T]) bool {
|
|
|
|
|
|
return job != nil && job.Attempts+1 >= q.maxAttempts
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-04-02 21:57:48 +08:00
|
|
|
|
// Enqueue 入队:始终写入/覆盖 payload 并更新到期时间。
|
|
|
|
|
|
// 这样即使 pollOnce 在 handleJob 完成后删除了旧 job key,
|
|
|
|
|
|
// 新的 Enqueue 调用也能重建 job 数据,避免 ZSet 中存在孤立条目。
|
2026-03-17 11:31:59 +08:00
|
|
|
|
func (q *RedisDelayQueue[T]) Enqueue(ctx context.Context, queue string, payload T, runAt time.Time, id string) (string, error) {
|
|
|
|
|
|
if id == "" {
|
|
|
|
|
|
id = uuid.NewString()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
jobKey := q.jobKey(queue, id)
|
2026-04-02 21:57:48 +08:00
|
|
|
|
b, err := json.Marshal(&jobData[T]{Payload: payload, Attempts: 0})
|
2026-03-17 11:31:59 +08:00
|
|
|
|
if err != nil {
|
|
|
|
|
|
return "", err
|
|
|
|
|
|
}
|
2026-04-02 21:57:48 +08:00
|
|
|
|
if err := q.rdb.Set(ctx, jobKey, b, q.jobTTL).Err(); err != nil {
|
|
|
|
|
|
return "", err
|
2026-03-17 11:31:59 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
zkey := q.zsetKey(queue)
|
|
|
|
|
|
score := float64(runAt.UnixMilli())
|
|
|
|
|
|
if err := q.rdb.ZAdd(ctx, zkey, redis.Z{Score: score, Member: id}).Err(); err != nil {
|
2026-04-02 21:57:48 +08:00
|
|
|
|
_ = q.rdb.Del(ctx, jobKey).Err()
|
2026-03-17 11:31:59 +08:00
|
|
|
|
return "", err
|
|
|
|
|
|
}
|
|
|
|
|
|
return id, nil
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-07-14 15:59:48 +08:00
|
|
|
|
// EnqueueIfMissing 仅在延迟队列中不存在指定 ID 时入队。
|
|
|
|
|
|
func (q *RedisDelayQueue[T]) EnqueueIfMissing(ctx context.Context, queue string, payload T, runAt time.Time, id string) (string, bool, error) {
|
|
|
|
|
|
if id == "" {
|
|
|
|
|
|
id = uuid.NewString()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
b, err := json.Marshal(&jobData[T]{Payload: payload, Attempts: 0})
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
return "", false, err
|
|
|
|
|
|
}
|
|
|
|
|
|
added, err := enqueueIfMissingScript.Run(ctx, q.rdb, []string{
|
|
|
|
|
|
q.jobKey(queue, id),
|
|
|
|
|
|
q.zsetKey(queue),
|
|
|
|
|
|
}, id, b, q.jobTTL.Milliseconds(), runAt.UnixMilli()).Int()
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
return "", false, err
|
|
|
|
|
|
}
|
|
|
|
|
|
return id, added == 1, nil
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-03-17 11:31:59 +08:00
|
|
|
|
// GetJobInfo 查询任务信息
|
|
|
|
|
|
func (q *RedisDelayQueue[T]) GetJobInfo(ctx context.Context, queue, id string) (*Job[T], time.Time, bool, error) {
|
2026-07-13 22:41:19 +08:00
|
|
|
|
runAt, ok, err := q.GetRunAt(ctx, queue, id)
|
|
|
|
|
|
if err != nil || !ok {
|
|
|
|
|
|
return nil, time.Time{}, ok, err
|
2026-03-17 11:31:59 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
job, err := q.loadJob(ctx, queue, id)
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
return nil, time.Time{}, false, err
|
|
|
|
|
|
}
|
|
|
|
|
|
if job == nil {
|
|
|
|
|
|
return nil, time.Time{}, false, nil
|
|
|
|
|
|
}
|
|
|
|
|
|
return job, runAt, true, nil
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-07-13 22:41:19 +08:00
|
|
|
|
func (q *RedisDelayQueue[T]) GetRunAt(ctx context.Context, queue, id string) (time.Time, bool, error) {
|
|
|
|
|
|
score, err := q.rdb.ZScore(ctx, q.zsetKey(queue), id).Result()
|
|
|
|
|
|
if errors.Is(err, redis.Nil) {
|
|
|
|
|
|
return time.Time{}, false, nil
|
|
|
|
|
|
}
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
return time.Time{}, false, err
|
|
|
|
|
|
}
|
|
|
|
|
|
return time.UnixMilli(int64(score)), true, nil
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-03-17 11:31:59 +08:00
|
|
|
|
// Remove 移除任务
|
|
|
|
|
|
func (q *RedisDelayQueue[T]) Remove(ctx context.Context, queue, id string) error {
|
|
|
|
|
|
if err := q.rdb.ZRem(ctx, q.zsetKey(queue), id).Err(); err != nil {
|
|
|
|
|
|
return err
|
|
|
|
|
|
}
|
|
|
|
|
|
return q.rdb.Del(ctx, q.jobKey(queue, id)).Err()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// RemoveByPrefix 批量移除匹配前缀的任务
|
|
|
|
|
|
func (q *RedisDelayQueue[T]) RemoveByPrefix(ctx context.Context, queue, prefix string) (int, error) {
|
|
|
|
|
|
zkey := q.zsetKey(queue)
|
|
|
|
|
|
cursor := uint64(0)
|
|
|
|
|
|
removed := 0
|
|
|
|
|
|
for {
|
|
|
|
|
|
res, newCursor, err := q.rdb.ZScan(ctx, zkey, cursor, prefix+"*", 200).Result()
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
return removed, err
|
|
|
|
|
|
}
|
|
|
|
|
|
for i := 0; i+1 < len(res); i += 2 {
|
|
|
|
|
|
id := res[i]
|
|
|
|
|
|
if err := q.Remove(ctx, queue, id); err != nil {
|
|
|
|
|
|
return removed, err
|
|
|
|
|
|
}
|
|
|
|
|
|
removed++
|
|
|
|
|
|
}
|
|
|
|
|
|
cursor = newCursor
|
|
|
|
|
|
if cursor == 0 {
|
|
|
|
|
|
break
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
return removed, nil
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// StartConsumer 启动消费循环(阻塞)
|
|
|
|
|
|
func (q *RedisDelayQueue[T]) StartConsumer(ctx context.Context, queue string, handler Handler[T]) error {
|
|
|
|
|
|
ticker := time.NewTicker(q.pollInterval)
|
|
|
|
|
|
defer ticker.Stop()
|
|
|
|
|
|
for {
|
|
|
|
|
|
select {
|
|
|
|
|
|
case <-ctx.Done():
|
|
|
|
|
|
return ctx.Err()
|
|
|
|
|
|
case <-ticker.C:
|
|
|
|
|
|
if err := q.pollOnce(ctx, queue, handler); err != nil {
|
|
|
|
|
|
q.logger.Warn("delayqueue poll error", "queue", queue, "err", err)
|
|
|
|
|
|
return err
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (q *RedisDelayQueue[T]) pollOnce(ctx context.Context, queue string, handler Handler[T]) error {
|
|
|
|
|
|
ids, err := q.claimDue(ctx, queue, q.batchSize)
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
return err
|
|
|
|
|
|
}
|
2026-05-24 19:14:34 +08:00
|
|
|
|
if len(ids) > 0 {
|
|
|
|
|
|
q.logger.Debug("delayqueue jobs claimed", "queue", queue, "count", len(ids), "ids", ids)
|
|
|
|
|
|
}
|
2026-03-17 11:31:59 +08:00
|
|
|
|
for _, id := range ids {
|
|
|
|
|
|
job, err := q.loadJob(ctx, queue, id)
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
q.logger.Error("load job failed", "queue", queue, "id", id, "err", err)
|
|
|
|
|
|
continue
|
|
|
|
|
|
}
|
|
|
|
|
|
if job == nil {
|
|
|
|
|
|
continue
|
|
|
|
|
|
}
|
|
|
|
|
|
if err := handler(ctx, job); err != nil {
|
2026-08-05 14:03:57 +08:00
|
|
|
|
if errors.Is(err, ErrJobRescheduled) {
|
|
|
|
|
|
q.logger.Debug("delayqueue job rescheduled", "queue", queue, "id", id)
|
|
|
|
|
|
continue
|
|
|
|
|
|
}
|
2026-03-17 11:31:59 +08:00
|
|
|
|
q.logger.Warn("handle job failed, will requeue", "queue", queue, "id", id, "attempts", job.Attempts+1, "err", err)
|
|
|
|
|
|
job.Attempts++
|
2026-07-14 18:23:24 +08:00
|
|
|
|
retryAfterMaxAttempts := errors.Is(err, ErrRetryAfterMaxAttempts)
|
|
|
|
|
|
if job.Attempts >= q.maxAttempts && !retryAfterMaxAttempts {
|
2026-03-17 11:31:59 +08:00
|
|
|
|
_ = q.deleteJob(ctx, queue, id)
|
|
|
|
|
|
continue
|
|
|
|
|
|
}
|
2026-07-14 18:23:24 +08:00
|
|
|
|
if retryAfterMaxAttempts && job.Attempts >= q.maxAttempts {
|
|
|
|
|
|
job.Attempts = max(q.maxAttempts-1, 0)
|
|
|
|
|
|
}
|
2026-03-17 11:31:59 +08:00
|
|
|
|
if err := q.saveJob(ctx, queue, id, job.Attempts, job.Payload); err != nil {
|
|
|
|
|
|
q.logger.Error("save job failed when requeue", "queue", queue, "id", id, "err", err)
|
|
|
|
|
|
continue
|
|
|
|
|
|
}
|
2026-04-02 21:57:48 +08:00
|
|
|
|
// 直接操作 ZSet 重新入队,不调用 Enqueue 以避免覆盖 saveJob 保存的 attempts
|
|
|
|
|
|
requeueAt := time.Now().Add(q.requeueDelay)
|
|
|
|
|
|
if err := q.rdb.ZAdd(ctx, q.zsetKey(queue), redis.Z{
|
|
|
|
|
|
Score: float64(requeueAt.UnixMilli()),
|
|
|
|
|
|
Member: id,
|
|
|
|
|
|
}).Err(); err != nil {
|
|
|
|
|
|
q.logger.Error("requeue zadd failed", "queue", queue, "id", id, "err", err)
|
|
|
|
|
|
}
|
2026-03-17 11:31:59 +08:00
|
|
|
|
continue
|
|
|
|
|
|
}
|
|
|
|
|
|
if err := q.deleteJob(ctx, queue, id); err != nil {
|
|
|
|
|
|
if ctx.Err() != nil || errors.Is(err, redis.ErrClosed) {
|
|
|
|
|
|
q.logger.Debug("skip delete during shutdown", "queue", queue, "id", id, "err", err)
|
|
|
|
|
|
} else {
|
|
|
|
|
|
q.logger.Error("delete job payload failed", "queue", queue, "id", id, "err", err)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
return nil
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (q *RedisDelayQueue[T]) claimDue(ctx context.Context, queue string, count int) ([]string, error) {
|
|
|
|
|
|
zkey := q.zsetKey(queue)
|
|
|
|
|
|
now := time.Now().UnixMilli()
|
|
|
|
|
|
res, err := claimScript.Run(ctx, q.rdb, []string{zkey}, now, count).Result()
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
return nil, err
|
|
|
|
|
|
}
|
|
|
|
|
|
items := make([]string, 0)
|
|
|
|
|
|
switch v := res.(type) {
|
|
|
|
|
|
case []any:
|
|
|
|
|
|
for _, it := range v {
|
|
|
|
|
|
items = append(items, fmt.Sprintf("%v", it))
|
|
|
|
|
|
}
|
|
|
|
|
|
case []string:
|
|
|
|
|
|
items = append(items, v...)
|
|
|
|
|
|
}
|
|
|
|
|
|
return items, nil
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (q *RedisDelayQueue[T]) loadJob(ctx context.Context, queue, id string) (*Job[T], error) {
|
|
|
|
|
|
jobKey := q.jobKey(queue, id)
|
|
|
|
|
|
b, err := q.rdb.Get(ctx, jobKey).Bytes()
|
|
|
|
|
|
if err == redis.Nil {
|
|
|
|
|
|
return nil, nil
|
|
|
|
|
|
}
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
return nil, err
|
|
|
|
|
|
}
|
|
|
|
|
|
var data jobData[T]
|
|
|
|
|
|
if err := json.Unmarshal(b, &data); err != nil {
|
|
|
|
|
|
return nil, err
|
|
|
|
|
|
}
|
|
|
|
|
|
return &Job[T]{
|
|
|
|
|
|
ID: id,
|
|
|
|
|
|
Queue: queue,
|
|
|
|
|
|
Payload: data.Payload,
|
|
|
|
|
|
Attempts: data.Attempts,
|
|
|
|
|
|
}, nil
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (q *RedisDelayQueue[T]) saveJob(ctx context.Context, queue, id string, attempts int, payload T) error {
|
|
|
|
|
|
jobKey := q.jobKey(queue, id)
|
|
|
|
|
|
b, err := json.Marshal(&jobData[T]{Payload: payload, Attempts: attempts})
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
return err
|
|
|
|
|
|
}
|
|
|
|
|
|
return q.rdb.Set(ctx, jobKey, b, q.jobTTL).Err()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (q *RedisDelayQueue[T]) deleteJob(ctx context.Context, queue, id string) error {
|
|
|
|
|
|
return q.rdb.Del(ctx, q.jobKey(queue, id)).Err()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// ExtendTTL 续期任务存储 TTL
|
|
|
|
|
|
func (q *RedisDelayQueue[T]) ExtendTTL(ctx context.Context, queue, id string, ttl time.Duration) error {
|
|
|
|
|
|
return q.rdb.Expire(ctx, q.jobKey(queue, id), ttl).Err()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (q *RedisDelayQueue[T]) zsetKey(queue string) string {
|
|
|
|
|
|
return fmt.Sprintf("%s:dq:%s:delayed", q.prefix, queue)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (q *RedisDelayQueue[T]) jobKey(queue, id string) string {
|
|
|
|
|
|
return fmt.Sprintf("%s:dq:%s:job:%s", q.prefix, queue, id)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
type jobData[T any] struct {
|
|
|
|
|
|
Payload T `json:"payload"`
|
|
|
|
|
|
Attempts int `json:"attempts"`
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
var claimScript = redis.NewScript(`
|
|
|
|
|
|
local zkey = KEYS[1]
|
|
|
|
|
|
local now = tonumber(ARGV[1])
|
|
|
|
|
|
local count = tonumber(ARGV[2])
|
|
|
|
|
|
local items = redis.call('ZRANGEBYSCORE', zkey, '-inf', now, 'LIMIT', 0, count)
|
|
|
|
|
|
if #items > 0 then
|
|
|
|
|
|
for i=1,#items do
|
|
|
|
|
|
redis.call('ZREM', zkey, items[i])
|
|
|
|
|
|
end
|
|
|
|
|
|
end
|
|
|
|
|
|
return items
|
|
|
|
|
|
`)
|
2026-07-14 15:59:48 +08:00
|
|
|
|
|
|
|
|
|
|
var enqueueIfMissingScript = redis.NewScript(`
|
|
|
|
|
|
if redis.call('ZSCORE', KEYS[2], ARGV[1]) then
|
|
|
|
|
|
return 0
|
|
|
|
|
|
end
|
|
|
|
|
|
redis.call('SET', KEYS[1], ARGV[2], 'PX', ARGV[3])
|
|
|
|
|
|
redis.call('ZADD', KEYS[2], ARGV[4], ARGV[1])
|
|
|
|
|
|
return 1
|
|
|
|
|
|
`)
|