package worker import ( "context" "encoding/json" "fmt" "log/slog" "os" "strings" "sync" "time" "wucher/internal/config" "wucher/internal/domain/auth" "wucher/internal/queue" ) type OutboxMetricsSnapshot struct { Claimed int64 `json:"claimed"` Published int64 `json:"published"` Retried int64 `json:"retried"` Dead int64 `json:"dead"` Failures int64 `json:"failures"` ApproximatePending int64 `json:"approximate_pending"` LastSuccessAt time.Time `json:"last_success_at"` LastFailureAt time.Time `json:"last_failure_at"` LastError string `json:"last_error,omitempty"` } type outboxMetrics struct { mu sync.Mutex snapshot OutboxMetricsSnapshot } func (m *outboxMetrics) Snapshot() OutboxMetricsSnapshot { if m == nil { return OutboxMetricsSnapshot{} } m.mu.Lock() defer m.mu.Unlock() return m.snapshot } func (m *outboxMetrics) update(fn func(snapshot *OutboxMetricsSnapshot)) { if m == nil || fn == nil { return } m.mu.Lock() defer m.mu.Unlock() fn(&m.snapshot) } type OutboxDispatcher struct { repo auth.EmailOutboxRepository producer queue.QueueProducer logger *slog.Logger retryPolicy queue.RetryPolicy pollInterval time.Duration dispatchTimeout time.Duration lockTTL time.Duration maxAttempts int batchSize int workerID string metrics *outboxMetrics } func NewOutboxDispatcher( cfg config.QueueConfig, repo auth.EmailOutboxRepository, producer queue.QueueProducer, logger *slog.Logger, ) *OutboxDispatcher { if !cfg.OutboxEnabled || repo == nil || producer == nil { return nil } if logger == nil { logger = slog.Default() } workerID := fmt.Sprintf("%s-%d", hostname(), os.Getpid()) return &OutboxDispatcher{ repo: repo, producer: producer, logger: logger.With("component", "outbox-dispatcher", "backend", producer.Backend()), retryPolicy: queue.NewExponentialBackoffPolicy(cfg.WorkerBackoffMin, cfg.WorkerBackoffMax, cfg.WorkerPermanentFailureDelay), pollInterval: cfg.OutboxPollInterval, dispatchTimeout: cfg.OutboxDispatchTimeout, lockTTL: cfg.OutboxLockTTL, maxAttempts: cfg.OutboxMaxAttempts, batchSize: cfg.OutboxBatchSize, workerID: workerID, metrics: &outboxMetrics{}, } } func (d *OutboxDispatcher) Metrics() OutboxMetricsSnapshot { if d == nil { return OutboxMetricsSnapshot{} } return d.metrics.Snapshot() } func (d *OutboxDispatcher) Run(ctx context.Context) error { if d == nil { return nil } if d.pollInterval <= 0 { d.pollInterval = time.Second } for { processed, err := d.dispatchOnce(ctx) if err != nil { if ctx.Err() != nil { return nil } d.metrics.update(func(snapshot *OutboxMetricsSnapshot) { snapshot.Failures++ snapshot.LastFailureAt = time.Now().UTC() snapshot.LastError = err.Error() }) d.logger.Error("outbox dispatch cycle failed", slog.Any("error", err)) } if ctx.Err() != nil { return nil } if processed > 0 { continue } select { case <-ctx.Done(): return nil case <-time.After(d.pollInterval): } } } func (d *OutboxDispatcher) dispatchOnce(ctx context.Context) (int, error) { now := time.Now().UTC() messages, err := d.repo.ClaimPendingEmailOutboxMessages(ctx, d.batchSize, now, d.lockTTL, d.workerID) if err != nil { return 0, err } if len(messages) == 0 { d.refreshPending(ctx, now) return 0, nil } d.metrics.update(func(snapshot *OutboxMetricsSnapshot) { snapshot.Claimed += int64(len(messages)) }) for i := range messages { d.dispatchMessage(ctx, &messages[i]) } d.refreshPending(ctx, time.Now().UTC()) return len(messages), nil } func (d *OutboxDispatcher) dispatchMessage(ctx context.Context, message *auth.EmailOutboxMessage) { if message == nil { return } outbound, err := message.OutboundMessage() if err != nil { d.failPermanently(ctx, message, err) return } if shouldSkipOutboxEmailPublish(outbound) { publishedAt := time.Now().UTC() if markErr := d.repo.MarkEmailOutboxMessagePublished(ctx, message.ID, publishedAt); markErr != nil { d.metrics.update(func(snapshot *OutboxMetricsSnapshot) { snapshot.Failures++ snapshot.LastFailureAt = time.Now().UTC() snapshot.LastError = markErr.Error() }) d.logger.Error("outbox skip marked failed", slog.String("message_id", message.MessageID), slog.Any("error", markErr), ) return } d.metrics.update(func(snapshot *OutboxMetricsSnapshot) { snapshot.Published++ snapshot.LastSuccessAt = publishedAt snapshot.LastError = "" }) d.logger.Info("outbox message skipped", slog.String("message_id", message.MessageID), slog.String("job_type", message.JobType), slog.String("correlation_id", message.CorrelationID), ) return } dispatchCtx := ctx cancel := func() {} if d.dispatchTimeout > 0 { dispatchCtx, cancel = context.WithTimeout(ctx, d.dispatchTimeout) } err = d.producer.Publish(dispatchCtx, *outbound) cancel() if err == nil { publishedAt := time.Now().UTC() if markErr := d.repo.MarkEmailOutboxMessagePublished(ctx, message.ID, publishedAt); markErr != nil { d.metrics.update(func(snapshot *OutboxMetricsSnapshot) { snapshot.Failures++ snapshot.LastFailureAt = time.Now().UTC() snapshot.LastError = markErr.Error() }) d.logger.Error("outbox publish marked failed", slog.String("message_id", message.MessageID), slog.Any("error", markErr), ) return } d.metrics.update(func(snapshot *OutboxMetricsSnapshot) { snapshot.Published++ snapshot.LastSuccessAt = publishedAt snapshot.LastError = "" }) d.logger.Info("outbox message published", slog.String("message_id", message.MessageID), slog.String("job_type", message.JobType), slog.String("correlation_id", message.CorrelationID), slog.Int("attempt", message.Attempts), ) return } if message.Attempts >= d.maxAttempts { d.failPermanently(ctx, message, err) return } decision := d.retryPolicy.Decide(&queue.Delivery{ID: message.MessageID, ReceiveCount: message.Attempts}, nil, err) nextAttemptAt := time.Now().UTC().Add(decision.Delay) if retryErr := d.repo.MarkEmailOutboxMessageRetry(ctx, message.ID, nextAttemptAt, err.Error()); retryErr != nil { d.metrics.update(func(snapshot *OutboxMetricsSnapshot) { snapshot.Failures++ snapshot.LastFailureAt = time.Now().UTC() snapshot.LastError = retryErr.Error() }) d.logger.Error("outbox retry scheduling failed", slog.String("message_id", message.MessageID), slog.Any("error", retryErr), ) return } d.metrics.update(func(snapshot *OutboxMetricsSnapshot) { snapshot.Retried++ snapshot.LastFailureAt = time.Now().UTC() snapshot.LastError = err.Error() }) d.logger.Warn("outbox publish failed; scheduled retry", slog.String("message_id", message.MessageID), slog.String("job_type", message.JobType), slog.Duration("retry_in", decision.Delay), slog.Int("attempt", message.Attempts), slog.Any("error", err), ) } func shouldSkipOutboxEmailPublish(outbound *queue.OutboundMessage) bool { if outbound == nil || len(outbound.Body) == 0 { return false } var envelope struct { Payload struct { MessageType string `json:"message_type"` Metadata map[string]string `json:"metadata"` Tags map[string]string `json:"tags"` } `json:"payload"` } if err := json.Unmarshal(outbound.Body, &envelope); err == nil { if strings.EqualFold(strings.TrimSpace(envelope.Payload.Metadata["skip_send"]), "true") { return true } if strings.EqualFold(strings.TrimSpace(envelope.Payload.MessageType), "email_verify") && strings.EqualFold(strings.TrimSpace(envelope.Payload.Tags["source"]), "register") { return true } return false } var legacyJob struct { MessageType string `json:"message_type"` Metadata map[string]string `json:"metadata"` Tags map[string]string `json:"tags"` } if err := json.Unmarshal(outbound.Body, &legacyJob); err != nil { return false } if strings.EqualFold(strings.TrimSpace(legacyJob.Metadata["skip_send"]), "true") { return true } return strings.EqualFold(strings.TrimSpace(legacyJob.MessageType), "email_verify") && strings.EqualFold(strings.TrimSpace(legacyJob.Tags["source"]), "register") } func (d *OutboxDispatcher) failPermanently(ctx context.Context, message *auth.EmailOutboxMessage, err error) { if message == nil || err == nil { return } failedAt := time.Now().UTC() if markErr := d.repo.MarkEmailOutboxMessageDead(ctx, message.ID, failedAt, err.Error()); markErr != nil { d.metrics.update(func(snapshot *OutboxMetricsSnapshot) { snapshot.Failures++ snapshot.LastFailureAt = failedAt snapshot.LastError = markErr.Error() }) d.logger.Error("outbox dead-letter marking failed", slog.String("message_id", message.MessageID), slog.Any("error", markErr), ) return } d.metrics.update(func(snapshot *OutboxMetricsSnapshot) { snapshot.Dead++ snapshot.LastFailureAt = failedAt snapshot.LastError = err.Error() }) d.logger.Error("outbox message marked dead", slog.String("message_id", message.MessageID), slog.String("job_type", message.JobType), slog.Int("attempt", message.Attempts), slog.Any("error", err), ) } func (d *OutboxDispatcher) refreshPending(ctx context.Context, now time.Time) { count, err := d.repo.CountPendingEmailOutboxMessages(ctx, now) if err != nil { return } d.metrics.update(func(snapshot *OutboxMetricsSnapshot) { snapshot.ApproximatePending = count }) } func hostname() string { name, err := os.Hostname() if err != nil || strings.TrimSpace(name) == "" { return "worker" } return strings.TrimSpace(name) }