init push
This commit is contained in:
342
internal/app/worker/outbox_dispatcher.go
Normal file
342
internal/app/worker/outbox_dispatcher.go
Normal file
@@ -0,0 +1,342 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"wucher/internal/config"
|
||||
"wucher/internal/domain/auth"
|
||||
"wucher/internal/queue"
|
||||
)
|
||||
|
||||
type OutboxMetricsSnapshot struct {
|
||||
Claimed int64 `json:"claimed"`
|
||||
Published int64 `json:"published"`
|
||||
Retried int64 `json:"retried"`
|
||||
Dead int64 `json:"dead"`
|
||||
Failures int64 `json:"failures"`
|
||||
ApproximatePending int64 `json:"approximate_pending"`
|
||||
LastSuccessAt time.Time `json:"last_success_at"`
|
||||
LastFailureAt time.Time `json:"last_failure_at"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
}
|
||||
|
||||
type outboxMetrics struct {
|
||||
mu sync.Mutex
|
||||
snapshot OutboxMetricsSnapshot
|
||||
}
|
||||
|
||||
func (m *outboxMetrics) Snapshot() OutboxMetricsSnapshot {
|
||||
if m == nil {
|
||||
return OutboxMetricsSnapshot{}
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.snapshot
|
||||
}
|
||||
|
||||
func (m *outboxMetrics) update(fn func(snapshot *OutboxMetricsSnapshot)) {
|
||||
if m == nil || fn == nil {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
fn(&m.snapshot)
|
||||
}
|
||||
|
||||
type OutboxDispatcher struct {
|
||||
repo auth.EmailOutboxRepository
|
||||
producer queue.QueueProducer
|
||||
logger *slog.Logger
|
||||
retryPolicy queue.RetryPolicy
|
||||
pollInterval time.Duration
|
||||
dispatchTimeout time.Duration
|
||||
lockTTL time.Duration
|
||||
maxAttempts int
|
||||
batchSize int
|
||||
workerID string
|
||||
metrics *outboxMetrics
|
||||
}
|
||||
|
||||
func NewOutboxDispatcher(
|
||||
cfg config.QueueConfig,
|
||||
repo auth.EmailOutboxRepository,
|
||||
producer queue.QueueProducer,
|
||||
logger *slog.Logger,
|
||||
) *OutboxDispatcher {
|
||||
if !cfg.OutboxEnabled || repo == nil || producer == nil {
|
||||
return nil
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
workerID := fmt.Sprintf("%s-%d", hostname(), os.Getpid())
|
||||
return &OutboxDispatcher{
|
||||
repo: repo,
|
||||
producer: producer,
|
||||
logger: logger.With("component", "outbox-dispatcher", "backend", producer.Backend()),
|
||||
retryPolicy: queue.NewExponentialBackoffPolicy(cfg.WorkerBackoffMin, cfg.WorkerBackoffMax, cfg.WorkerPermanentFailureDelay),
|
||||
pollInterval: cfg.OutboxPollInterval,
|
||||
dispatchTimeout: cfg.OutboxDispatchTimeout,
|
||||
lockTTL: cfg.OutboxLockTTL,
|
||||
maxAttempts: cfg.OutboxMaxAttempts,
|
||||
batchSize: cfg.OutboxBatchSize,
|
||||
workerID: workerID,
|
||||
metrics: &outboxMetrics{},
|
||||
}
|
||||
}
|
||||
|
||||
func (d *OutboxDispatcher) Metrics() OutboxMetricsSnapshot {
|
||||
if d == nil {
|
||||
return OutboxMetricsSnapshot{}
|
||||
}
|
||||
return d.metrics.Snapshot()
|
||||
}
|
||||
|
||||
func (d *OutboxDispatcher) Run(ctx context.Context) error {
|
||||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
if d.pollInterval <= 0 {
|
||||
d.pollInterval = time.Second
|
||||
}
|
||||
for {
|
||||
processed, err := d.dispatchOnce(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
|
||||
snapshot.Failures++
|
||||
snapshot.LastFailureAt = time.Now().UTC()
|
||||
snapshot.LastError = err.Error()
|
||||
})
|
||||
d.logger.Error("outbox dispatch cycle failed", slog.Any("error", err))
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if processed > 0 {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(d.pollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *OutboxDispatcher) dispatchOnce(ctx context.Context) (int, error) {
|
||||
now := time.Now().UTC()
|
||||
messages, err := d.repo.ClaimPendingEmailOutboxMessages(ctx, d.batchSize, now, d.lockTTL, d.workerID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(messages) == 0 {
|
||||
d.refreshPending(ctx, now)
|
||||
return 0, nil
|
||||
}
|
||||
d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
|
||||
snapshot.Claimed += int64(len(messages))
|
||||
})
|
||||
for i := range messages {
|
||||
d.dispatchMessage(ctx, &messages[i])
|
||||
}
|
||||
d.refreshPending(ctx, time.Now().UTC())
|
||||
return len(messages), nil
|
||||
}
|
||||
|
||||
func (d *OutboxDispatcher) dispatchMessage(ctx context.Context, message *auth.EmailOutboxMessage) {
|
||||
if message == nil {
|
||||
return
|
||||
}
|
||||
outbound, err := message.OutboundMessage()
|
||||
if err != nil {
|
||||
d.failPermanently(ctx, message, err)
|
||||
return
|
||||
}
|
||||
if shouldSkipOutboxEmailPublish(outbound) {
|
||||
publishedAt := time.Now().UTC()
|
||||
if markErr := d.repo.MarkEmailOutboxMessagePublished(ctx, message.ID, publishedAt); markErr != nil {
|
||||
d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
|
||||
snapshot.Failures++
|
||||
snapshot.LastFailureAt = time.Now().UTC()
|
||||
snapshot.LastError = markErr.Error()
|
||||
})
|
||||
d.logger.Error("outbox skip marked failed",
|
||||
slog.String("message_id", message.MessageID),
|
||||
slog.Any("error", markErr),
|
||||
)
|
||||
return
|
||||
}
|
||||
d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
|
||||
snapshot.Published++
|
||||
snapshot.LastSuccessAt = publishedAt
|
||||
snapshot.LastError = ""
|
||||
})
|
||||
d.logger.Info("outbox message skipped",
|
||||
slog.String("message_id", message.MessageID),
|
||||
slog.String("job_type", message.JobType),
|
||||
slog.String("correlation_id", message.CorrelationID),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
dispatchCtx := ctx
|
||||
cancel := func() {}
|
||||
if d.dispatchTimeout > 0 {
|
||||
dispatchCtx, cancel = context.WithTimeout(ctx, d.dispatchTimeout)
|
||||
}
|
||||
err = d.producer.Publish(dispatchCtx, *outbound)
|
||||
cancel()
|
||||
if err == nil {
|
||||
publishedAt := time.Now().UTC()
|
||||
if markErr := d.repo.MarkEmailOutboxMessagePublished(ctx, message.ID, publishedAt); markErr != nil {
|
||||
d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
|
||||
snapshot.Failures++
|
||||
snapshot.LastFailureAt = time.Now().UTC()
|
||||
snapshot.LastError = markErr.Error()
|
||||
})
|
||||
d.logger.Error("outbox publish marked failed",
|
||||
slog.String("message_id", message.MessageID),
|
||||
slog.Any("error", markErr),
|
||||
)
|
||||
return
|
||||
}
|
||||
d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
|
||||
snapshot.Published++
|
||||
snapshot.LastSuccessAt = publishedAt
|
||||
snapshot.LastError = ""
|
||||
})
|
||||
d.logger.Info("outbox message published",
|
||||
slog.String("message_id", message.MessageID),
|
||||
slog.String("job_type", message.JobType),
|
||||
slog.String("correlation_id", message.CorrelationID),
|
||||
slog.Int("attempt", message.Attempts),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if message.Attempts >= d.maxAttempts {
|
||||
d.failPermanently(ctx, message, err)
|
||||
return
|
||||
}
|
||||
|
||||
decision := d.retryPolicy.Decide(&queue.Delivery{ID: message.MessageID, ReceiveCount: message.Attempts}, nil, err)
|
||||
nextAttemptAt := time.Now().UTC().Add(decision.Delay)
|
||||
if retryErr := d.repo.MarkEmailOutboxMessageRetry(ctx, message.ID, nextAttemptAt, err.Error()); retryErr != nil {
|
||||
d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
|
||||
snapshot.Failures++
|
||||
snapshot.LastFailureAt = time.Now().UTC()
|
||||
snapshot.LastError = retryErr.Error()
|
||||
})
|
||||
d.logger.Error("outbox retry scheduling failed",
|
||||
slog.String("message_id", message.MessageID),
|
||||
slog.Any("error", retryErr),
|
||||
)
|
||||
return
|
||||
}
|
||||
d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
|
||||
snapshot.Retried++
|
||||
snapshot.LastFailureAt = time.Now().UTC()
|
||||
snapshot.LastError = err.Error()
|
||||
})
|
||||
d.logger.Warn("outbox publish failed; scheduled retry",
|
||||
slog.String("message_id", message.MessageID),
|
||||
slog.String("job_type", message.JobType),
|
||||
slog.Duration("retry_in", decision.Delay),
|
||||
slog.Int("attempt", message.Attempts),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
}
|
||||
|
||||
func shouldSkipOutboxEmailPublish(outbound *queue.OutboundMessage) bool {
|
||||
if outbound == nil || len(outbound.Body) == 0 {
|
||||
return false
|
||||
}
|
||||
var envelope struct {
|
||||
Payload struct {
|
||||
MessageType string `json:"message_type"`
|
||||
Metadata map[string]string `json:"metadata"`
|
||||
Tags map[string]string `json:"tags"`
|
||||
} `json:"payload"`
|
||||
}
|
||||
if err := json.Unmarshal(outbound.Body, &envelope); err == nil {
|
||||
if strings.EqualFold(strings.TrimSpace(envelope.Payload.Metadata["skip_send"]), "true") {
|
||||
return true
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(envelope.Payload.MessageType), "email_verify") &&
|
||||
strings.EqualFold(strings.TrimSpace(envelope.Payload.Tags["source"]), "register") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
var legacyJob struct {
|
||||
MessageType string `json:"message_type"`
|
||||
Metadata map[string]string `json:"metadata"`
|
||||
Tags map[string]string `json:"tags"`
|
||||
}
|
||||
if err := json.Unmarshal(outbound.Body, &legacyJob); err != nil {
|
||||
return false
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(legacyJob.Metadata["skip_send"]), "true") {
|
||||
return true
|
||||
}
|
||||
return strings.EqualFold(strings.TrimSpace(legacyJob.MessageType), "email_verify") &&
|
||||
strings.EqualFold(strings.TrimSpace(legacyJob.Tags["source"]), "register")
|
||||
}
|
||||
|
||||
func (d *OutboxDispatcher) failPermanently(ctx context.Context, message *auth.EmailOutboxMessage, err error) {
|
||||
if message == nil || err == nil {
|
||||
return
|
||||
}
|
||||
failedAt := time.Now().UTC()
|
||||
if markErr := d.repo.MarkEmailOutboxMessageDead(ctx, message.ID, failedAt, err.Error()); markErr != nil {
|
||||
d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
|
||||
snapshot.Failures++
|
||||
snapshot.LastFailureAt = failedAt
|
||||
snapshot.LastError = markErr.Error()
|
||||
})
|
||||
d.logger.Error("outbox dead-letter marking failed",
|
||||
slog.String("message_id", message.MessageID),
|
||||
slog.Any("error", markErr),
|
||||
)
|
||||
return
|
||||
}
|
||||
d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
|
||||
snapshot.Dead++
|
||||
snapshot.LastFailureAt = failedAt
|
||||
snapshot.LastError = err.Error()
|
||||
})
|
||||
d.logger.Error("outbox message marked dead",
|
||||
slog.String("message_id", message.MessageID),
|
||||
slog.String("job_type", message.JobType),
|
||||
slog.Int("attempt", message.Attempts),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
}
|
||||
|
||||
func (d *OutboxDispatcher) refreshPending(ctx context.Context, now time.Time) {
|
||||
count, err := d.repo.CountPendingEmailOutboxMessages(ctx, now)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
|
||||
snapshot.ApproximatePending = count
|
||||
})
|
||||
}
|
||||
|
||||
func hostname() string {
|
||||
name, err := os.Hostname()
|
||||
if err != nil || strings.TrimSpace(name) == "" {
|
||||
return "worker"
|
||||
}
|
||||
return strings.TrimSpace(name)
|
||||
}
|
||||
Reference in New Issue
Block a user