fm_be/internal/app/worker/outbox_dispatcher.go

package worker

import (
	"context"
	"encoding/json"
	"fmt"
	"log/slog"
	"os"
	"strings"
	"sync"
	"time"

	"wucher/internal/config"
	"wucher/internal/domain/auth"
	"wucher/internal/queue"
)

type OutboxMetricsSnapshot struct {
	Claimed            int64     `json:"claimed"`
	Published          int64     `json:"published"`
	Retried            int64     `json:"retried"`
	Dead               int64     `json:"dead"`
	Failures           int64     `json:"failures"`
	ApproximatePending int64     `json:"approximate_pending"`
	LastSuccessAt      time.Time `json:"last_success_at"`
	LastFailureAt      time.Time `json:"last_failure_at"`
	LastError          string    `json:"last_error,omitempty"`
}

type outboxMetrics struct {
	mu       sync.Mutex
	snapshot OutboxMetricsSnapshot
}

func (m *outboxMetrics) Snapshot() OutboxMetricsSnapshot {
	if m == nil {
		return OutboxMetricsSnapshot{}
	}
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.snapshot
}

func (m *outboxMetrics) update(fn func(snapshot *OutboxMetricsSnapshot)) {
	if m == nil || fn == nil {
		return
	}
	m.mu.Lock()
	defer m.mu.Unlock()
	fn(&m.snapshot)
}

type OutboxDispatcher struct {
	repo            auth.EmailOutboxRepository
	producer        queue.QueueProducer
	logger          *slog.Logger
	retryPolicy     queue.RetryPolicy
	pollInterval    time.Duration
	dispatchTimeout time.Duration
	lockTTL         time.Duration
	maxAttempts     int
	batchSize       int
	workerID        string
	metrics         *outboxMetrics
}

func NewOutboxDispatcher(
	cfg config.QueueConfig,
	repo auth.EmailOutboxRepository,
	producer queue.QueueProducer,
	logger *slog.Logger,
) *OutboxDispatcher {
	if !cfg.OutboxEnabled || repo == nil || producer == nil {
		return nil
	}
	if logger == nil {
		logger = slog.Default()
	}
	workerID := fmt.Sprintf("%s-%d", hostname(), os.Getpid())
	return &OutboxDispatcher{
		repo:            repo,
		producer:        producer,
		logger:          logger.With("component", "outbox-dispatcher", "backend", producer.Backend()),
		retryPolicy:     queue.NewExponentialBackoffPolicy(cfg.WorkerBackoffMin, cfg.WorkerBackoffMax, cfg.WorkerPermanentFailureDelay),
		pollInterval:    cfg.OutboxPollInterval,
		dispatchTimeout: cfg.OutboxDispatchTimeout,
		lockTTL:         cfg.OutboxLockTTL,
		maxAttempts:     cfg.OutboxMaxAttempts,
		batchSize:       cfg.OutboxBatchSize,
		workerID:        workerID,
		metrics:         &outboxMetrics{},
	}
}

func (d *OutboxDispatcher) Metrics() OutboxMetricsSnapshot {
	if d == nil {
		return OutboxMetricsSnapshot{}
	}
	return d.metrics.Snapshot()
}

func (d *OutboxDispatcher) Run(ctx context.Context) error {
	if d == nil {
		return nil
	}
	if d.pollInterval <= 0 {
		d.pollInterval = time.Second
	}
	for {
		processed, err := d.dispatchOnce(ctx)
		if err != nil {
			if ctx.Err() != nil {
				return nil
			}
			d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
				snapshot.Failures++
				snapshot.LastFailureAt = time.Now().UTC()
				snapshot.LastError = err.Error()
			})
			d.logger.Error("outbox dispatch cycle failed", slog.Any("error", err))
		}
		if ctx.Err() != nil {
			return nil
		}
		if processed > 0 {
			continue
		}
		select {
		case <-ctx.Done():
			return nil
		case <-time.After(d.pollInterval):
		}
	}
}

func (d *OutboxDispatcher) dispatchOnce(ctx context.Context) (int, error) {
	now := time.Now().UTC()
	messages, err := d.repo.ClaimPendingEmailOutboxMessages(ctx, d.batchSize, now, d.lockTTL, d.workerID)
	if err != nil {
		return 0, err
	}
	if len(messages) == 0 {
		d.refreshPending(ctx, now)
		return 0, nil
	}
	d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
		snapshot.Claimed += int64(len(messages))
	})
	for i := range messages {
		d.dispatchMessage(ctx, &messages[i])
	}
	d.refreshPending(ctx, time.Now().UTC())
	return len(messages), nil
}

func (d *OutboxDispatcher) dispatchMessage(ctx context.Context, message *auth.EmailOutboxMessage) {
	if message == nil {
		return
	}
	outbound, err := message.OutboundMessage()
	if err != nil {
		d.failPermanently(ctx, message, err)
		return
	}
	if shouldSkipOutboxEmailPublish(outbound) {
		publishedAt := time.Now().UTC()
		if markErr := d.repo.MarkEmailOutboxMessagePublished(ctx, message.ID, publishedAt); markErr != nil {
			d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
				snapshot.Failures++
				snapshot.LastFailureAt = time.Now().UTC()
				snapshot.LastError = markErr.Error()
			})
			d.logger.Error("outbox skip marked failed",
				slog.String("message_id", message.MessageID),
				slog.Any("error", markErr),
			)
			return
		}
		d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
			snapshot.Published++
			snapshot.LastSuccessAt = publishedAt
			snapshot.LastError = ""
		})
		d.logger.Info("outbox message skipped",
			slog.String("message_id", message.MessageID),
			slog.String("job_type", message.JobType),
			slog.String("correlation_id", message.CorrelationID),
		)
		return
	}

	dispatchCtx := ctx
	cancel := func() {}
	if d.dispatchTimeout > 0 {
		dispatchCtx, cancel = context.WithTimeout(ctx, d.dispatchTimeout)
	}
	err = d.producer.Publish(dispatchCtx, *outbound)
	cancel()
	if err == nil {
		publishedAt := time.Now().UTC()
		if markErr := d.repo.MarkEmailOutboxMessagePublished(ctx, message.ID, publishedAt); markErr != nil {
			d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
				snapshot.Failures++
				snapshot.LastFailureAt = time.Now().UTC()
				snapshot.LastError = markErr.Error()
			})
			d.logger.Error("outbox publish marked failed",
				slog.String("message_id", message.MessageID),
				slog.Any("error", markErr),
			)
			return
		}
		d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
			snapshot.Published++
			snapshot.LastSuccessAt = publishedAt
			snapshot.LastError = ""
		})
		d.logger.Info("outbox message published",
			slog.String("message_id", message.MessageID),
			slog.String("job_type", message.JobType),
			slog.String("correlation_id", message.CorrelationID),
			slog.Int("attempt", message.Attempts),
		)
		return
	}

	if message.Attempts >= d.maxAttempts {
		d.failPermanently(ctx, message, err)
		return
	}

	decision := d.retryPolicy.Decide(&queue.Delivery{ID: message.MessageID, ReceiveCount: message.Attempts}, nil, err)
	nextAttemptAt := time.Now().UTC().Add(decision.Delay)
	if retryErr := d.repo.MarkEmailOutboxMessageRetry(ctx, message.ID, nextAttemptAt, err.Error()); retryErr != nil {
		d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
			snapshot.Failures++
			snapshot.LastFailureAt = time.Now().UTC()
			snapshot.LastError = retryErr.Error()
		})
		d.logger.Error("outbox retry scheduling failed",
			slog.String("message_id", message.MessageID),
			slog.Any("error", retryErr),
		)
		return
	}
	d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
		snapshot.Retried++
		snapshot.LastFailureAt = time.Now().UTC()
		snapshot.LastError = err.Error()
	})
	d.logger.Warn("outbox publish failed; scheduled retry",
		slog.String("message_id", message.MessageID),
		slog.String("job_type", message.JobType),
		slog.Duration("retry_in", decision.Delay),
		slog.Int("attempt", message.Attempts),
		slog.Any("error", err),
	)
}

func shouldSkipOutboxEmailPublish(outbound *queue.OutboundMessage) bool {
	if outbound == nil || len(outbound.Body) == 0 {
		return false
	}
	var envelope struct {
		Payload struct {
			MessageType string            `json:"message_type"`
			Metadata    map[string]string `json:"metadata"`
			Tags        map[string]string `json:"tags"`
		} `json:"payload"`
	}
	if err := json.Unmarshal(outbound.Body, &envelope); err == nil {
		if strings.EqualFold(strings.TrimSpace(envelope.Payload.Metadata["skip_send"]), "true") {
			return true
		}
		if strings.EqualFold(strings.TrimSpace(envelope.Payload.MessageType), "email_verify") &&
			strings.EqualFold(strings.TrimSpace(envelope.Payload.Tags["source"]), "register") {
			return true
		}
		return false
	}
	var legacyJob struct {
		MessageType string            `json:"message_type"`
		Metadata    map[string]string `json:"metadata"`
		Tags        map[string]string `json:"tags"`
	}
	if err := json.Unmarshal(outbound.Body, &legacyJob); err != nil {
		return false
	}
	if strings.EqualFold(strings.TrimSpace(legacyJob.Metadata["skip_send"]), "true") {
		return true
	}
	return strings.EqualFold(strings.TrimSpace(legacyJob.MessageType), "email_verify") &&
		strings.EqualFold(strings.TrimSpace(legacyJob.Tags["source"]), "register")
}

func (d *OutboxDispatcher) failPermanently(ctx context.Context, message *auth.EmailOutboxMessage, err error) {
	if message == nil || err == nil {
		return
	}
	failedAt := time.Now().UTC()
	if markErr := d.repo.MarkEmailOutboxMessageDead(ctx, message.ID, failedAt, err.Error()); markErr != nil {
		d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
			snapshot.Failures++
			snapshot.LastFailureAt = failedAt
			snapshot.LastError = markErr.Error()
		})
		d.logger.Error("outbox dead-letter marking failed",
			slog.String("message_id", message.MessageID),
			slog.Any("error", markErr),
		)
		return
	}
	d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
		snapshot.Dead++
		snapshot.LastFailureAt = failedAt
		snapshot.LastError = err.Error()
	})
	d.logger.Error("outbox message marked dead",
		slog.String("message_id", message.MessageID),
		slog.String("job_type", message.JobType),
		slog.Int("attempt", message.Attempts),
		slog.Any("error", err),
	)
}

func (d *OutboxDispatcher) refreshPending(ctx context.Context, now time.Time) {
	count, err := d.repo.CountPendingEmailOutboxMessages(ctx, now)
	if err != nil {
		return
	}
	d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
		snapshot.ApproximatePending = count
	})
}

func hostname() string {
	name, err := os.Hostname()
	if err != nil || strings.TrimSpace(name) == "" {
		return "worker"
	}
	return strings.TrimSpace(name)
}