init push

internal/app/worker/file_processing_outbox_dispatcher.go

@@ -0,0 +1,241 @@
+package worker
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"time"
+
+	"wucher/internal/config"
+	filemanager "wucher/internal/domain/file_manager"
+	"wucher/internal/queue"
+)
+
+type FileProcessingOutboxDispatcher struct {
+	repo            filemanager.FileProcessingOutboxRepository
+	producer        queue.QueueProducer
+	logger          *slog.Logger
+	retryPolicy     queue.RetryPolicy
+	pollInterval    time.Duration
+	dispatchTimeout time.Duration
+	lockTTL         time.Duration
+	maxAttempts     int
+	batchSize       int
+	workerID        string
+	metrics         *outboxMetrics
+}
+
+func NewFileProcessingOutboxDispatcher(
+	cfg config.FileQueueConfig,
+	repo filemanager.FileProcessingOutboxRepository,
+	producer queue.QueueProducer,
+	logger *slog.Logger,
+) *FileProcessingOutboxDispatcher {
+	if !cfg.OutboxEnabled || repo == nil || producer == nil {
+		return nil
+	}
+	if logger == nil {
+		logger = slog.Default()
+	}
+	workerID := fmt.Sprintf("%s-%d", hostname(), os.Getpid())
+	return &FileProcessingOutboxDispatcher{
+		repo:            repo,
+		producer:        producer,
+		logger:          logger.With("component", "file-outbox-dispatcher", "backend", producer.Backend()),
+		retryPolicy:     queue.NewExponentialBackoffPolicy(cfg.WorkerBackoffMin, cfg.WorkerBackoffMax, cfg.WorkerPermanentFailureDelay),
+		pollInterval:    cfg.OutboxPollInterval,
+		dispatchTimeout: cfg.OutboxDispatchTimeout,
+		lockTTL:         cfg.OutboxLockTTL,
+		maxAttempts:     cfg.OutboxMaxAttempts,
+		batchSize:       cfg.OutboxBatchSize,
+		workerID:        workerID,
+		metrics:         &outboxMetrics{},
+	}
+}
+
+func (d *FileProcessingOutboxDispatcher) Metrics() OutboxMetricsSnapshot {
+	if d == nil {
+		return OutboxMetricsSnapshot{}
+	}
+	return d.metrics.Snapshot()
+}
+
+func (d *FileProcessingOutboxDispatcher) Run(ctx context.Context) error {
+	if d == nil {
+		return nil
+	}
+	if d.pollInterval <= 0 {
+		d.pollInterval = time.Second
+	}
+	for {
+		processed, err := d.dispatchOnce(ctx)
+		if err != nil {
+			if ctx.Err() != nil {
+				return nil
+			}
+			d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+				snapshot.Failures++
+				snapshot.LastFailureAt = time.Now().UTC()
+				snapshot.LastError = err.Error()
+			})
+			d.logger.Error("file outbox dispatch cycle failed", slog.Any("error", err))
+		}
+		if ctx.Err() != nil {
+			return nil
+		}
+		if processed > 0 {
+			continue
+		}
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-time.After(d.pollInterval):
+		}
+	}
+}
+
+func (d *FileProcessingOutboxDispatcher) dispatchOnce(ctx context.Context) (int, error) {
+	now := time.Now().UTC()
+	messages, err := d.repo.ClaimPendingFileProcessingOutboxMessages(ctx, d.batchSize, now, d.lockTTL, d.workerID)
+	if err != nil {
+		return 0, err
+	}
+	if len(messages) == 0 {
+		d.refreshPending(ctx, now)
+		return 0, nil
+	}
+	d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+		snapshot.Claimed += int64(len(messages))
+	})
+	for i := range messages {
+		d.dispatchMessage(ctx, &messages[i])
+	}
+	d.refreshPending(ctx, time.Now().UTC())
+	return len(messages), nil
+}
+
+func (d *FileProcessingOutboxDispatcher) dispatchMessage(ctx context.Context, message *filemanager.FileProcessingOutboxMessage) {
+	if message == nil {
+		return
+	}
+	outbound, err := message.OutboundMessage()
+	if err != nil {
+		d.failPermanently(ctx, message, err)
+		return
+	}
+	queueMessage := queue.OutboundMessage{
+		ID:              outbound.ID,
+		Body:            outbound.Body,
+		Attributes:      outbound.Attributes,
+		MessageGroupID:  outbound.MessageGroupID,
+		DeduplicationID: outbound.DeduplicationID,
+	}
+
+	dispatchCtx := ctx
+	cancel := func() {}
+	if d.dispatchTimeout > 0 {
+		dispatchCtx, cancel = context.WithTimeout(ctx, d.dispatchTimeout)
+	}
+	err = d.producer.Publish(dispatchCtx, queueMessage)
+	cancel()
+	if err == nil {
+		publishedAt := time.Now().UTC()
+		if markErr := d.repo.MarkFileProcessingOutboxMessagePublished(ctx, message.ID, publishedAt); markErr != nil {
+			d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+				snapshot.Failures++
+				snapshot.LastFailureAt = time.Now().UTC()
+				snapshot.LastError = markErr.Error()
+			})
+			d.logger.Error("file outbox publish marked failed",
+				slog.String("message_id", message.MessageID),
+				slog.Any("error", markErr),
+			)
+			return
+		}
+		d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+			snapshot.Published++
+			snapshot.LastSuccessAt = publishedAt
+			snapshot.LastError = ""
+		})
+		d.logger.Info("file outbox message published",
+			slog.String("message_id", message.MessageID),
+			slog.String("job_type", message.JobType),
+			slog.String("correlation_id", message.CorrelationID),
+			slog.Int("attempt", message.Attempts),
+		)
+		return
+	}
+
+	if message.Attempts >= d.maxAttempts {
+		d.failPermanently(ctx, message, err)
+		return
+	}
+
+	decision := d.retryPolicy.Decide(&queue.Delivery{ID: message.MessageID, ReceiveCount: message.Attempts}, nil, err)
+	nextAttemptAt := time.Now().UTC().Add(decision.Delay)
+	if retryErr := d.repo.MarkFileProcessingOutboxMessageRetry(ctx, message.ID, nextAttemptAt, err.Error()); retryErr != nil {
+		d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+			snapshot.Failures++
+			snapshot.LastFailureAt = time.Now().UTC()
+			snapshot.LastError = retryErr.Error()
+		})
+		d.logger.Error("file outbox retry scheduling failed",
+			slog.String("message_id", message.MessageID),
+			slog.Any("error", retryErr),
+		)
+		return
+	}
+	d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+		snapshot.Retried++
+		snapshot.LastFailureAt = time.Now().UTC()
+		snapshot.LastError = err.Error()
+	})
+	d.logger.Warn("file outbox publish failed; scheduled retry",
+		slog.String("message_id", message.MessageID),
+		slog.String("job_type", message.JobType),
+		slog.Duration("retry_in", decision.Delay),
+		slog.Int("attempt", message.Attempts),
+		slog.Any("error", err),
+	)
+}
+
+func (d *FileProcessingOutboxDispatcher) failPermanently(ctx context.Context, message *filemanager.FileProcessingOutboxMessage, err error) {
+	if message == nil || err == nil {
+		return
+	}
+	failedAt := time.Now().UTC()
+	if markErr := d.repo.MarkFileProcessingOutboxMessageDead(ctx, message.ID, failedAt, err.Error()); markErr != nil {
+		d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+			snapshot.Failures++
+			snapshot.LastFailureAt = failedAt
+			snapshot.LastError = markErr.Error()
+		})
+		d.logger.Error("file outbox dead-letter marking failed",
+			slog.String("message_id", message.MessageID),
+			slog.Any("error", markErr),
+		)
+		return
+	}
+	d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+		snapshot.Dead++
+		snapshot.LastFailureAt = failedAt
+		snapshot.LastError = err.Error()
+	})
+	d.logger.Error("file outbox message marked dead",
+		slog.String("message_id", message.MessageID),
+		slog.String("job_type", message.JobType),
+		slog.Int("attempt", message.Attempts),
+		slog.Any("error", err),
+	)
+}
+
+func (d *FileProcessingOutboxDispatcher) refreshPending(ctx context.Context, now time.Time) {
+	count, err := d.repo.CountPendingFileProcessingOutboxMessages(ctx, now)
+	if err != nil {
+		return
+	}
+	d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+		snapshot.ApproximatePending = count
+	})
+}

internal/app/worker/file_processing_outbox_dispatcher_test.go

@@ -0,0 +1,219 @@
+package worker
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"wucher/internal/config"
+	filemanager "wucher/internal/domain/file_manager"
+	"wucher/internal/queue"
+)
+
+type stubFileOutboxRepo struct {
+	messages map[string]*filemanager.FileProcessingOutboxMessage
+}
+
+func newStubFileOutboxRepo(messages ...*filemanager.FileProcessingOutboxMessage) *stubFileOutboxRepo {
+	repo := &stubFileOutboxRepo{messages: map[string]*filemanager.FileProcessingOutboxMessage{}}
+	for _, message := range messages {
+		if message == nil {
+			continue
+		}
+		cp := *message
+		cp.ID = append([]byte(nil), message.ID...)
+		cp.Body = append([]byte(nil), message.Body...)
+		cp.Attributes = append([]byte(nil), message.Attributes...)
+		repo.messages[string(cp.ID)] = &cp
+	}
+	return repo
+}
+
+func (r *stubFileOutboxRepo) CreateFileProcessingOutboxMessage(_ context.Context, message *filemanager.FileProcessingOutboxMessage) error {
+	if message == nil {
+		return errors.New("nil message")
+	}
+	cp := *message
+	cp.ID = append([]byte(nil), message.ID...)
+	cp.Body = append([]byte(nil), message.Body...)
+	cp.Attributes = append([]byte(nil), message.Attributes...)
+	r.messages[string(cp.ID)] = &cp
+	return nil
+}
+
+func (r *stubFileOutboxRepo) ClaimPendingFileProcessingOutboxMessages(_ context.Context, limit int, now time.Time, lockTTL time.Duration, workerID string) ([]filemanager.FileProcessingOutboxMessage, error) {
+	if limit <= 0 {
+		limit = 1
+	}
+	if lockTTL <= 0 {
+		lockTTL = time.Minute
+	}
+	out := make([]filemanager.FileProcessingOutboxMessage, 0, limit)
+	staleBefore := now.Add(-lockTTL)
+	for _, message := range r.messages {
+		if len(out) >= limit {
+			break
+		}
+		switch {
+		case message.Status == filemanager.FileProcessingOutboxStatusPending && !message.AvailableAt.After(now):
+		case message.Status == filemanager.FileProcessingOutboxStatusProcessing && message.LockedAt != nil && message.LockedAt.Before(staleBefore):
+		default:
+			continue
+		}
+		message.Status = filemanager.FileProcessingOutboxStatusProcessing
+		message.Attempts++
+		ts := now
+		message.LockedAt = &ts
+		message.LockedBy = workerID
+		cp := *message
+		out = append(out, cp)
+	}
+	return out, nil
+}
+
+func (r *stubFileOutboxRepo) MarkFileProcessingOutboxMessagePublished(_ context.Context, id []byte, publishedAt time.Time) error {
+	message := r.messages[string(id)]
+	if message == nil {
+		return errors.New("not found")
+	}
+	message.Status = filemanager.FileProcessingOutboxStatusPublished
+	message.PublishedAt = &publishedAt
+	message.LockedAt = nil
+	message.LockedBy = ""
+	message.LastError = ""
+	return nil
+}
+
+func (r *stubFileOutboxRepo) MarkFileProcessingOutboxMessageRetry(_ context.Context, id []byte, availableAt time.Time, lastErr string) error {
+	message := r.messages[string(id)]
+	if message == nil {
+		return errors.New("not found")
+	}
+	message.Status = filemanager.FileProcessingOutboxStatusPending
+	message.AvailableAt = availableAt
+	message.LockedAt = nil
+	message.LockedBy = ""
+	message.LastError = lastErr
+	return nil
+}
+
+func (r *stubFileOutboxRepo) MarkFileProcessingOutboxMessageDead(_ context.Context, id []byte, failedAt time.Time, lastErr string) error {
+	message := r.messages[string(id)]
+	if message == nil {
+		return errors.New("not found")
+	}
+	message.Status = filemanager.FileProcessingOutboxStatusDead
+	message.AvailableAt = failedAt
+	message.LockedAt = nil
+	message.LockedBy = ""
+	message.LastError = lastErr
+	return nil
+}
+
+func (r *stubFileOutboxRepo) CountPendingFileProcessingOutboxMessages(_ context.Context, now time.Time) (int64, error) {
+	var count int64
+	for _, message := range r.messages {
+		if (message.Status == filemanager.FileProcessingOutboxStatusPending && !message.AvailableAt.After(now)) || message.Status == filemanager.FileProcessingOutboxStatusProcessing {
+			count++
+		}
+	}
+	return count, nil
+}
+
+func newFileOutboxRecord(t *testing.T, job queue.FileProcessingJob) *filemanager.FileProcessingOutboxMessage {
+	t.Helper()
+	serializer := queue.NewFileProcessingJSONSerializer("v1", queue.QueueTypeStandard, "")
+	message, err := serializer.Serialize(context.Background(), job)
+	if err != nil {
+		t.Fatalf("serialize: %v", err)
+	}
+	record, err := filemanager.NewFileProcessingOutboxMessage(time.Now().UTC(), &filemanager.FileProcessingOutboundMessage{
+		ID:              message.ID,
+		Body:            append([]byte(nil), message.Body...),
+		Attributes:      message.Attributes,
+		MessageGroupID:  message.MessageGroupID,
+		DeduplicationID: message.DeduplicationID,
+	})
+	if err != nil {
+		t.Fatalf("new outbox message: %v", err)
+	}
+	return record
+}
+
+func TestFileProcessingOutboxDispatcherPublishSuccess(t *testing.T) {
+	record := newFileOutboxRecord(t, queue.FileProcessingJob{
+		FileUUID:  "01961f5c-6471-7b81-9df5-2e2e1cf7e498",
+		Bucket:    "bucket-a",
+		ObjectKey: "files/2026/04/05/object.pdf",
+		SizeBytes: 128,
+	})
+	repo := newStubFileOutboxRepo(record)
+	producer := &stubQueueProducer{}
+	dispatcher := NewFileProcessingOutboxDispatcher(config.FileQueueConfig{
+		OutboxEnabled:         true,
+		OutboxBatchSize:       10,
+		OutboxDispatchTimeout: time.Second,
+		OutboxLockTTL:         time.Minute,
+		OutboxMaxAttempts:     3,
+		WorkerBackoffMin:      time.Second,
+		WorkerBackoffMax:      time.Minute,
+	}, repo, producer, nil)
+
+	processed, err := dispatcher.dispatchOnce(context.Background())
+	if err != nil {
+		t.Fatalf("dispatch once: %v", err)
+	}
+	if processed != 1 {
+		t.Fatalf("expected one processed message, got %d", processed)
+	}
+	if len(producer.published) != 1 {
+		t.Fatalf("expected one published message, got %d", len(producer.published))
+	}
+	if repo.messages[string(record.ID)].Status != filemanager.FileProcessingOutboxStatusPublished {
+		t.Fatalf("expected message published, got %s", repo.messages[string(record.ID)].Status)
+	}
+	if got := dispatcher.Metrics().Published; got != 1 {
+		t.Fatalf("expected published metric=1, got %d", got)
+	}
+}
+
+func TestFileProcessingOutboxDispatcherRetryOnPublishFailure(t *testing.T) {
+	record := newFileOutboxRecord(t, queue.FileProcessingJob{
+		FileUUID:  "01961f5c-6471-7b81-9df5-2e2e1cf7e499",
+		Bucket:    "bucket-a",
+		ObjectKey: "files/2026/04/05/object.pdf",
+		SizeBytes: 256,
+	})
+	repo := newStubFileOutboxRepo(record)
+	producer := &stubQueueProducer{err: errors.New("queue unavailable")}
+	dispatcher := NewFileProcessingOutboxDispatcher(config.FileQueueConfig{
+		OutboxEnabled:     true,
+		OutboxBatchSize:   10,
+		OutboxLockTTL:     time.Minute,
+		OutboxMaxAttempts: 3,
+		WorkerBackoffMin:  time.Second,
+		WorkerBackoffMax:  time.Minute,
+	}, repo, producer, nil)
+
+	processed, err := dispatcher.dispatchOnce(context.Background())
+	if err != nil {
+		t.Fatalf("dispatch once: %v", err)
+	}
+	if processed != 1 {
+		t.Fatalf("expected one processed message, got %d", processed)
+	}
+	got := repo.messages[string(record.ID)]
+	if got.Status != filemanager.FileProcessingOutboxStatusPending {
+		t.Fatalf("expected message returned to pending, got %s", got.Status)
+	}
+	if got.Attempts != 1 {
+		t.Fatalf("expected attempts=1, got %d", got.Attempts)
+	}
+	if !got.AvailableAt.After(time.Now().Add(-time.Second)) {
+		t.Fatalf("expected retry to schedule next attempt in the future")
+	}
+	if metrics := dispatcher.Metrics(); metrics.Retried != 1 {
+		t.Fatalf("expected retried metric=1, got %d", metrics.Retried)
+	}
+}

internal/app/worker/file_processing_runner.go

@@ -0,0 +1,193 @@
+package worker
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"log/slog"
+	"net/http"
+	"time"
+
+	"wucher/internal/config"
+	"wucher/internal/shared/pkg/metrics"
+	"wucher/internal/queue"
+)
+
+type FileProcessingRunner struct {
+	worker            *queue.FileProcessingWorker
+	consumer          queue.QueueConsumer
+	dispatcher        *FileProcessingOutboxDispatcher
+	logger            *slog.Logger
+	monitorAddr       string
+	depthPollInterval time.Duration
+	shutdownTimeout   time.Duration
+	backend           string
+	monitorServer     *http.Server
+}
+
+func NewFileProcessingRunner(
+	cfg config.FileQueueConfig,
+	consumer queue.QueueConsumer,
+	serializer queue.FileProcessingMessageSerializer,
+	handler queue.FileProcessingJobHandler,
+	idempotency queue.IdempotencyStore,
+	dispatcher *FileProcessingOutboxDispatcher,
+	logger *slog.Logger,
+) *FileProcessingRunner {
+	if logger == nil {
+		logger = slog.Default()
+	}
+	metrics := queue.NewMetrics()
+	health := queue.NewHealthReporter(cfg.WorkerHealthErrorThreshold)
+	worker := queue.NewFileProcessingWorker(queue.WorkerConfig{
+		Concurrency:      cfg.WorkerConcurrency,
+		MaxBatchSize:     cfg.WorkerMaxBatchSize,
+		ProcessTimeout:   cfg.WorkerProcessTimeout,
+		ProcessingTTL:    cfg.ProcessingTTL,
+		IdempotencyTTL:   cfg.IdempotencyTTL,
+		PollErrorDelay:   cfg.WorkerBackoffMin,
+		MetricsComponent: "file_worker",
+	}, consumer, serializer, handler, queue.NewExponentialBackoffPolicy(cfg.WorkerBackoffMin, cfg.WorkerBackoffMax, cfg.WorkerPermanentFailureDelay), idempotency, metrics, health, logger)
+
+	backend := ""
+	if consumer != nil {
+		backend = consumer.Backend()
+	}
+	return &FileProcessingRunner{
+		worker:            worker,
+		consumer:          consumer,
+		dispatcher:        dispatcher,
+		logger:            logger,
+		monitorAddr:       cfg.WorkerMonitorAddr,
+		depthPollInterval: cfg.WorkerDepthPollInterval,
+		shutdownTimeout:   cfg.WorkerShutdownTimeout,
+		backend:           backend,
+	}
+}
+
+func (r *FileProcessingRunner) Start(ctx context.Context) error {
+	if r.worker == nil {
+		return errors.New("worker is required")
+	}
+	runCtx, cancelRun := context.WithCancel(ctx)
+	defer cancelRun()
+	if err := r.startMonitor(); err != nil {
+		return err
+	}
+	pollerDone := make(chan struct{})
+	go func() {
+		defer close(pollerDone)
+		r.runDepthPoller(runCtx)
+	}()
+
+	workerErrCh := make(chan error, 1)
+	go func() {
+		workerErrCh <- r.worker.Run(runCtx)
+	}()
+
+	var dispatcherErrCh chan error
+	if r.dispatcher != nil {
+		dispatcherErrCh = make(chan error, 1)
+		go func() {
+			dispatcherErrCh <- r.dispatcher.Run(runCtx)
+		}()
+	}
+
+	var err error
+	select {
+	case err = <-workerErrCh:
+		cancelRun()
+		if dispatcherErrCh != nil {
+			if dispatcherErr := <-dispatcherErrCh; err == nil {
+				err = dispatcherErr
+			}
+		}
+	case err = <-dispatcherErrCh:
+		cancelRun()
+		if workerErr := <-workerErrCh; err == nil {
+			err = workerErr
+		}
+	}
+
+	cancelRun()
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), r.shutdownTimeout)
+	defer cancel()
+	if r.monitorServer != nil {
+		_ = r.monitorServer.Shutdown(shutdownCtx)
+	}
+	if r.consumer != nil {
+		_ = r.consumer.Close()
+	}
+	<-pollerDone
+	return err
+}
+
+func (r *FileProcessingRunner) startMonitor() error {
+	if r.monitorAddr == "" {
+		return nil
+	}
+	mux := http.NewServeMux()
+	mux.HandleFunc("/health", r.handleHealth)
+	mux.Handle("/metrics", metrics.PromHTTPHandler())
+	r.monitorServer = &http.Server{
+		Addr:              r.monitorAddr,
+		Handler:           mux,
+		ReadHeaderTimeout: 5 * time.Second,
+	}
+	go func() {
+		r.logger.Info("file worker monitor listening",
+			slog.String("addr", r.monitorAddr),
+			slog.String("backend", r.backend),
+		)
+		if err := r.monitorServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			r.logger.Error("file worker monitor failed", slog.Any("error", err))
+		}
+	}()
+	return nil
+}
+
+func (r *FileProcessingRunner) runDepthPoller(ctx context.Context) {
+	if r.depthPollInterval <= 0 {
+		return
+	}
+	ticker := time.NewTicker(r.depthPollInterval)
+	defer ticker.Stop()
+	for {
+		refreshCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		_ = r.worker.RefreshQueueDepth(refreshCtx)
+		cancel()
+
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		}
+	}
+}
+
+func (r *FileProcessingRunner) handleHealth(w http.ResponseWriter, _ *http.Request) {
+	snapshot := struct {
+		Backend string                 `json:"backend"`
+		Health  queue.HealthSnapshot   `json:"health"`
+		Metrics queue.MetricsSnapshot  `json:"metrics"`
+		Outbox  *OutboxMetricsSnapshot `json:"outbox,omitempty"`
+	}{
+		Backend: r.backend,
+		Health:  r.worker.Health().Snapshot(),
+		Metrics: r.worker.Metrics().Snapshot(),
+	}
+	if r.dispatcher != nil {
+		snapshot.Outbox = ptr(r.dispatcher.Metrics())
+	}
+	statusCode := http.StatusOK
+	if snapshot.Health.Status == "degraded" {
+		statusCode = http.StatusServiceUnavailable
+	}
+	writeFileWorkerJSON(w, statusCode, snapshot)
+}
+
+func writeFileWorkerJSON(w http.ResponseWriter, statusCode int, payload any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(statusCode)
+	_ = json.NewEncoder(w).Encode(payload)
+}

internal/app/worker/outbox_dispatcher.go

@@ -0,0 +1,342 @@
+package worker
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"wucher/internal/config"
+	"wucher/internal/domain/auth"
+	"wucher/internal/queue"
+)
+
+type OutboxMetricsSnapshot struct {
+	Claimed            int64     `json:"claimed"`
+	Published          int64     `json:"published"`
+	Retried            int64     `json:"retried"`
+	Dead               int64     `json:"dead"`
+	Failures           int64     `json:"failures"`
+	ApproximatePending int64     `json:"approximate_pending"`
+	LastSuccessAt      time.Time `json:"last_success_at"`
+	LastFailureAt      time.Time `json:"last_failure_at"`
+	LastError          string    `json:"last_error,omitempty"`
+}
+
+type outboxMetrics struct {
+	mu       sync.Mutex
+	snapshot OutboxMetricsSnapshot
+}
+
+func (m *outboxMetrics) Snapshot() OutboxMetricsSnapshot {
+	if m == nil {
+		return OutboxMetricsSnapshot{}
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.snapshot
+}
+
+func (m *outboxMetrics) update(fn func(snapshot *OutboxMetricsSnapshot)) {
+	if m == nil || fn == nil {
+		return
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	fn(&m.snapshot)
+}
+
+type OutboxDispatcher struct {
+	repo            auth.EmailOutboxRepository
+	producer        queue.QueueProducer
+	logger          *slog.Logger
+	retryPolicy     queue.RetryPolicy
+	pollInterval    time.Duration
+	dispatchTimeout time.Duration
+	lockTTL         time.Duration
+	maxAttempts     int
+	batchSize       int
+	workerID        string
+	metrics         *outboxMetrics
+}
+
+func NewOutboxDispatcher(
+	cfg config.QueueConfig,
+	repo auth.EmailOutboxRepository,
+	producer queue.QueueProducer,
+	logger *slog.Logger,
+) *OutboxDispatcher {
+	if !cfg.OutboxEnabled || repo == nil || producer == nil {
+		return nil
+	}
+	if logger == nil {
+		logger = slog.Default()
+	}
+	workerID := fmt.Sprintf("%s-%d", hostname(), os.Getpid())
+	return &OutboxDispatcher{
+		repo:            repo,
+		producer:        producer,
+		logger:          logger.With("component", "outbox-dispatcher", "backend", producer.Backend()),
+		retryPolicy:     queue.NewExponentialBackoffPolicy(cfg.WorkerBackoffMin, cfg.WorkerBackoffMax, cfg.WorkerPermanentFailureDelay),
+		pollInterval:    cfg.OutboxPollInterval,
+		dispatchTimeout: cfg.OutboxDispatchTimeout,
+		lockTTL:         cfg.OutboxLockTTL,
+		maxAttempts:     cfg.OutboxMaxAttempts,
+		batchSize:       cfg.OutboxBatchSize,
+		workerID:        workerID,
+		metrics:         &outboxMetrics{},
+	}
+}
+
+func (d *OutboxDispatcher) Metrics() OutboxMetricsSnapshot {
+	if d == nil {
+		return OutboxMetricsSnapshot{}
+	}
+	return d.metrics.Snapshot()
+}
+
+func (d *OutboxDispatcher) Run(ctx context.Context) error {
+	if d == nil {
+		return nil
+	}
+	if d.pollInterval <= 0 {
+		d.pollInterval = time.Second
+	}
+	for {
+		processed, err := d.dispatchOnce(ctx)
+		if err != nil {
+			if ctx.Err() != nil {
+				return nil
+			}
+			d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+				snapshot.Failures++
+				snapshot.LastFailureAt = time.Now().UTC()
+				snapshot.LastError = err.Error()
+			})
+			d.logger.Error("outbox dispatch cycle failed", slog.Any("error", err))
+		}
+		if ctx.Err() != nil {
+			return nil
+		}
+		if processed > 0 {
+			continue
+		}
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-time.After(d.pollInterval):
+		}
+	}
+}
+
+func (d *OutboxDispatcher) dispatchOnce(ctx context.Context) (int, error) {
+	now := time.Now().UTC()
+	messages, err := d.repo.ClaimPendingEmailOutboxMessages(ctx, d.batchSize, now, d.lockTTL, d.workerID)
+	if err != nil {
+		return 0, err
+	}
+	if len(messages) == 0 {
+		d.refreshPending(ctx, now)
+		return 0, nil
+	}
+	d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+		snapshot.Claimed += int64(len(messages))
+	})
+	for i := range messages {
+		d.dispatchMessage(ctx, &messages[i])
+	}
+	d.refreshPending(ctx, time.Now().UTC())
+	return len(messages), nil
+}
+
+func (d *OutboxDispatcher) dispatchMessage(ctx context.Context, message *auth.EmailOutboxMessage) {
+	if message == nil {
+		return
+	}
+	outbound, err := message.OutboundMessage()
+	if err != nil {
+		d.failPermanently(ctx, message, err)
+		return
+	}
+	if shouldSkipOutboxEmailPublish(outbound) {
+		publishedAt := time.Now().UTC()
+		if markErr := d.repo.MarkEmailOutboxMessagePublished(ctx, message.ID, publishedAt); markErr != nil {
+			d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+				snapshot.Failures++
+				snapshot.LastFailureAt = time.Now().UTC()
+				snapshot.LastError = markErr.Error()
+			})
+			d.logger.Error("outbox skip marked failed",
+				slog.String("message_id", message.MessageID),
+				slog.Any("error", markErr),
+			)
+			return
+		}
+		d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+			snapshot.Published++
+			snapshot.LastSuccessAt = publishedAt
+			snapshot.LastError = ""
+		})
+		d.logger.Info("outbox message skipped",
+			slog.String("message_id", message.MessageID),
+			slog.String("job_type", message.JobType),
+			slog.String("correlation_id", message.CorrelationID),
+		)
+		return
+	}
+
+	dispatchCtx := ctx
+	cancel := func() {}
+	if d.dispatchTimeout > 0 {
+		dispatchCtx, cancel = context.WithTimeout(ctx, d.dispatchTimeout)
+	}
+	err = d.producer.Publish(dispatchCtx, *outbound)
+	cancel()
+	if err == nil {
+		publishedAt := time.Now().UTC()
+		if markErr := d.repo.MarkEmailOutboxMessagePublished(ctx, message.ID, publishedAt); markErr != nil {
+			d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+				snapshot.Failures++
+				snapshot.LastFailureAt = time.Now().UTC()
+				snapshot.LastError = markErr.Error()
+			})
+			d.logger.Error("outbox publish marked failed",
+				slog.String("message_id", message.MessageID),
+				slog.Any("error", markErr),
+			)
+			return
+		}
+		d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+			snapshot.Published++
+			snapshot.LastSuccessAt = publishedAt
+			snapshot.LastError = ""
+		})
+		d.logger.Info("outbox message published",
+			slog.String("message_id", message.MessageID),
+			slog.String("job_type", message.JobType),
+			slog.String("correlation_id", message.CorrelationID),
+			slog.Int("attempt", message.Attempts),
+		)
+		return
+	}
+
+	if message.Attempts >= d.maxAttempts {
+		d.failPermanently(ctx, message, err)
+		return
+	}
+
+	decision := d.retryPolicy.Decide(&queue.Delivery{ID: message.MessageID, ReceiveCount: message.Attempts}, nil, err)
+	nextAttemptAt := time.Now().UTC().Add(decision.Delay)
+	if retryErr := d.repo.MarkEmailOutboxMessageRetry(ctx, message.ID, nextAttemptAt, err.Error()); retryErr != nil {
+		d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+			snapshot.Failures++
+			snapshot.LastFailureAt = time.Now().UTC()
+			snapshot.LastError = retryErr.Error()
+		})
+		d.logger.Error("outbox retry scheduling failed",
+			slog.String("message_id", message.MessageID),
+			slog.Any("error", retryErr),
+		)
+		return
+	}
+	d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+		snapshot.Retried++
+		snapshot.LastFailureAt = time.Now().UTC()
+		snapshot.LastError = err.Error()
+	})
+	d.logger.Warn("outbox publish failed; scheduled retry",
+		slog.String("message_id", message.MessageID),
+		slog.String("job_type", message.JobType),
+		slog.Duration("retry_in", decision.Delay),
+		slog.Int("attempt", message.Attempts),
+		slog.Any("error", err),
+	)
+}
+
+func shouldSkipOutboxEmailPublish(outbound *queue.OutboundMessage) bool {
+	if outbound == nil || len(outbound.Body) == 0 {
+		return false
+	}
+	var envelope struct {
+		Payload struct {
+			MessageType string            `json:"message_type"`
+			Metadata    map[string]string `json:"metadata"`
+			Tags        map[string]string `json:"tags"`
+		} `json:"payload"`
+	}
+	if err := json.Unmarshal(outbound.Body, &envelope); err == nil {
+		if strings.EqualFold(strings.TrimSpace(envelope.Payload.Metadata["skip_send"]), "true") {
+			return true
+		}
+		if strings.EqualFold(strings.TrimSpace(envelope.Payload.MessageType), "email_verify") &&
+			strings.EqualFold(strings.TrimSpace(envelope.Payload.Tags["source"]), "register") {
+			return true
+		}
+		return false
+	}
+	var legacyJob struct {
+		MessageType string            `json:"message_type"`
+		Metadata    map[string]string `json:"metadata"`
+		Tags        map[string]string `json:"tags"`
+	}
+	if err := json.Unmarshal(outbound.Body, &legacyJob); err != nil {
+		return false
+	}
+	if strings.EqualFold(strings.TrimSpace(legacyJob.Metadata["skip_send"]), "true") {
+		return true
+	}
+	return strings.EqualFold(strings.TrimSpace(legacyJob.MessageType), "email_verify") &&
+		strings.EqualFold(strings.TrimSpace(legacyJob.Tags["source"]), "register")
+}
+
+func (d *OutboxDispatcher) failPermanently(ctx context.Context, message *auth.EmailOutboxMessage, err error) {
+	if message == nil || err == nil {
+		return
+	}
+	failedAt := time.Now().UTC()
+	if markErr := d.repo.MarkEmailOutboxMessageDead(ctx, message.ID, failedAt, err.Error()); markErr != nil {
+		d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+			snapshot.Failures++
+			snapshot.LastFailureAt = failedAt
+			snapshot.LastError = markErr.Error()
+		})
+		d.logger.Error("outbox dead-letter marking failed",
+			slog.String("message_id", message.MessageID),
+			slog.Any("error", markErr),
+		)
+		return
+	}
+	d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+		snapshot.Dead++
+		snapshot.LastFailureAt = failedAt
+		snapshot.LastError = err.Error()
+	})
+	d.logger.Error("outbox message marked dead",
+		slog.String("message_id", message.MessageID),
+		slog.String("job_type", message.JobType),
+		slog.Int("attempt", message.Attempts),
+		slog.Any("error", err),
+	)
+}
+
+func (d *OutboxDispatcher) refreshPending(ctx context.Context, now time.Time) {
+	count, err := d.repo.CountPendingEmailOutboxMessages(ctx, now)
+	if err != nil {
+		return
+	}
+	d.metrics.update(func(snapshot *OutboxMetricsSnapshot) {
+		snapshot.ApproximatePending = count
+	})
+}
+
+func hostname() string {
+	name, err := os.Hostname()
+	if err != nil || strings.TrimSpace(name) == "" {
+		return "worker"
+	}
+	return strings.TrimSpace(name)
+}

internal/app/worker/outbox_dispatcher_test.go

@@ -0,0 +1,304 @@
+package worker
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"wucher/internal/config"
+	"wucher/internal/domain/auth"
+	"wucher/internal/queue"
+)
+
+type stubOutboxRepo struct {
+	messages map[string]*auth.EmailOutboxMessage
+}
+
+func newStubOutboxRepo(messages ...*auth.EmailOutboxMessage) *stubOutboxRepo {
+	repo := &stubOutboxRepo{messages: map[string]*auth.EmailOutboxMessage{}}
+	for _, message := range messages {
+		if message == nil {
+			continue
+		}
+		cp := *message
+		cp.ID = append([]byte(nil), message.ID...)
+		cp.Body = append([]byte(nil), message.Body...)
+		cp.Attributes = append([]byte(nil), message.Attributes...)
+		repo.messages[string(cp.ID)] = &cp
+	}
+	return repo
+}
+
+func (r *stubOutboxRepo) CreateEmailOutboxMessage(_ context.Context, message *auth.EmailOutboxMessage) error {
+	if message == nil {
+		return errors.New("nil message")
+	}
+	cp := *message
+	cp.ID = append([]byte(nil), message.ID...)
+	cp.Body = append([]byte(nil), message.Body...)
+	cp.Attributes = append([]byte(nil), message.Attributes...)
+	r.messages[string(cp.ID)] = &cp
+	return nil
+}
+
+func (r *stubOutboxRepo) ClaimPendingEmailOutboxMessages(_ context.Context, limit int, now time.Time, lockTTL time.Duration, workerID string) ([]auth.EmailOutboxMessage, error) {
+	if limit <= 0 {
+		limit = 1
+	}
+	if lockTTL <= 0 {
+		lockTTL = time.Minute
+	}
+	out := make([]auth.EmailOutboxMessage, 0, limit)
+	staleBefore := now.Add(-lockTTL)
+	for _, message := range r.messages {
+		if len(out) >= limit {
+			break
+		}
+		switch {
+		case message.Status == auth.EmailOutboxStatusPending && !message.AvailableAt.After(now):
+		case message.Status == auth.EmailOutboxStatusProcessing && message.LockedAt != nil && message.LockedAt.Before(staleBefore):
+		default:
+			continue
+		}
+		message.Status = auth.EmailOutboxStatusProcessing
+		message.Attempts++
+		ts := now
+		message.LockedAt = &ts
+		message.LockedBy = workerID
+		cp := *message
+		out = append(out, cp)
+	}
+	return out, nil
+}
+
+func (r *stubOutboxRepo) MarkEmailOutboxMessagePublished(_ context.Context, id []byte, publishedAt time.Time) error {
+	message := r.messages[string(id)]
+	if message == nil {
+		return errors.New("not found")
+	}
+	message.Status = auth.EmailOutboxStatusPublished
+	message.PublishedAt = &publishedAt
+	message.LockedAt = nil
+	message.LockedBy = ""
+	message.LastError = ""
+	return nil
+}
+
+func (r *stubOutboxRepo) MarkEmailOutboxMessageRetry(_ context.Context, id []byte, availableAt time.Time, lastErr string) error {
+	message := r.messages[string(id)]
+	if message == nil {
+		return errors.New("not found")
+	}
+	message.Status = auth.EmailOutboxStatusPending
+	message.AvailableAt = availableAt
+	message.LockedAt = nil
+	message.LockedBy = ""
+	message.LastError = lastErr
+	return nil
+}
+
+func (r *stubOutboxRepo) MarkEmailOutboxMessageDead(_ context.Context, id []byte, failedAt time.Time, lastErr string) error {
+	message := r.messages[string(id)]
+	if message == nil {
+		return errors.New("not found")
+	}
+	message.Status = auth.EmailOutboxStatusDead
+	message.AvailableAt = failedAt
+	message.LockedAt = nil
+	message.LockedBy = ""
+	message.LastError = lastErr
+	return nil
+}
+
+func (r *stubOutboxRepo) CountPendingEmailOutboxMessages(_ context.Context, now time.Time) (int64, error) {
+	var count int64
+	for _, message := range r.messages {
+		if (message.Status == auth.EmailOutboxStatusPending && !message.AvailableAt.After(now)) || message.Status == auth.EmailOutboxStatusProcessing {
+			count++
+		}
+	}
+	return count, nil
+}
+
+type stubQueueProducer struct {
+	published []queue.OutboundMessage
+	err       error
+}
+
+func (p *stubQueueProducer) Backend() string {
+	return "stub"
+}
+
+func (p *stubQueueProducer) Publish(_ context.Context, message queue.OutboundMessage) error {
+	if p.err != nil {
+		return p.err
+	}
+	p.published = append(p.published, message)
+	return nil
+}
+
+func newOutboxRecord(t *testing.T, job queue.EmailJob) *auth.EmailOutboxMessage {
+	t.Helper()
+	serializer := queue.NewJSONSerializer("v1", true, queue.QueueTypeStandard, "")
+	message, err := serializer.Serialize(context.Background(), job)
+	if err != nil {
+		t.Fatalf("serialize: %v", err)
+	}
+	record, err := auth.NewEmailOutboxMessage(time.Now().UTC(), message)
+	if err != nil {
+		t.Fatalf("new outbox message: %v", err)
+	}
+	return record
+}
+
+func TestOutboxDispatcherPublishSuccess(t *testing.T) {
+	record := newOutboxRecord(t, queue.EmailJob{
+		Type:    "email_verify",
+		To:      "user@example.com",
+		Subject: "Verify",
+		Body:    "hello",
+	})
+	repo := newStubOutboxRepo(record)
+	producer := &stubQueueProducer{}
+	dispatcher := NewOutboxDispatcher(config.QueueConfig{
+		OutboxEnabled:         true,
+		OutboxBatchSize:       10,
+		OutboxDispatchTimeout: time.Second,
+		OutboxLockTTL:         time.Minute,
+		OutboxMaxAttempts:     3,
+		WorkerBackoffMin:      time.Second,
+		WorkerBackoffMax:      time.Minute,
+	}, repo, producer, nil)
+
+	processed, err := dispatcher.dispatchOnce(context.Background())
+	if err != nil {
+		t.Fatalf("dispatch once: %v", err)
+	}
+	if processed != 1 {
+		t.Fatalf("expected one processed message, got %d", processed)
+	}
+	if len(producer.published) != 1 {
+		t.Fatalf("expected one published message, got %d", len(producer.published))
+	}
+	if repo.messages[string(record.ID)].Status != auth.EmailOutboxStatusPublished {
+		t.Fatalf("expected message published, got %s", repo.messages[string(record.ID)].Status)
+	}
+	if got := dispatcher.Metrics().Published; got != 1 {
+		t.Fatalf("expected published metric=1, got %d", got)
+	}
+}
+
+func TestOutboxDispatcherSkipRegisterVerifyEmail(t *testing.T) {
+	record := newOutboxRecord(t, queue.EmailJob{
+		MessageType: "email_verify",
+		To:          "user@example.com",
+		Subject:     "Verify",
+		TextBody:    "hello",
+		Metadata: map[string]string{
+			"skip_send": "true",
+		},
+		Tags: map[string]string{
+			"source": "register",
+		},
+	})
+	repo := newStubOutboxRepo(record)
+	producer := &stubQueueProducer{}
+	dispatcher := NewOutboxDispatcher(config.QueueConfig{
+		OutboxEnabled:         true,
+		OutboxBatchSize:       10,
+		OutboxDispatchTimeout: time.Second,
+		OutboxLockTTL:         time.Minute,
+		OutboxMaxAttempts:     3,
+		WorkerBackoffMin:      time.Second,
+		WorkerBackoffMax:      time.Minute,
+	}, repo, producer, nil)
+
+	processed, err := dispatcher.dispatchOnce(context.Background())
+	if err != nil {
+		t.Fatalf("dispatch once: %v", err)
+	}
+	if processed != 1 {
+		t.Fatalf("expected one processed message, got %d", processed)
+	}
+	if len(producer.published) != 0 {
+		t.Fatalf("expected zero published message, got %d", len(producer.published))
+	}
+	if repo.messages[string(record.ID)].Status != auth.EmailOutboxStatusPublished {
+		t.Fatalf("expected message published status after skip, got %s", repo.messages[string(record.ID)].Status)
+	}
+}
+
+func TestOutboxDispatcherRetryOnPublishFailure(t *testing.T) {
+	record := newOutboxRecord(t, queue.EmailJob{
+		Type:    "password_reset",
+		To:      "user@example.com",
+		Subject: "Reset",
+		Body:    "reset",
+	})
+	repo := newStubOutboxRepo(record)
+	producer := &stubQueueProducer{err: errors.New("queue unavailable")}
+	dispatcher := NewOutboxDispatcher(config.QueueConfig{
+		OutboxEnabled:     true,
+		OutboxBatchSize:   10,
+		OutboxLockTTL:     time.Minute,
+		OutboxMaxAttempts: 3,
+		WorkerBackoffMin:  time.Second,
+		WorkerBackoffMax:  time.Minute,
+	}, repo, producer, nil)
+
+	processed, err := dispatcher.dispatchOnce(context.Background())
+	if err != nil {
+		t.Fatalf("dispatch once: %v", err)
+	}
+	if processed != 1 {
+		t.Fatalf("expected one processed message, got %d", processed)
+	}
+	got := repo.messages[string(record.ID)]
+	if got.Status != auth.EmailOutboxStatusPending {
+		t.Fatalf("expected message returned to pending, got %s", got.Status)
+	}
+	if got.Attempts != 1 {
+		t.Fatalf("expected attempts=1, got %d", got.Attempts)
+	}
+	if !got.AvailableAt.After(time.Now().Add(-time.Second)) {
+		t.Fatalf("expected retry to schedule next attempt in the future")
+	}
+	if metrics := dispatcher.Metrics(); metrics.Retried != 1 {
+		t.Fatalf("expected retried metric=1, got %d", metrics.Retried)
+	}
+}
+
+func TestOutboxDispatcherMarksDeadAfterMaxAttempts(t *testing.T) {
+	record := newOutboxRecord(t, queue.EmailJob{
+		Type:    "invite_set_password",
+		To:      "user@example.com",
+		Subject: "Invite",
+		Body:    "invite",
+	})
+	record.Attempts = 2
+	repo := newStubOutboxRepo(record)
+	producer := &stubQueueProducer{err: errors.New("queue unavailable")}
+	dispatcher := NewOutboxDispatcher(config.QueueConfig{
+		OutboxEnabled:     true,
+		OutboxBatchSize:   10,
+		OutboxLockTTL:     time.Minute,
+		OutboxMaxAttempts: 3,
+		WorkerBackoffMin:  time.Second,
+		WorkerBackoffMax:  time.Minute,
+	}, repo, producer, nil)
+
+	processed, err := dispatcher.dispatchOnce(context.Background())
+	if err != nil {
+		t.Fatalf("dispatch once: %v", err)
+	}
+	if processed != 1 {
+		t.Fatalf("expected one processed message, got %d", processed)
+	}
+	if got := repo.messages[string(record.ID)].Status; got != auth.EmailOutboxStatusDead {
+		t.Fatalf("expected dead status, got %s", got)
+	}
+	if got := dispatcher.Metrics().Dead; got != 1 {
+		t.Fatalf("expected dead metric=1, got %d", got)
+	}
+}

internal/app/worker/runner.go

@@ -0,0 +1,196 @@
+package worker
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"log/slog"
+	"net/http"
+	"time"
+
+	"wucher/internal/config"
+	"wucher/internal/shared/pkg/metrics"
+	"wucher/internal/queue"
+)
+
+type Runner struct {
+	worker            *queue.Worker
+	consumer          queue.QueueConsumer
+	dispatcher        *OutboxDispatcher
+	logger            *slog.Logger
+	monitorAddr       string
+	depthPollInterval time.Duration
+	shutdownTimeout   time.Duration
+	backend           string
+	monitorServer     *http.Server
+}
+
+func NewRunner(
+	cfg config.QueueConfig,
+	consumer queue.QueueConsumer,
+	serializer queue.MessageSerializer,
+	handler queue.EmailJobHandler,
+	idempotency queue.IdempotencyStore,
+	dispatcher *OutboxDispatcher,
+	logger *slog.Logger,
+) *Runner {
+	if logger == nil {
+		logger = slog.Default()
+	}
+	metrics := queue.NewMetrics()
+	health := queue.NewHealthReporter(cfg.WorkerHealthErrorThreshold)
+	worker := queue.NewWorker(queue.WorkerConfig{
+		Concurrency:      cfg.WorkerConcurrency,
+		MaxBatchSize:     cfg.WorkerMaxBatchSize,
+		ProcessTimeout:   cfg.WorkerProcessTimeout,
+		ProcessingTTL:    cfg.ProcessingTTL,
+		IdempotencyTTL:   cfg.IdempotencyTTL,
+		PollErrorDelay:   cfg.WorkerBackoffMin,
+		MetricsComponent: "email_worker",
+	}, consumer, serializer, handler, queue.NewExponentialBackoffPolicy(cfg.WorkerBackoffMin, cfg.WorkerBackoffMax, cfg.WorkerPermanentFailureDelay), idempotency, metrics, health, logger)
+
+	backend := ""
+	if consumer != nil {
+		backend = consumer.Backend()
+	}
+	return &Runner{
+		worker:            worker,
+		consumer:          consumer,
+		dispatcher:        dispatcher,
+		logger:            logger,
+		monitorAddr:       cfg.WorkerMonitorAddr,
+		depthPollInterval: cfg.WorkerDepthPollInterval,
+		shutdownTimeout:   cfg.WorkerShutdownTimeout,
+		backend:           backend,
+	}
+}
+
+func (r *Runner) Start(ctx context.Context) error {
+	if r.worker == nil {
+		return errors.New("worker is required")
+	}
+	runCtx, cancelRun := context.WithCancel(ctx)
+	defer cancelRun()
+	if err := r.startMonitor(); err != nil {
+		return err
+	}
+	pollerDone := make(chan struct{})
+	go func() {
+		defer close(pollerDone)
+		r.runDepthPoller(runCtx)
+	}()
+
+	workerErrCh := make(chan error, 1)
+	go func() {
+		workerErrCh <- r.worker.Run(runCtx)
+	}()
+
+	var dispatcherErrCh chan error
+	if r.dispatcher != nil {
+		dispatcherErrCh = make(chan error, 1)
+		go func() {
+			dispatcherErrCh <- r.dispatcher.Run(runCtx)
+		}()
+	}
+
+	var err error
+	select {
+	case err = <-workerErrCh:
+		cancelRun()
+		if dispatcherErrCh != nil {
+			if dispatcherErr := <-dispatcherErrCh; err == nil {
+				err = dispatcherErr
+			}
+		}
+	case err = <-dispatcherErrCh:
+		cancelRun()
+		if workerErr := <-workerErrCh; err == nil {
+			err = workerErr
+		}
+	}
+
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), r.shutdownTimeout)
+	defer cancel()
+	if r.monitorServer != nil {
+		_ = r.monitorServer.Shutdown(shutdownCtx)
+	}
+	if r.consumer != nil {
+		_ = r.consumer.Close()
+	}
+	<-pollerDone
+	return err
+}
+
+func (r *Runner) startMonitor() error {
+	if r.monitorAddr == "" {
+		return nil
+	}
+	mux := http.NewServeMux()
+	mux.HandleFunc("/health", r.handleHealth)
+	mux.Handle("/metrics", metrics.PromHTTPHandler())
+	r.monitorServer = &http.Server{
+		Addr:              r.monitorAddr,
+		Handler:           mux,
+		ReadHeaderTimeout: 5 * time.Second,
+	}
+	go func() {
+		r.logger.Info("worker monitor listening",
+			slog.String("addr", r.monitorAddr),
+			slog.String("backend", r.backend),
+		)
+		if err := r.monitorServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			r.logger.Error("worker monitor failed", slog.Any("error", err))
+		}
+	}()
+	return nil
+}
+
+func (r *Runner) runDepthPoller(ctx context.Context) {
+	if r.depthPollInterval <= 0 {
+		return
+	}
+	ticker := time.NewTicker(r.depthPollInterval)
+	defer ticker.Stop()
+	for {
+		refreshCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		_ = r.worker.RefreshQueueDepth(refreshCtx)
+		cancel()
+
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		}
+	}
+}
+
+func (r *Runner) handleHealth(w http.ResponseWriter, _ *http.Request) {
+	snapshot := struct {
+		Backend string                 `json:"backend"`
+		Health  queue.HealthSnapshot   `json:"health"`
+		Metrics queue.MetricsSnapshot  `json:"metrics"`
+		Outbox  *OutboxMetricsSnapshot `json:"outbox,omitempty"`
+	}{
+		Backend: r.backend,
+		Health:  r.worker.Health().Snapshot(),
+		Metrics: r.worker.Metrics().Snapshot(),
+	}
+	if r.dispatcher != nil {
+		snapshot.Outbox = ptr(r.dispatcher.Metrics())
+	}
+	statusCode := http.StatusOK
+	if snapshot.Health.Status == "degraded" {
+		statusCode = http.StatusServiceUnavailable
+	}
+	writeJSON(w, statusCode, snapshot)
+}
+
+func writeJSON(w http.ResponseWriter, statusCode int, payload any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(statusCode)
+	_ = json.NewEncoder(w).Encode(payload)
+}
+
+func ptr[T any](v T) *T {
+	return &v
+}