init push

This commit is contained in:
2026-07-16 22:16:45 +07:00
commit 8b068bdb10
1021 changed files with 332816 additions and 0 deletions

45
internal/queue/context.go Normal file
View File

@@ -0,0 +1,45 @@
package queue
import (
"context"
"strings"
)
type contextKey string
const (
correlationIDKey contextKey = "queue-correlation-id"
traceIDKey contextKey = "queue-trace-id"
)
func WithCorrelationID(ctx context.Context, correlationID string) context.Context {
correlationID = strings.TrimSpace(correlationID)
if correlationID == "" {
return ctx
}
return context.WithValue(ctx, correlationIDKey, correlationID)
}
func CorrelationIDFromContext(ctx context.Context) string {
if ctx == nil {
return ""
}
value, _ := ctx.Value(correlationIDKey).(string)
return strings.TrimSpace(value)
}
func WithTraceID(ctx context.Context, traceID string) context.Context {
traceID = strings.TrimSpace(traceID)
if traceID == "" {
return ctx
}
return context.WithValue(ctx, traceIDKey, traceID)
}
func TraceIDFromContext(ctx context.Context) string {
if ctx == nil {
return ""
}
value, _ := ctx.Value(traceIDKey).(string)
return strings.TrimSpace(value)
}

145
internal/queue/enqueuer.go Normal file
View File

@@ -0,0 +1,145 @@
package queue
import (
"context"
"errors"
"log/slog"
)
type ProducerMetrics struct {
published int64
publishFailures int64
secondaryFailures int64
}
type EmailEnqueuer struct {
serializer MessageSerializer
producer QueueProducer
logger *slog.Logger
}
func NewEmailEnqueuer(serializer MessageSerializer, producer QueueProducer, logger *slog.Logger) *EmailEnqueuer {
if logger == nil {
logger = slog.Default()
}
return &EmailEnqueuer{
serializer: serializer,
producer: producer,
logger: logger,
}
}
func (e *EmailEnqueuer) Enqueue(ctx context.Context, job EmailJob) error {
message, err := e.serializer.Serialize(ctx, job)
if err != nil {
return err
}
if err := e.producer.Publish(ctx, *message); err != nil {
e.logger.Error("queue publish failed",
slog.String("backend", e.producer.Backend()),
slog.String("message_id", message.ID),
slog.String("job_type", job.NormalizedType()),
slog.String("recipient_hash", job.RecipientHash()),
slog.String("correlation_id", message.Attributes["correlation_id"]),
slog.Any("error", err),
)
return err
}
e.logger.Info("queue published",
slog.String("backend", e.producer.Backend()),
slog.String("message_id", message.ID),
slog.String("job_type", job.NormalizedType()),
slog.String("recipient_hash", job.RecipientHash()),
slog.String("correlation_id", message.Attributes["correlation_id"]),
)
return nil
}
type DualProducer struct {
primary QueueProducer
secondary QueueProducer
secondaryRequired bool
logger *slog.Logger
}
func NewDualProducer(primary, secondary QueueProducer, secondaryRequired bool, logger *slog.Logger) *DualProducer {
if logger == nil {
logger = slog.Default()
}
return &DualProducer{
primary: primary,
secondary: secondary,
secondaryRequired: secondaryRequired,
logger: logger,
}
}
func (p *DualProducer) Backend() string {
return "dual"
}
func (p *DualProducer) Publish(ctx context.Context, message OutboundMessage) error {
if p.primary == nil {
return errors.New("primary producer is required")
}
if err := p.primary.Publish(ctx, message); err != nil {
return err
}
if p.secondary == nil {
return nil
}
if err := p.secondary.Publish(ctx, message); err != nil {
p.logger.Error("queue secondary publish failed",
slog.String("primary_backend", p.primary.Backend()),
slog.String("secondary_backend", p.secondary.Backend()),
slog.String("message_id", message.ID),
slog.Any("error", err),
)
if p.secondaryRequired {
return err
}
}
return nil
}
type DualEmailJobProducer struct {
primary EmailJobProducer
secondary EmailJobProducer
secondaryRequired bool
logger *slog.Logger
}
func NewDualEmailJobProducer(primary, secondary EmailJobProducer, secondaryRequired bool, logger *slog.Logger) *DualEmailJobProducer {
if logger == nil {
logger = slog.Default()
}
return &DualEmailJobProducer{
primary: primary,
secondary: secondary,
secondaryRequired: secondaryRequired,
logger: logger,
}
}
func (p *DualEmailJobProducer) Enqueue(ctx context.Context, job EmailJob) error {
if p.primary == nil {
return errors.New("primary email producer is required")
}
if err := p.primary.Enqueue(ctx, job); err != nil {
return err
}
if p.secondary == nil {
return nil
}
if err := p.secondary.Enqueue(ctx, job); err != nil {
p.logger.Error("queue secondary enqueue failed",
slog.String("job_type", job.NormalizedType()),
slog.String("recipient_hash", job.RecipientHash()),
slog.Any("error", err),
)
if p.secondaryRequired {
return err
}
}
return nil
}

51
internal/queue/errors.go Normal file
View File

@@ -0,0 +1,51 @@
package queue
import "errors"
type permanentError struct {
err error
}
func (e permanentError) Error() string {
return e.err.Error()
}
func (e permanentError) Unwrap() error {
return e.err
}
type transientError struct {
err error
}
func (e transientError) Error() string {
return e.err.Error()
}
func (e transientError) Unwrap() error {
return e.err
}
func Permanent(err error) error {
if err == nil {
return nil
}
return permanentError{err: err}
}
func Transient(err error) error {
if err == nil {
return nil
}
return transientError{err: err}
}
func IsPermanent(err error) bool {
var target permanentError
return errors.As(err, &target)
}
func IsTransient(err error) bool {
var target transientError
return errors.As(err, &target)
}

View File

@@ -0,0 +1,47 @@
package queue
import (
"context"
"log/slog"
)
type FileProcessingEnqueuer struct {
serializer FileProcessingMessageSerializer
producer QueueProducer
logger *slog.Logger
}
func NewFileProcessingEnqueuer(serializer FileProcessingMessageSerializer, producer QueueProducer, logger *slog.Logger) *FileProcessingEnqueuer {
if logger == nil {
logger = slog.Default()
}
return &FileProcessingEnqueuer{
serializer: serializer,
producer: producer,
logger: logger,
}
}
func (e *FileProcessingEnqueuer) Enqueue(ctx context.Context, job FileProcessingJob) error {
message, err := e.serializer.Serialize(ctx, job)
if err != nil {
return err
}
if err := e.producer.Publish(ctx, *message); err != nil {
e.logger.Error("file queue publish failed",
slog.String("backend", e.producer.Backend()),
slog.String("message_id", message.ID),
slog.String("file_uuid", job.Canonical().FileUUID),
slog.String("correlation_id", message.Attributes["correlation_id"]),
slog.Any("error", err),
)
return err
}
e.logger.Info("file queue published",
slog.String("backend", e.producer.Backend()),
slog.String("message_id", message.ID),
slog.String("file_uuid", job.Canonical().FileUUID),
slog.String("correlation_id", message.Attributes["correlation_id"]),
)
return nil
}

View File

@@ -0,0 +1,62 @@
package queue
import (
"context"
"errors"
filemanager "wucher/internal/domain/file_manager"
)
type FileProcessingJobProcessor interface {
Process(ctx context.Context, job filemanager.FileProcessingJob, correlationID string) error
}
type FileProcessingPermanentClassifier func(error) bool
type FileProcessingJobQueueHandler struct {
processor FileProcessingJobProcessor
isPermanent FileProcessingPermanentClassifier
}
func NewFileProcessingJobQueueHandler(
processor FileProcessingJobProcessor,
isPermanent FileProcessingPermanentClassifier,
) *FileProcessingJobQueueHandler {
return &FileProcessingJobQueueHandler{
processor: processor,
isPermanent: isPermanent,
}
}
func (h *FileProcessingJobQueueHandler) Handle(ctx context.Context, envelope *FileProcessingEnvelope) error {
if envelope == nil {
return Permanent(errors.New("file processing envelope is required"))
}
if h.processor == nil {
return Permanent(errors.New("file processing processor is required"))
}
job := envelope.Payload.Canonical()
if err := job.Validate(); err != nil {
return Permanent(err)
}
err := h.processor.Process(ctx, filemanager.FileProcessingJob{
JobID: job.JobID,
FileUUID: job.FileUUID,
Bucket: job.Bucket,
ObjectKey: job.ObjectKey,
UploadedBy: job.UploadedBy,
MimeType: job.MimeType,
SizeBytes: job.SizeBytes,
TraceID: job.TraceID,
RequestID: job.RequestID,
}, envelope.CorrelationID)
if err == nil {
return nil
}
if h.isPermanent != nil && h.isPermanent(err) {
return Permanent(err)
}
return Transient(err)
}

View File

@@ -0,0 +1,122 @@
package queue
import (
"context"
"errors"
"testing"
filemanager "wucher/internal/domain/file_manager"
"wucher/internal/shared/pkg/uuidv7"
)
type stubFileProcessingProcessor struct {
err error
}
func (p stubFileProcessingProcessor) Process(context.Context, filemanager.FileProcessingJob, string) error {
return p.err
}
func TestFileProcessingJobQueueHandlerHandleSuccess(t *testing.T) {
fileUUID, err := uuidv7.String(uuidv7.MustNew())
if err != nil {
t.Fatalf("uuid string: %v", err)
}
handler := NewFileProcessingJobQueueHandler(stubFileProcessingProcessor{}, nil)
err = handler.Handle(context.Background(), &FileProcessingEnvelope{
MessageID: "msg-1",
Payload: FileProcessingJob{
FileUUID: fileUUID,
Bucket: "bucket-a",
ObjectKey: "files/a.pdf",
SizeBytes: 12,
},
})
if err != nil {
t.Fatalf("handle: %v", err)
}
}
func TestFileProcessingJobQueueHandlerHandleNilEnvelopePermanent(t *testing.T) {
handler := NewFileProcessingJobQueueHandler(stubFileProcessingProcessor{}, nil)
err := handler.Handle(context.Background(), nil)
if err == nil {
t.Fatalf("expected error")
}
if !IsPermanent(err) {
t.Fatalf("expected permanent error")
}
}
func TestFileProcessingJobQueueHandlerHandleInvalidPayloadPermanent(t *testing.T) {
handler := NewFileProcessingJobQueueHandler(stubFileProcessingProcessor{}, nil)
err := handler.Handle(context.Background(), &FileProcessingEnvelope{
MessageID: "msg-2",
Payload: FileProcessingJob{
FileUUID: "not-a-uuid",
Bucket: "bucket-a",
},
})
if err == nil {
t.Fatalf("expected error")
}
if !IsPermanent(err) {
t.Fatalf("expected permanent error")
}
}
func TestFileProcessingJobQueueHandlerHandleTransientError(t *testing.T) {
fileUUID, err := uuidv7.String(uuidv7.MustNew())
if err != nil {
t.Fatalf("uuid string: %v", err)
}
handler := NewFileProcessingJobQueueHandler(stubFileProcessingProcessor{
err: errors.New("temporary timeout"),
}, nil)
err = handler.Handle(context.Background(), &FileProcessingEnvelope{
MessageID: "msg-3",
Payload: FileProcessingJob{
FileUUID: fileUUID,
Bucket: "bucket-a",
ObjectKey: "files/a.pdf",
SizeBytes: 12,
},
})
if err == nil {
t.Fatalf("expected error")
}
if !IsTransient(err) {
t.Fatalf("expected transient error")
}
}
func TestFileProcessingJobQueueHandlerHandlePermanentClassification(t *testing.T) {
fileUUID, err := uuidv7.String(uuidv7.MustNew())
if err != nil {
t.Fatalf("uuid string: %v", err)
}
permanentErr := errors.New("not found")
handler := NewFileProcessingJobQueueHandler(stubFileProcessingProcessor{
err: permanentErr,
}, func(err error) bool {
return errors.Is(err, permanentErr)
})
err = handler.Handle(context.Background(), &FileProcessingEnvelope{
MessageID: "msg-4",
Payload: FileProcessingJob{
FileUUID: fileUUID,
Bucket: "bucket-a",
ObjectKey: "files/a.pdf",
SizeBytes: 12,
},
})
if err == nil {
t.Fatalf("expected error")
}
if !IsPermanent(err) {
t.Fatalf("expected permanent error")
}
}

View File

@@ -0,0 +1,112 @@
package queue
import (
"context"
"encoding/json"
"errors"
"strings"
"time"
)
type FileProcessingJSONSerializer struct {
version string
queueType string
messageGroupID string
}
func NewFileProcessingJSONSerializer(version, queueType, messageGroupID string) *FileProcessingJSONSerializer {
version = strings.TrimSpace(version)
if version == "" {
version = "v1"
}
queueType = strings.ToLower(strings.TrimSpace(queueType))
if queueType == "" {
queueType = QueueTypeStandard
}
return &FileProcessingJSONSerializer{
version: version,
queueType: queueType,
messageGroupID: strings.TrimSpace(messageGroupID),
}
}
func (s *FileProcessingJSONSerializer) Serialize(ctx context.Context, job FileProcessingJob) (*OutboundMessage, error) {
envelope, err := NewFileProcessingEnvelope(ctx, s.version, job)
if err != nil {
return nil, err
}
body, err := json.Marshal(envelope)
if err != nil {
return nil, err
}
message := &OutboundMessage{
ID: envelope.MessageID,
Body: body,
Attributes: map[string]string{
"message_kind": envelope.Kind,
"message_id": envelope.MessageID,
"correlation_id": envelope.CorrelationID,
"trace_id": envelope.TraceID,
"schema_version": envelope.Version,
"job_type": "file_processing",
"file_uuid": envelope.Payload.FileUUID,
},
}
if s.queueType == QueueTypeFIFO {
message.MessageGroupID = s.messageGroupID
if message.MessageGroupID == "" {
message.MessageGroupID = "file-processing"
}
message.DeduplicationID = envelope.DeduplicationKey()
}
return message, nil
}
func (s *FileProcessingJSONSerializer) Deserialize(body []byte, _ map[string]string) (*FileProcessingEnvelope, error) {
var envelope FileProcessingEnvelope
if err := json.Unmarshal(body, &envelope); err != nil {
return nil, errors.Join(ErrInvalidFileProcessingEnvelope, err)
}
if err := validateFileProcessingEnvelope(&envelope); err != nil {
return nil, err
}
return &envelope, nil
}
func validateFileProcessingEnvelope(envelope *FileProcessingEnvelope) error {
if envelope == nil {
return ErrInvalidFileProcessingEnvelope
}
if strings.TrimSpace(envelope.Version) == "" {
return errors.Join(ErrInvalidFileProcessingEnvelope, errors.New("version is required"))
}
if strings.TrimSpace(envelope.MessageID) == "" {
return errors.Join(ErrInvalidFileProcessingEnvelope, errors.New("message id is required"))
}
if strings.TrimSpace(envelope.Kind) == "" {
return errors.Join(ErrInvalidFileProcessingEnvelope, errors.New("kind is required"))
}
if !strings.EqualFold(strings.TrimSpace(envelope.Kind), FileProcessingKind) {
return errors.Join(ErrInvalidFileProcessingEnvelope, errors.New("unsupported message kind"))
}
if envelope.OccurredAt.IsZero() {
return errors.Join(ErrInvalidFileProcessingEnvelope, errors.New("occurred at is required"))
}
envelope.Payload = envelope.Payload.Canonical()
if err := envelope.Payload.Validate(); err != nil {
return errors.Join(ErrInvalidFileProcessingEnvelope, err)
}
if strings.TrimSpace(envelope.CorrelationID) == "" {
envelope.CorrelationID = envelope.MessageID
}
if strings.TrimSpace(envelope.TraceID) == "" {
envelope.TraceID = envelope.CorrelationID
}
if strings.TrimSpace(envelope.IdempotencyKey) == "" {
envelope.IdempotencyKey = envelope.MessageID
}
if envelope.Payload.CreatedAt.IsZero() {
envelope.Payload.CreatedAt = time.Now().UTC()
}
return nil
}

View File

@@ -0,0 +1,53 @@
package queue
import (
"context"
"testing"
"wucher/internal/shared/pkg/uuidv7"
)
func TestFileProcessingJSONSerializerRoundTrip(t *testing.T) {
serializer := NewFileProcessingJSONSerializer("v1", QueueTypeStandard, "")
fileUUID, err := uuidv7.String(uuidv7.MustNew())
if err != nil {
t.Fatalf("uuid string: %v", err)
}
ctx := WithCorrelationID(context.Background(), "upload-req-1")
message, err := serializer.Serialize(ctx, FileProcessingJob{
FileUUID: fileUUID,
Bucket: "wucher-file-dev",
ObjectKey: "files/2026/04/03/a.pdf",
MimeType: "application/pdf",
SizeBytes: 1234,
})
if err != nil {
t.Fatalf("serialize: %v", err)
}
envelope, err := serializer.Deserialize(message.Body, message.Attributes)
if err != nil {
t.Fatalf("deserialize: %v", err)
}
if envelope.MessageID == "" {
t.Fatalf("expected message id")
}
if envelope.CorrelationID != "upload-req-1" {
t.Fatalf("unexpected correlation id: %q", envelope.CorrelationID)
}
if envelope.Payload.FileUUID != fileUUID {
t.Fatalf("unexpected file uuid: %q", envelope.Payload.FileUUID)
}
if envelope.Payload.ObjectKey != "files/2026/04/03/a.pdf" {
t.Fatalf("unexpected object key: %q", envelope.Payload.ObjectKey)
}
}
func TestFileProcessingJSONSerializerRejectsInvalidPayload(t *testing.T) {
serializer := NewFileProcessingJSONSerializer("v1", QueueTypeStandard, "")
_, err := serializer.Deserialize([]byte(`{"version":"v1","message_id":"m1","kind":"file_manager.processing","occurred_at":"2026-04-03T00:00:00Z","payload":{"bucket":"b","object_key":"k","size_bytes":1}}`), nil)
if err == nil {
t.Fatalf("expected invalid payload error")
}
}

View File

@@ -0,0 +1,141 @@
package queue
import (
"context"
"errors"
"strings"
"time"
"wucher/internal/shared/pkg/uuidv7"
)
const (
FileProcessingKind = "file_manager.processing"
)
var (
ErrInvalidFileProcessingJob = errors.New("invalid file processing job")
ErrInvalidFileProcessingEnvelope = errors.New("invalid file processing envelope")
)
type FileProcessingJob struct {
JobID string `json:"job_id,omitempty"`
FileUUID string `json:"file_uuid"`
Bucket string `json:"bucket"`
ObjectKey string `json:"object_key"`
UploadedBy string `json:"uploaded_by,omitempty"`
MimeType string `json:"mime_type,omitempty"`
SizeBytes int64 `json:"size_bytes"`
CreatedAt time.Time `json:"created_at,omitempty"`
TraceID string `json:"trace_id,omitempty"`
RequestID string `json:"request_id,omitempty"`
}
func (j FileProcessingJob) Canonical() FileProcessingJob {
job := j
job.JobID = strings.TrimSpace(job.JobID)
job.FileUUID = strings.TrimSpace(job.FileUUID)
job.Bucket = strings.TrimSpace(job.Bucket)
job.ObjectKey = strings.TrimSpace(job.ObjectKey)
job.UploadedBy = strings.TrimSpace(job.UploadedBy)
job.MimeType = strings.ToLower(strings.TrimSpace(job.MimeType))
job.TraceID = strings.TrimSpace(job.TraceID)
job.RequestID = strings.TrimSpace(job.RequestID)
return job
}
func (j FileProcessingJob) Validate() error {
job := j.Canonical()
if job.FileUUID == "" {
return errors.Join(ErrInvalidFileProcessingJob, errors.New("file uuid is required"))
}
if _, err := uuidv7.ParseString(job.FileUUID); err != nil {
return errors.Join(ErrInvalidFileProcessingJob, errors.New("file uuid is invalid"))
}
if job.Bucket == "" {
return errors.Join(ErrInvalidFileProcessingJob, errors.New("bucket is required"))
}
if job.ObjectKey == "" {
return errors.Join(ErrInvalidFileProcessingJob, errors.New("object key is required"))
}
if job.SizeBytes < 0 {
return errors.Join(ErrInvalidFileProcessingJob, errors.New("size bytes must be non-negative"))
}
return nil
}
type FileProcessingEnvelope struct {
Version string `json:"version"`
MessageID string `json:"message_id"`
Kind string `json:"kind"`
OccurredAt time.Time `json:"occurred_at"`
CorrelationID string `json:"correlation_id,omitempty"`
TraceID string `json:"trace_id,omitempty"`
IdempotencyKey string `json:"idempotency_key,omitempty"`
Payload FileProcessingJob `json:"payload"`
}
func NewFileProcessingEnvelope(ctx context.Context, version string, job FileProcessingJob) (*FileProcessingEnvelope, error) {
job = job.Canonical()
if err := job.Validate(); err != nil {
return nil, err
}
messageID, err := newMessageID()
if err != nil {
return nil, err
}
correlationID := CorrelationIDFromContext(ctx)
if correlationID == "" {
correlationID = messageID
}
traceID := TraceIDFromContext(ctx)
if traceID == "" {
traceID = correlationID
}
now := time.Now().UTC()
if strings.TrimSpace(version) == "" {
version = "v1"
}
if job.JobID == "" {
job.JobID = messageID
}
if job.CreatedAt.IsZero() {
job.CreatedAt = now
}
if job.TraceID == "" {
job.TraceID = traceID
}
if job.RequestID == "" {
job.RequestID = correlationID
}
return &FileProcessingEnvelope{
Version: version,
MessageID: messageID,
Kind: FileProcessingKind,
OccurredAt: now,
CorrelationID: correlationID,
TraceID: traceID,
IdempotencyKey: job.JobID,
Payload: job,
}, nil
}
func (e FileProcessingEnvelope) DeduplicationKey() string {
if v := strings.TrimSpace(e.IdempotencyKey); v != "" {
return v
}
return strings.TrimSpace(e.MessageID)
}
type FileProcessingMessageSerializer interface {
Serialize(ctx context.Context, job FileProcessingJob) (*OutboundMessage, error)
Deserialize(body []byte, attributes map[string]string) (*FileProcessingEnvelope, error)
}
type FileProcessingJobProducer interface {
Enqueue(ctx context.Context, job FileProcessingJob) error
}
type FileProcessingJobHandler interface {
Handle(ctx context.Context, envelope *FileProcessingEnvelope) error
}

View File

@@ -0,0 +1,299 @@
package queue
import (
"context"
"errors"
"log/slog"
"time"
"wucher/internal/shared/pkg/metrics"
)
type FileProcessingWorker struct {
cfg WorkerConfig
consumer QueueConsumer
serializer FileProcessingMessageSerializer
handler FileProcessingJobHandler
retry RetryPolicy
idem IdempotencyStore
metrics *Metrics
health *HealthReporter
logger *slog.Logger
component string
}
type fileProcessResult struct {
delivery *Delivery
envelope *FileProcessingEnvelope
decision RetryDecision
err error
}
func NewFileProcessingWorker(
cfg WorkerConfig,
consumer QueueConsumer,
serializer FileProcessingMessageSerializer,
handler FileProcessingJobHandler,
retry RetryPolicy,
idem IdempotencyStore,
metrics *Metrics,
health *HealthReporter,
logger *slog.Logger,
) *FileProcessingWorker {
if cfg.Concurrency <= 0 {
cfg.Concurrency = 1
}
if cfg.MaxBatchSize <= 0 || cfg.MaxBatchSize > cfg.Concurrency {
cfg.MaxBatchSize = cfg.Concurrency
}
if cfg.ProcessTimeout <= 0 {
cfg.ProcessTimeout = 30 * time.Second
}
if cfg.ProcessingTTL <= 0 {
cfg.ProcessingTTL = 15 * time.Minute
}
if cfg.IdempotencyTTL <= 0 {
cfg.IdempotencyTTL = 24 * time.Hour
}
if cfg.PollErrorDelay <= 0 {
cfg.PollErrorDelay = time.Second
}
if retry == nil {
retry = NewExponentialBackoffPolicy(time.Second, 5*time.Minute, 30*time.Second)
}
if idem == nil {
idem = NoopIdempotencyStore{}
}
if metrics == nil {
metrics = NewMetrics()
}
if health == nil {
health = NewHealthReporter(3)
}
if logger == nil {
logger = slog.Default()
}
return &FileProcessingWorker{
cfg: cfg,
consumer: consumer,
serializer: serializer,
handler: handler,
retry: retry,
idem: idem,
metrics: metrics,
health: health,
logger: logger,
component: normalizeWorkerMetricsComponent(cfg.MetricsComponent),
}
}
func (w *FileProcessingWorker) Run(ctx context.Context) error {
if w.consumer == nil {
return errors.New("queue consumer is required")
}
if w.serializer == nil {
return errors.New("file message serializer is required")
}
if w.handler == nil {
return errors.New("file message handler is required")
}
for {
select {
case <-ctx.Done():
w.health.MarkDraining()
return nil
default:
}
deliveries, err := w.consumer.Receive(ctx, w.cfg.MaxBatchSize)
if err != nil {
if ctx.Err() != nil {
w.health.MarkDraining()
return nil
}
w.health.MarkPollFailure(err)
w.logger.Error("file queue receive failed",
slog.String("backend", w.consumer.Backend()),
slog.Any("error", err),
)
select {
case <-ctx.Done():
w.health.MarkDraining()
return nil
case <-time.After(w.cfg.PollErrorDelay):
}
continue
}
w.health.MarkPollSuccess()
if len(deliveries) == 0 {
continue
}
w.metrics.RecordReceive(len(deliveries))
results := make(chan fileProcessResult, len(deliveries))
for _, delivery := range deliveries {
delivery := delivery
go func() {
results <- w.processDelivery(ctx, delivery)
}()
}
acks := make([]*Delivery, 0, len(deliveries))
for range deliveries {
result := <-results
switch result.decision.Action {
case RetryActionAck:
acks = append(acks, result.delivery)
case RetryActionRetry:
if err := w.consumer.Retry(ctx, result.delivery, result.decision.Delay); err != nil {
w.metrics.RecordRetryFailure()
w.logger.Error("file queue retry scheduling failed",
slog.String("backend", w.consumer.Backend()),
slog.String("message_id", result.delivery.ID),
slog.Any("error", err),
)
} else {
w.metrics.RecordRetry()
}
case RetryActionDeadLetter:
if err := w.consumer.DeadLetter(ctx, result.delivery, result.decision.Reason, result.err); err != nil {
w.metrics.RecordDeadLetterFailure()
w.logger.Error("file queue dead-letter failed",
slog.String("backend", w.consumer.Backend()),
slog.String("message_id", result.delivery.ID),
slog.Any("error", err),
)
} else {
w.metrics.RecordDeadLetter()
}
}
}
if len(acks) > 0 {
if err := w.consumer.Ack(ctx, acks); err != nil {
w.metrics.RecordAckFailure()
w.logger.Error("file queue ack failed",
slog.String("backend", w.consumer.Backend()),
slog.Int("count", len(acks)),
slog.Any("error", err),
)
}
}
}
}
func (w *FileProcessingWorker) RefreshQueueDepth(ctx context.Context) error {
if w.consumer == nil {
return errors.New("queue consumer is required")
}
depth, err := w.consumer.ApproximateDepth(ctx)
if err != nil {
return err
}
w.metrics.SetQueueDepth(depth)
return nil
}
func (w *FileProcessingWorker) Metrics() *Metrics {
return w.metrics
}
func (w *FileProcessingWorker) Health() *HealthReporter {
return w.health
}
func (w *FileProcessingWorker) processDelivery(ctx context.Context, delivery *Delivery) (result fileProcessResult) {
started := time.Now()
var jobErr error
defer func() {
metrics.ObserveWorkerJob(w.component, "file_processing", jobErr, started)
}()
envelope, err := w.serializer.Deserialize(delivery.Body, delivery.Attributes)
if err != nil {
w.metrics.RecordMalformed()
w.metrics.RecordFailure()
jobErr = err
return fileProcessResult{
delivery: delivery,
decision: w.retry.Decide(delivery, nil, Permanent(err)),
err: err,
}
}
idempotencyKey := envelope.DeduplicationKey()
state, err := w.idem.Acquire(ctx, idempotencyKey, w.cfg.ProcessingTTL)
if err != nil {
w.metrics.RecordFailure()
jobErr = err
return fileProcessResult{
delivery: delivery,
envelope: envelope,
decision: w.retry.Decide(delivery, nil, Transient(err)),
err: err,
}
}
switch state {
case IdempotencyStateDuplicate:
w.metrics.RecordDuplicate()
w.logger.Info("file queue duplicate skipped",
slog.String("message_id", envelope.MessageID),
slog.String("file_uuid", envelope.Payload.FileUUID),
slog.String("correlation_id", envelope.CorrelationID),
)
return fileProcessResult{
delivery: delivery,
envelope: envelope,
decision: RetryDecision{Action: RetryActionAck},
}
case IdempotencyStateInProgress:
inProgressErr := Transient(errors.New("message already in progress"))
jobErr = inProgressErr
return fileProcessResult{
delivery: delivery,
envelope: envelope,
decision: w.retry.Decide(delivery, nil, inProgressErr),
err: inProgressErr,
}
}
processCtx, cancel := context.WithTimeout(ctx, w.cfg.ProcessTimeout)
defer cancel()
if err := w.handler.Handle(processCtx, envelope); err != nil {
_ = w.idem.Release(ctx, idempotencyKey)
w.metrics.RecordFailure()
jobErr = err
return fileProcessResult{
delivery: delivery,
envelope: envelope,
decision: w.retry.Decide(delivery, nil, err),
err: err,
}
}
if err := w.idem.MarkCompleted(ctx, idempotencyKey, w.cfg.IdempotencyTTL); err != nil {
w.metrics.RecordFailure()
w.logger.Error("file queue idempotency completion failed",
slog.String("message_id", envelope.MessageID),
slog.String("file_uuid", envelope.Payload.FileUUID),
slog.String("correlation_id", envelope.CorrelationID),
slog.Any("error", err),
)
jobErr = err
return fileProcessResult{
delivery: delivery,
envelope: envelope,
decision: w.retry.Decide(delivery, nil, Transient(err)),
err: err,
}
}
w.metrics.RecordProcessed(time.Since(started))
jobErr = nil
return fileProcessResult{
delivery: delivery,
envelope: envelope,
decision: RetryDecision{Action: RetryActionAck},
}
}

View File

@@ -0,0 +1,74 @@
package queue
import (
"context"
"errors"
"testing"
"wucher/internal/config"
"wucher/internal/shared/pkg/logger"
"wucher/internal/shared/pkg/uuidv7"
)
type stubFileProcessingHandler struct {
err error
}
func (h stubFileProcessingHandler) Handle(context.Context, *FileProcessingEnvelope) error {
return h.err
}
func TestFileProcessingWorkerProcessDeliveryDuplicate(t *testing.T) {
worker := NewFileProcessingWorker(WorkerConfig{}, nil, NewFileProcessingJSONSerializer("v1", QueueTypeStandard, ""), stubFileProcessingHandler{}, nil, stubIdempotencyStore{state: IdempotencyStateDuplicate}, nil, nil, logger.New(config.LoggingConfig{Format: "json"}))
fileUUID, _ := uuidv7.String(uuidv7.MustNew())
message, err := NewFileProcessingJSONSerializer("v1", QueueTypeStandard, "").Serialize(context.Background(), FileProcessingJob{
FileUUID: fileUUID,
Bucket: "bucket",
ObjectKey: "files/doc.pdf",
SizeBytes: 10,
})
if err != nil {
t.Fatalf("serialize: %v", err)
}
result := worker.processDelivery(context.Background(), &Delivery{ID: message.ID, Body: message.Body, Attributes: message.Attributes, ReceiveCount: 1})
if result.decision.Action != RetryActionAck {
t.Fatalf("expected ack decision, got %s", result.decision.Action)
}
if worker.Metrics().Snapshot().Duplicates != 1 {
t.Fatalf("expected duplicate metric to increment")
}
}
func TestFileProcessingWorkerProcessDeliveryMalformedPayload(t *testing.T) {
worker := NewFileProcessingWorker(WorkerConfig{}, nil, NewFileProcessingJSONSerializer("v1", QueueTypeStandard, ""), stubFileProcessingHandler{}, nil, stubIdempotencyStore{}, nil, nil, logger.New(config.LoggingConfig{Format: "json"}))
result := worker.processDelivery(context.Background(), &Delivery{ID: "msg-1", Body: []byte(`{"bad":true}`), ReceiveCount: 1})
if result.decision.Action != RetryActionDeadLetter {
t.Fatalf("expected dead-letter decision, got %s", result.decision.Action)
}
if worker.Metrics().Snapshot().Malformed != 1 {
t.Fatalf("expected malformed metric to increment")
}
}
func TestFileProcessingWorkerProcessDeliveryTransientError(t *testing.T) {
worker := NewFileProcessingWorker(WorkerConfig{}, nil, NewFileProcessingJSONSerializer("v1", QueueTypeStandard, ""), stubFileProcessingHandler{err: Transient(errors.New("db timeout"))}, nil, stubIdempotencyStore{}, nil, nil, logger.New(config.LoggingConfig{Format: "json"}))
fileUUID, _ := uuidv7.String(uuidv7.MustNew())
message, err := NewFileProcessingJSONSerializer("v1", QueueTypeStandard, "").Serialize(context.Background(), FileProcessingJob{
FileUUID: fileUUID,
Bucket: "bucket",
ObjectKey: "files/doc.pdf",
SizeBytes: 10,
})
if err != nil {
t.Fatalf("serialize: %v", err)
}
result := worker.processDelivery(context.Background(), &Delivery{ID: message.ID, Body: message.Body, Attributes: message.Attributes, ReceiveCount: 2})
if result.decision.Action != RetryActionRetry {
t.Fatalf("expected retry decision, got %s", result.decision.Action)
}
if result.decision.Delay <= 0 {
t.Fatalf("expected positive retry delay")
}
}

90
internal/queue/health.go Normal file
View File

@@ -0,0 +1,90 @@
package queue
import (
"sync"
"time"
)
type HealthReporter struct {
mu sync.RWMutex
startedAt time.Time
lastPollAt time.Time
lastSuccessAt time.Time
lastFailureAt time.Time
lastError string
consecutivePollFailures int
errorThreshold int
draining bool
}
type HealthSnapshot struct {
Status string `json:"status"`
StartedAt time.Time `json:"started_at"`
LastPollAt time.Time `json:"last_poll_at,omitempty"`
LastSuccessAt time.Time `json:"last_success_at,omitempty"`
LastFailureAt time.Time `json:"last_failure_at,omitempty"`
LastError string `json:"last_error,omitempty"`
ConsecutivePollFailure int `json:"consecutive_poll_failures"`
Draining bool `json:"draining"`
}
func NewHealthReporter(errorThreshold int) *HealthReporter {
if errorThreshold <= 0 {
errorThreshold = 3
}
now := time.Now().UTC()
return &HealthReporter{
startedAt: now,
lastPollAt: now,
errorThreshold: errorThreshold,
}
}
func (r *HealthReporter) MarkPollSuccess() {
r.mu.Lock()
defer r.mu.Unlock()
now := time.Now().UTC()
r.lastPollAt = now
r.lastSuccessAt = now
r.consecutivePollFailures = 0
r.lastError = ""
}
func (r *HealthReporter) MarkPollFailure(err error) {
r.mu.Lock()
defer r.mu.Unlock()
now := time.Now().UTC()
r.lastPollAt = now
r.lastFailureAt = now
r.consecutivePollFailures++
if err != nil {
r.lastError = err.Error()
}
}
func (r *HealthReporter) MarkDraining() {
r.mu.Lock()
defer r.mu.Unlock()
r.draining = true
}
func (r *HealthReporter) Snapshot() HealthSnapshot {
r.mu.RLock()
defer r.mu.RUnlock()
status := "ok"
if r.draining {
status = "draining"
} else if r.consecutivePollFailures >= r.errorThreshold {
status = "degraded"
}
return HealthSnapshot{
Status: status,
StartedAt: r.startedAt,
LastPollAt: r.lastPollAt,
LastSuccessAt: r.lastSuccessAt,
LastFailureAt: r.lastFailureAt,
LastError: r.lastError,
ConsecutivePollFailure: r.consecutivePollFailures,
Draining: r.draining,
}
}

View File

@@ -0,0 +1,95 @@
package queue
import (
"context"
"sync"
"time"
)
type IdempotencyState string
const (
IdempotencyStateAcquired IdempotencyState = "acquired"
IdempotencyStateDuplicate IdempotencyState = "duplicate"
IdempotencyStateInProgress IdempotencyState = "in_progress"
)
type IdempotencyStore interface {
Acquire(ctx context.Context, key string, processingTTL time.Duration) (IdempotencyState, error)
MarkCompleted(ctx context.Context, key string, ttl time.Duration) error
Release(ctx context.Context, key string) error
}
type NoopIdempotencyStore struct{}
func (NoopIdempotencyStore) Acquire(context.Context, string, time.Duration) (IdempotencyState, error) {
return IdempotencyStateAcquired, nil
}
func (NoopIdempotencyStore) MarkCompleted(context.Context, string, time.Duration) error {
return nil
}
func (NoopIdempotencyStore) Release(context.Context, string) error {
return nil
}
type MemoryIdempotencyStore struct {
mu sync.Mutex
records map[string]memoryIdempotencyRecord
now func() time.Time
}
type memoryIdempotencyRecord struct {
state IdempotencyState
expiresAt time.Time
}
func NewMemoryIdempotencyStore() *MemoryIdempotencyStore {
return &MemoryIdempotencyStore{
records: make(map[string]memoryIdempotencyRecord),
now: func() time.Time { return time.Now().UTC() },
}
}
func (s *MemoryIdempotencyStore) Acquire(_ context.Context, key string, processingTTL time.Duration) (IdempotencyState, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.cleanupExpiredLocked()
if record, ok := s.records[key]; ok {
return record.state, nil
}
s.records[key] = memoryIdempotencyRecord{
state: IdempotencyStateInProgress,
expiresAt: s.now().Add(processingTTL),
}
return IdempotencyStateAcquired, nil
}
func (s *MemoryIdempotencyStore) MarkCompleted(_ context.Context, key string, ttl time.Duration) error {
s.mu.Lock()
defer s.mu.Unlock()
s.records[key] = memoryIdempotencyRecord{
state: IdempotencyStateDuplicate,
expiresAt: s.now().Add(ttl),
}
return nil
}
func (s *MemoryIdempotencyStore) Release(_ context.Context, key string) error {
s.mu.Lock()
defer s.mu.Unlock()
delete(s.records, key)
return nil
}
func (s *MemoryIdempotencyStore) cleanupExpiredLocked() {
now := s.now()
for key, record := range s.records {
if !record.expiresAt.IsZero() && now.After(record.expiresAt) {
delete(s.records, key)
}
}
}

112
internal/queue/metrics.go Normal file
View File

@@ -0,0 +1,112 @@
package queue
import (
"sync/atomic"
"time"
)
type Metrics struct {
received atomic.Int64
processed atomic.Int64
retried atomic.Int64
failed atomic.Int64
deadLettered atomic.Int64
duplicates atomic.Int64
malformed atomic.Int64
ackFailures atomic.Int64
retryFailures atomic.Int64
deadLetterFailures atomic.Int64
lastQueueDepth atomic.Int64
totalProcessingNanos atomic.Int64
lastProcessingNanos atomic.Int64
}
type MetricsSnapshot struct {
Received int64 `json:"received"`
Processed int64 `json:"processed"`
Retried int64 `json:"retried"`
Failed int64 `json:"failed"`
DeadLettered int64 `json:"dead_lettered"`
Duplicates int64 `json:"duplicates"`
Malformed int64 `json:"malformed"`
AckFailures int64 `json:"ack_failures"`
RetryFailures int64 `json:"retry_failures"`
DeadLetterFailures int64 `json:"dead_letter_failures"`
ApproximateQueueDepth int64 `json:"approximate_queue_depth"`
LastProcessingLatency time.Duration `json:"last_processing_latency"`
AverageProcessingLatency time.Duration `json:"average_processing_latency"`
}
func NewMetrics() *Metrics {
return &Metrics{}
}
func (m *Metrics) RecordReceive(n int) {
m.received.Add(int64(n))
}
func (m *Metrics) RecordProcessed(duration time.Duration) {
m.processed.Add(1)
m.lastProcessingNanos.Store(duration.Nanoseconds())
m.totalProcessingNanos.Add(duration.Nanoseconds())
}
func (m *Metrics) RecordRetry() {
m.retried.Add(1)
}
func (m *Metrics) RecordFailure() {
m.failed.Add(1)
}
func (m *Metrics) RecordDeadLetter() {
m.deadLettered.Add(1)
}
func (m *Metrics) RecordDuplicate() {
m.duplicates.Add(1)
}
func (m *Metrics) RecordMalformed() {
m.malformed.Add(1)
}
func (m *Metrics) RecordAckFailure() {
m.ackFailures.Add(1)
}
func (m *Metrics) RecordRetryFailure() {
m.retryFailures.Add(1)
}
func (m *Metrics) RecordDeadLetterFailure() {
m.deadLetterFailures.Add(1)
}
func (m *Metrics) SetQueueDepth(depth int64) {
m.lastQueueDepth.Store(depth)
}
func (m *Metrics) Snapshot() MetricsSnapshot {
processed := m.processed.Load()
total := m.totalProcessingNanos.Load()
avg := time.Duration(0)
if processed > 0 {
avg = time.Duration(total / processed)
}
return MetricsSnapshot{
Received: m.received.Load(),
Processed: processed,
Retried: m.retried.Load(),
Failed: m.failed.Load(),
DeadLettered: m.deadLettered.Load(),
Duplicates: m.duplicates.Load(),
Malformed: m.malformed.Load(),
AckFailures: m.ackFailures.Load(),
RetryFailures: m.retryFailures.Load(),
DeadLetterFailures: m.deadLetterFailures.Load(),
ApproximateQueueDepth: m.lastQueueDepth.Load(),
LastProcessingLatency: time.Duration(m.lastProcessingNanos.Load()),
AverageProcessingLatency: avg,
}
}

106
internal/queue/retry.go Normal file
View File

@@ -0,0 +1,106 @@
package queue
import (
"hash/fnv"
"math"
"time"
)
type RetryAction string
const (
RetryActionRetry RetryAction = "retry"
RetryActionDeadLetter RetryAction = "dead_letter"
RetryActionAck RetryAction = "ack"
)
type RetryDecision struct {
Action RetryAction
Delay time.Duration
Reason string
}
type RetryPolicy interface {
Decide(delivery *Delivery, envelope *Envelope, err error) RetryDecision
}
type ExponentialBackoffPolicy struct {
minDelay time.Duration
maxDelay time.Duration
permanentFailureDelay time.Duration
}
func NewExponentialBackoffPolicy(minDelay, maxDelay, permanentFailureDelay time.Duration) *ExponentialBackoffPolicy {
if minDelay <= 0 {
minDelay = time.Second
}
if maxDelay < minDelay {
maxDelay = minDelay
}
if permanentFailureDelay <= 0 {
permanentFailureDelay = 30 * time.Second
}
return &ExponentialBackoffPolicy{
minDelay: minDelay,
maxDelay: maxDelay,
permanentFailureDelay: permanentFailureDelay,
}
}
func (p *ExponentialBackoffPolicy) Decide(delivery *Delivery, _ *Envelope, err error) RetryDecision {
if err == nil {
return RetryDecision{Action: RetryActionAck}
}
if IsPermanent(err) {
return RetryDecision{
Action: RetryActionDeadLetter,
Delay: p.permanentFailureDelay,
Reason: err.Error(),
}
}
attempt := 1
if delivery != nil && delivery.ReceiveCount > 0 {
attempt = delivery.ReceiveCount
}
delay := p.backoffWithJitter(deliveryID(delivery), attempt)
return RetryDecision{
Action: RetryActionRetry,
Delay: delay,
Reason: err.Error(),
}
}
func (p *ExponentialBackoffPolicy) backoffWithJitter(seed string, attempt int) time.Duration {
if attempt < 1 {
attempt = 1
}
power := math.Pow(2, float64(attempt-1))
delay := time.Duration(float64(p.minDelay) * power)
if delay > p.maxDelay {
delay = p.maxDelay
}
if delay <= 0 {
return p.minDelay
}
jitter := deterministicJitter(seed, delay/2)
return delay/2 + jitter
}
func deterministicJitter(seed string, max time.Duration) time.Duration {
if max <= 0 {
return 0
}
h := fnv.New64a()
_, _ = h.Write([]byte(seed))
return time.Duration(h.Sum64() % uint64(max))
}
func deliveryID(delivery *Delivery) string {
if delivery == nil {
return "unknown"
}
if delivery.ID != "" {
return delivery.ID
}
return "unknown"
}

View File

@@ -0,0 +1,185 @@
package queue
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"strings"
"time"
)
var ErrInvalidEnvelope = errors.New("invalid queue envelope")
type JSONSerializer struct {
version string
enableLegacyFallback bool
queueType string
messageGroupID string
}
type LegacyJSONSerializer struct{}
func NewJSONSerializer(version string, enableLegacyFallback bool, queueType, messageGroupID string) *JSONSerializer {
version = strings.TrimSpace(version)
if version == "" {
version = "v1"
}
queueType = strings.ToLower(strings.TrimSpace(queueType))
if queueType == "" {
queueType = QueueTypeStandard
}
return &JSONSerializer{
version: version,
enableLegacyFallback: enableLegacyFallback,
queueType: queueType,
messageGroupID: strings.TrimSpace(messageGroupID),
}
}
func (s *JSONSerializer) Serialize(ctx context.Context, job EmailJob) (*OutboundMessage, error) {
envelope, err := NewEnvelope(ctx, s.version, job)
if err != nil {
return nil, err
}
body, err := json.Marshal(envelope)
if err != nil {
return nil, err
}
msg := &OutboundMessage{
ID: envelope.MessageID,
Body: body,
Attributes: map[string]string{
"message_kind": envelope.Kind,
"message_id": envelope.MessageID,
"correlation_id": envelope.CorrelationID,
"trace_id": envelope.TraceID,
"schema_version": envelope.Version,
"job_type": envelope.Payload.NormalizedType(),
},
}
if s.queueType == QueueTypeFIFO {
msg.MessageGroupID = s.messageGroupID
if msg.MessageGroupID == "" {
msg.MessageGroupID = "default"
}
msg.DeduplicationID = envelope.DeduplicationKey()
}
return msg, nil
}
func (s *JSONSerializer) Deserialize(body []byte, attributes map[string]string) (*Envelope, error) {
var envelope Envelope
if err := json.Unmarshal(body, &envelope); err == nil {
if err := validateEnvelope(&envelope); err == nil {
return &envelope, nil
}
}
if !s.enableLegacyFallback {
return nil, ErrInvalidEnvelope
}
var job EmailJob
if err := json.Unmarshal(body, &job); err != nil {
return nil, errors.Join(ErrInvalidEnvelope, err)
}
job = job.Canonical()
if err := job.Validate(); err != nil {
return nil, errors.Join(ErrInvalidEnvelope, err)
}
id := messageHash(body)
correlationID := strings.TrimSpace(attributes["correlation_id"])
if correlationID == "" {
correlationID = id
}
traceID := strings.TrimSpace(attributes["trace_id"])
if traceID == "" {
traceID = correlationID
}
return &Envelope{
Version: "legacy",
MessageID: id,
Kind: EmailKind,
OccurredAt: time.Now().UTC(),
CorrelationID: correlationID,
TraceID: traceID,
IdempotencyKey: id,
Payload: job,
}, nil
}
func (LegacyJSONSerializer) Serialize(ctx context.Context, job EmailJob) (*OutboundMessage, error) {
job = job.Canonical()
if err := job.Validate(); err != nil {
return nil, err
}
body, err := json.Marshal(job)
if err != nil {
return nil, err
}
id := messageHash(body)
correlationID := CorrelationIDFromContext(ctx)
if correlationID == "" {
correlationID = id
}
traceID := TraceIDFromContext(ctx)
if traceID == "" {
traceID = correlationID
}
return &OutboundMessage{
ID: id,
Body: body,
Attributes: map[string]string{
"message_kind": EmailKind,
"message_id": id,
"correlation_id": correlationID,
"trace_id": traceID,
"schema_version": "legacy",
"job_type": job.NormalizedType(),
},
}, nil
}
func (LegacyJSONSerializer) Deserialize(body []byte, attributes map[string]string) (*Envelope, error) {
return NewJSONSerializer("v1", true, QueueTypeStandard, "").Deserialize(body, attributes)
}
func validateEnvelope(envelope *Envelope) error {
if envelope == nil {
return ErrInvalidEnvelope
}
if strings.TrimSpace(envelope.Version) == "" {
return errors.Join(ErrInvalidEnvelope, errors.New("version is required"))
}
if strings.TrimSpace(envelope.MessageID) == "" {
return errors.Join(ErrInvalidEnvelope, errors.New("message id is required"))
}
if strings.TrimSpace(envelope.Kind) == "" {
return errors.Join(ErrInvalidEnvelope, errors.New("kind is required"))
}
if envelope.OccurredAt.IsZero() {
return errors.Join(ErrInvalidEnvelope, errors.New("occurred at is required"))
}
if err := envelope.Payload.Validate(); err != nil {
return errors.Join(ErrInvalidEnvelope, err)
}
envelope.Payload = envelope.Payload.Canonical()
if strings.TrimSpace(envelope.CorrelationID) == "" {
envelope.CorrelationID = envelope.MessageID
}
if strings.TrimSpace(envelope.TraceID) == "" {
envelope.TraceID = envelope.CorrelationID
}
if strings.TrimSpace(envelope.IdempotencyKey) == "" {
envelope.IdempotencyKey = envelope.MessageID
}
return nil
}
func messageHash(body []byte) string {
sum := sha256.Sum256(body)
return hex.EncodeToString(sum[:16])
}

View File

@@ -0,0 +1,54 @@
package queue
import (
"context"
"testing"
)
func TestJSONSerializerRoundTrip(t *testing.T) {
serializer := NewJSONSerializer("v1", true, QueueTypeStandard, "")
ctx := WithCorrelationID(context.Background(), "req-123")
message, err := serializer.Serialize(ctx, EmailJob{
Type: "invite_set_password",
To: "user@example.com",
Subject: "Welcome",
Body: "hello",
HTMLBody: "<b>hello</b>",
})
if err != nil {
t.Fatalf("serialize: %v", err)
}
envelope, err := serializer.Deserialize(message.Body, message.Attributes)
if err != nil {
t.Fatalf("deserialize: %v", err)
}
if envelope.MessageID == "" {
t.Fatalf("expected message id")
}
if envelope.CorrelationID != "req-123" {
t.Fatalf("expected correlation id req-123, got %q", envelope.CorrelationID)
}
if envelope.Payload.NormalizedType() != "invite_set_password" {
t.Fatalf("unexpected job type: %q", envelope.Payload.NormalizedType())
}
}
func TestJSONSerializerLegacyFallback(t *testing.T) {
serializer := NewJSONSerializer("v1", true, QueueTypeStandard, "")
envelope, err := serializer.Deserialize([]byte(`{"to":"user@example.com","subject":"Reset","body":"hello"}`), map[string]string{
"correlation_id": "legacy-1",
})
if err != nil {
t.Fatalf("deserialize legacy: %v", err)
}
if envelope.Version != "legacy" {
t.Fatalf("expected legacy version, got %q", envelope.Version)
}
if envelope.CorrelationID != "legacy-1" {
t.Fatalf("unexpected correlation id: %q", envelope.CorrelationID)
}
if envelope.Payload.To != "user@example.com" {
t.Fatalf("unexpected payload: %+v", envelope.Payload)
}
}

330
internal/queue/types.go Normal file
View File

@@ -0,0 +1,330 @@
package queue
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"net/mail"
"strings"
"time"
"wucher/internal/shared/pkg/uuidv7"
)
const (
BackendSQS = "sqs"
QueueTypeStandard = "standard"
QueueTypeFIFO = "fifo"
EmailKind = "email.dispatch"
)
var (
ErrInvalidEmailJob = errors.New("invalid email job")
)
type EmailJob struct {
JobID string `json:"job_id,omitempty"`
MessageType string `json:"message_type,omitempty"`
To string `json:"to"`
CC []string `json:"cc,omitempty"`
BCC []string `json:"bcc,omitempty"`
Subject string `json:"subject"`
TemplateName string `json:"template_name,omitempty"`
TemplateData map[string]any `json:"template_data,omitempty"`
TextBody string `json:"text_body,omitempty"`
HTMLBody string `json:"html_body,omitempty"`
Attachments []EmailAttachment `json:"attachments,omitempty"`
ReplyTo []string `json:"reply_to,omitempty"`
Tags map[string]string `json:"tags,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
CreatedAt time.Time `json:"created_at,omitempty"`
TraceID string `json:"trace_id,omitempty"`
RequestID string `json:"request_id,omitempty"`
// Legacy aliases kept for backward-compatible decoding and tests.
Type string `json:"type,omitempty"`
Body string `json:"body,omitempty"`
}
type EmailAttachment struct {
Filename string `json:"filename,omitempty"`
ContentType string `json:"content_type,omitempty"`
ContentID string `json:"content_id,omitempty"`
Content []byte `json:"content,omitempty"`
Inline bool `json:"inline,omitempty"`
}
func (j EmailJob) Validate() error {
job := j.Canonical()
if strings.TrimSpace(job.To) == "" {
return errors.Join(ErrInvalidEmailJob, errors.New("recipient is required"))
}
if err := validateEmailAddress(job.To); err != nil {
return errors.Join(ErrInvalidEmailJob, err)
}
if err := validateEmailAddresses(job.CC); err != nil {
return errors.Join(ErrInvalidEmailJob, err)
}
if err := validateEmailAddresses(job.BCC); err != nil {
return errors.Join(ErrInvalidEmailJob, err)
}
if err := validateEmailAddresses(job.ReplyTo); err != nil {
return errors.Join(ErrInvalidEmailJob, err)
}
hasBody := strings.TrimSpace(job.TextBody) != "" || strings.TrimSpace(job.HTMLBody) != ""
if strings.TrimSpace(job.Subject) == "" && (strings.TrimSpace(job.TemplateName) == "" || hasBody) {
return errors.Join(ErrInvalidEmailJob, errors.New("subject is required"))
}
if !hasBody && strings.TrimSpace(job.TemplateName) == "" {
return errors.Join(ErrInvalidEmailJob, errors.New("body is required"))
}
for i := range job.Attachments {
attachment := job.Attachments[i]
if len(attachment.Content) == 0 {
return errors.Join(ErrInvalidEmailJob, errors.New("attachment content is required"))
}
if strings.TrimSpace(attachment.ContentType) == "" {
return errors.Join(ErrInvalidEmailJob, errors.New("attachment content type is required"))
}
if attachment.Inline && strings.TrimSpace(attachment.ContentID) == "" {
return errors.Join(ErrInvalidEmailJob, errors.New("inline attachment content id is required"))
}
}
return nil
}
func (j EmailJob) RecipientHash() string {
sum := sha256.Sum256([]byte(strings.ToLower(strings.TrimSpace(j.Canonical().To))))
return hex.EncodeToString(sum[:8])
}
func (j EmailJob) NormalizedType() string {
if v := strings.TrimSpace(j.Canonical().MessageType); v != "" {
return v
}
return "generic"
}
func (j EmailJob) TextBodyValue() string {
return j.Canonical().TextBody
}
func (j EmailJob) Canonical() EmailJob {
job := j
job.To = strings.TrimSpace(job.To)
job.Subject = strings.TrimSpace(job.Subject)
job.TemplateName = strings.TrimSpace(job.TemplateName)
job.JobID = strings.TrimSpace(job.JobID)
job.TraceID = strings.TrimSpace(job.TraceID)
job.RequestID = strings.TrimSpace(job.RequestID)
if strings.TrimSpace(job.MessageType) == "" {
job.MessageType = strings.TrimSpace(job.Type)
}
if strings.TrimSpace(job.TextBody) == "" {
job.TextBody = job.Body
}
job.Type = ""
job.Body = ""
job.CC = cleanStrings(job.CC)
job.BCC = cleanStrings(job.BCC)
job.ReplyTo = cleanStrings(job.ReplyTo)
job.Tags = cleanStringMap(job.Tags)
job.Metadata = cleanStringMap(job.Metadata)
job.Attachments = canonicalEmailAttachments(job.Attachments)
return job
}
func canonicalEmailAttachments(values []EmailAttachment) []EmailAttachment {
if len(values) == 0 {
return nil
}
out := make([]EmailAttachment, 0, len(values))
for i := range values {
item := values[i]
item.Filename = strings.TrimSpace(item.Filename)
item.ContentType = strings.TrimSpace(item.ContentType)
item.ContentID = strings.Trim(strings.TrimSpace(item.ContentID), "<>")
if len(item.Content) == 0 {
continue
}
item.Content = append([]byte(nil), item.Content...)
out = append(out, item)
}
if len(out) == 0 {
return nil
}
return out
}
type Envelope struct {
Version string `json:"version"`
MessageID string `json:"message_id"`
Kind string `json:"kind"`
OccurredAt time.Time `json:"occurred_at"`
CorrelationID string `json:"correlation_id,omitempty"`
TraceID string `json:"trace_id,omitempty"`
IdempotencyKey string `json:"idempotency_key,omitempty"`
Payload EmailJob `json:"payload"`
}
func NewEnvelope(ctx context.Context, version string, job EmailJob) (*Envelope, error) {
job = job.Canonical()
if err := job.Validate(); err != nil {
return nil, err
}
id, err := newMessageID()
if err != nil {
return nil, err
}
correlationID := CorrelationIDFromContext(ctx)
if correlationID == "" {
correlationID = id
}
traceID := TraceIDFromContext(ctx)
if traceID == "" {
traceID = correlationID
}
now := time.Now().UTC()
if job.JobID == "" {
job.JobID = id
}
if job.CreatedAt.IsZero() {
job.CreatedAt = now
}
if job.TraceID == "" {
job.TraceID = traceID
}
if job.RequestID == "" {
job.RequestID = correlationID
}
return &Envelope{
Version: strings.TrimSpace(version),
MessageID: id,
Kind: EmailKind,
OccurredAt: now,
CorrelationID: correlationID,
TraceID: traceID,
IdempotencyKey: job.JobID,
Payload: job,
}, nil
}
func (e Envelope) DeduplicationKey() string {
if v := strings.TrimSpace(e.IdempotencyKey); v != "" {
return v
}
return strings.TrimSpace(e.MessageID)
}
type OutboundMessage struct {
ID string
Body []byte
Attributes map[string]string
MessageGroupID string
DeduplicationID string
}
type Delivery struct {
ID string
ReceiptHandle string
Body []byte
Attributes map[string]string
ReceiveCount int
EnqueuedAt time.Time
}
type MessageSerializer interface {
Serialize(ctx context.Context, job EmailJob) (*OutboundMessage, error)
Deserialize(body []byte, attributes map[string]string) (*Envelope, error)
}
type QueueProducer interface {
Backend() string
Publish(ctx context.Context, message OutboundMessage) error
}
type QueueConsumer interface {
Backend() string
Receive(ctx context.Context, maxMessages int) ([]*Delivery, error)
Ack(ctx context.Context, deliveries []*Delivery) error
Retry(ctx context.Context, delivery *Delivery, delay time.Duration) error
DeadLetter(ctx context.Context, delivery *Delivery, reason string, cause error) error
ApproximateDepth(ctx context.Context) (int64, error)
Close() error
}
type EmailJobProducer interface {
Enqueue(ctx context.Context, job EmailJob) error
}
type EmailJobHandler interface {
Handle(ctx context.Context, envelope *Envelope) error
}
func newMessageID() (string, error) {
id, err := uuidv7.String(uuidv7.MustNew())
if err != nil {
return "", err
}
return id, nil
}
func cleanStrings(values []string) []string {
if len(values) == 0 {
return nil
}
out := make([]string, 0, len(values))
for _, value := range values {
value = strings.TrimSpace(value)
if value == "" {
continue
}
out = append(out, value)
}
if len(out) == 0 {
return nil
}
return out
}
func cleanStringMap(values map[string]string) map[string]string {
if len(values) == 0 {
return nil
}
out := make(map[string]string, len(values))
for key, value := range values {
key = strings.TrimSpace(key)
value = strings.TrimSpace(value)
if key == "" || value == "" {
continue
}
out[key] = value
}
if len(out) == 0 {
return nil
}
return out
}
func validateEmailAddresses(addresses []string) error {
for _, address := range addresses {
if err := validateEmailAddress(address); err != nil {
return err
}
}
return nil
}
func validateEmailAddress(address string) error {
address = strings.TrimSpace(address)
if address == "" {
return nil
}
if _, err := mail.ParseAddress(address); err != nil {
return err
}
return nil
}

320
internal/queue/worker.go Normal file
View File

@@ -0,0 +1,320 @@
package queue
import (
"context"
"errors"
"log/slog"
"strings"
"time"
"wucher/internal/shared/pkg/metrics"
)
type WorkerConfig struct {
Concurrency int
MaxBatchSize int
ProcessTimeout time.Duration
ProcessingTTL time.Duration
IdempotencyTTL time.Duration
PollErrorDelay time.Duration
MetricsComponent string
}
type Worker struct {
cfg WorkerConfig
consumer QueueConsumer
serializer MessageSerializer
handler EmailJobHandler
retry RetryPolicy
idem IdempotencyStore
metrics *Metrics
health *HealthReporter
logger *slog.Logger
component string
}
type processResult struct {
delivery *Delivery
envelope *Envelope
decision RetryDecision
err error
}
func NewWorker(
cfg WorkerConfig,
consumer QueueConsumer,
serializer MessageSerializer,
handler EmailJobHandler,
retry RetryPolicy,
idem IdempotencyStore,
metrics *Metrics,
health *HealthReporter,
logger *slog.Logger,
) *Worker {
if cfg.Concurrency <= 0 {
cfg.Concurrency = 1
}
if cfg.MaxBatchSize <= 0 || cfg.MaxBatchSize > cfg.Concurrency {
cfg.MaxBatchSize = cfg.Concurrency
}
if cfg.ProcessTimeout <= 0 {
cfg.ProcessTimeout = 30 * time.Second
}
if cfg.ProcessingTTL <= 0 {
cfg.ProcessingTTL = 15 * time.Minute
}
if cfg.IdempotencyTTL <= 0 {
cfg.IdempotencyTTL = 24 * time.Hour
}
if cfg.PollErrorDelay <= 0 {
cfg.PollErrorDelay = time.Second
}
if retry == nil {
retry = NewExponentialBackoffPolicy(time.Second, 5*time.Minute, 30*time.Second)
}
if idem == nil {
idem = NoopIdempotencyStore{}
}
if metrics == nil {
metrics = NewMetrics()
}
if health == nil {
health = NewHealthReporter(3)
}
if logger == nil {
logger = slog.Default()
}
return &Worker{
cfg: cfg,
consumer: consumer,
serializer: serializer,
handler: handler,
retry: retry,
idem: idem,
metrics: metrics,
health: health,
logger: logger,
component: normalizeWorkerMetricsComponent(cfg.MetricsComponent),
}
}
func (w *Worker) Run(ctx context.Context) error {
if w.consumer == nil {
return errors.New("queue consumer is required")
}
if w.serializer == nil {
return errors.New("message serializer is required")
}
if w.handler == nil {
return errors.New("message handler is required")
}
for {
select {
case <-ctx.Done():
w.health.MarkDraining()
return nil
default:
}
deliveries, err := w.consumer.Receive(ctx, w.cfg.MaxBatchSize)
if err != nil {
if ctx.Err() != nil {
w.health.MarkDraining()
return nil
}
w.health.MarkPollFailure(err)
w.logger.Error("queue receive failed",
slog.String("backend", w.consumer.Backend()),
slog.Any("error", err),
)
select {
case <-ctx.Done():
w.health.MarkDraining()
return nil
case <-time.After(w.cfg.PollErrorDelay):
}
continue
}
w.health.MarkPollSuccess()
if len(deliveries) == 0 {
continue
}
w.metrics.RecordReceive(len(deliveries))
results := make(chan processResult, len(deliveries))
for _, delivery := range deliveries {
delivery := delivery
go func() {
results <- w.processDelivery(ctx, delivery)
}()
}
acks := make([]*Delivery, 0, len(deliveries))
for range deliveries {
result := <-results
switch result.decision.Action {
case RetryActionAck:
acks = append(acks, result.delivery)
case RetryActionRetry:
if err := w.consumer.Retry(ctx, result.delivery, result.decision.Delay); err != nil {
w.metrics.RecordRetryFailure()
w.logger.Error("queue retry scheduling failed",
slog.String("backend", w.consumer.Backend()),
slog.String("message_id", result.delivery.ID),
slog.Any("error", err),
)
} else {
w.metrics.RecordRetry()
}
case RetryActionDeadLetter:
if err := w.consumer.DeadLetter(ctx, result.delivery, result.decision.Reason, result.err); err != nil {
w.metrics.RecordDeadLetterFailure()
w.logger.Error("queue dead-letter failed",
slog.String("backend", w.consumer.Backend()),
slog.String("message_id", result.delivery.ID),
slog.Any("error", err),
)
} else {
w.metrics.RecordDeadLetter()
}
}
}
if len(acks) > 0 {
if err := w.consumer.Ack(ctx, acks); err != nil {
w.metrics.RecordAckFailure()
w.logger.Error("queue ack failed",
slog.String("backend", w.consumer.Backend()),
slog.Int("count", len(acks)),
slog.Any("error", err),
)
}
}
}
}
func (w *Worker) RefreshQueueDepth(ctx context.Context) error {
if w.consumer == nil {
return errors.New("queue consumer is required")
}
depth, err := w.consumer.ApproximateDepth(ctx)
if err != nil {
return err
}
w.metrics.SetQueueDepth(depth)
return nil
}
func (w *Worker) Metrics() *Metrics {
return w.metrics
}
func (w *Worker) Health() *HealthReporter {
return w.health
}
func (w *Worker) processDelivery(ctx context.Context, delivery *Delivery) (result processResult) {
started := time.Now()
jobType := "unknown"
var jobErr error
defer func() {
metrics.ObserveWorkerJob(w.component, jobType, jobErr, started)
}()
envelope, err := w.serializer.Deserialize(delivery.Body, delivery.Attributes)
if err != nil {
w.metrics.RecordMalformed()
w.metrics.RecordFailure()
jobErr = err
return processResult{
delivery: delivery,
decision: w.retry.Decide(delivery, nil, Permanent(err)),
err: err,
}
}
jobType = strings.TrimSpace(envelope.Payload.NormalizedType())
idempotencyKey := envelope.DeduplicationKey()
state, err := w.idem.Acquire(ctx, idempotencyKey, w.cfg.ProcessingTTL)
if err != nil {
w.metrics.RecordFailure()
jobErr = err
return processResult{
delivery: delivery,
envelope: envelope,
decision: w.retry.Decide(delivery, envelope, Transient(err)),
err: err,
}
}
switch state {
case IdempotencyStateDuplicate:
w.metrics.RecordDuplicate()
w.logger.Info("queue duplicate skipped",
slog.String("message_id", envelope.MessageID),
slog.String("job_type", envelope.Payload.NormalizedType()),
slog.String("recipient_hash", envelope.Payload.RecipientHash()),
slog.String("correlation_id", envelope.CorrelationID),
)
return processResult{
delivery: delivery,
envelope: envelope,
decision: RetryDecision{Action: RetryActionAck},
}
case IdempotencyStateInProgress:
inProgressErr := Transient(errors.New("message already in progress"))
jobErr = inProgressErr
return processResult{
delivery: delivery,
envelope: envelope,
decision: w.retry.Decide(delivery, envelope, inProgressErr),
err: inProgressErr,
}
}
processCtx, cancel := context.WithTimeout(ctx, w.cfg.ProcessTimeout)
defer cancel()
if err := w.handler.Handle(processCtx, envelope); err != nil {
_ = w.idem.Release(ctx, idempotencyKey)
w.metrics.RecordFailure()
jobErr = err
return processResult{
delivery: delivery,
envelope: envelope,
decision: w.retry.Decide(delivery, envelope, err),
err: err,
}
}
if err := w.idem.MarkCompleted(ctx, idempotencyKey, w.cfg.IdempotencyTTL); err != nil {
w.metrics.RecordFailure()
w.logger.Error("queue idempotency completion failed",
slog.String("message_id", envelope.MessageID),
slog.String("correlation_id", envelope.CorrelationID),
slog.Any("error", err),
)
jobErr = err
return processResult{
delivery: delivery,
envelope: envelope,
decision: w.retry.Decide(delivery, envelope, Transient(err)),
err: err,
}
}
w.metrics.RecordProcessed(time.Since(started))
jobErr = nil
return processResult{
delivery: delivery,
envelope: envelope,
decision: RetryDecision{Action: RetryActionAck},
}
}
func normalizeWorkerMetricsComponent(component string) string {
component = strings.ToLower(strings.TrimSpace(component))
if component == "" {
return "worker"
}
return component
}

View File

@@ -0,0 +1,93 @@
package queue
import (
"context"
"errors"
"testing"
"time"
"wucher/internal/config"
"wucher/internal/shared/pkg/logger"
)
type stubHandler struct {
err error
}
func (h stubHandler) Handle(context.Context, *Envelope) error {
return h.err
}
type stubIdempotencyStore struct {
state IdempotencyState
err error
}
func (s stubIdempotencyStore) Acquire(context.Context, string, time.Duration) (IdempotencyState, error) {
if s.err != nil {
return "", s.err
}
if s.state != "" {
return s.state, nil
}
return IdempotencyStateAcquired, nil
}
func (s stubIdempotencyStore) MarkCompleted(context.Context, string, time.Duration) error {
return nil
}
func (s stubIdempotencyStore) Release(context.Context, string) error {
return nil
}
func TestWorkerProcessDeliveryDuplicate(t *testing.T) {
worker := NewWorker(WorkerConfig{}, nil, NewJSONSerializer("v1", true, QueueTypeStandard, ""), stubHandler{}, nil, stubIdempotencyStore{state: IdempotencyStateDuplicate}, nil, nil, logger.New(config.LoggingConfig{Format: "json"}))
message, err := NewJSONSerializer("v1", true, QueueTypeStandard, "").Serialize(context.Background(), EmailJob{
To: "user@example.com",
Subject: "Hello",
Body: "World",
})
if err != nil {
t.Fatalf("serialize: %v", err)
}
result := worker.processDelivery(context.Background(), &Delivery{ID: message.ID, Body: message.Body, Attributes: message.Attributes, ReceiveCount: 1})
if result.decision.Action != RetryActionAck {
t.Fatalf("expected ack decision, got %s", result.decision.Action)
}
if worker.Metrics().Snapshot().Duplicates != 1 {
t.Fatalf("expected duplicate metric to increment")
}
}
func TestWorkerProcessDeliveryMalformedPayload(t *testing.T) {
worker := NewWorker(WorkerConfig{}, nil, NewJSONSerializer("v1", false, QueueTypeStandard, ""), stubHandler{}, nil, stubIdempotencyStore{}, nil, nil, logger.New(config.LoggingConfig{Format: "json"}))
result := worker.processDelivery(context.Background(), &Delivery{ID: "msg-1", Body: []byte(`{"bad":true}`), ReceiveCount: 1})
if result.decision.Action != RetryActionDeadLetter {
t.Fatalf("expected dead-letter decision, got %s", result.decision.Action)
}
if worker.Metrics().Snapshot().Malformed != 1 {
t.Fatalf("expected malformed metric to increment")
}
}
func TestWorkerProcessDeliveryTransientError(t *testing.T) {
worker := NewWorker(WorkerConfig{}, nil, NewJSONSerializer("v1", true, QueueTypeStandard, ""), stubHandler{err: Transient(errors.New("ses timeout"))}, nil, stubIdempotencyStore{}, nil, nil, logger.New(config.LoggingConfig{Format: "json"}))
message, err := NewJSONSerializer("v1", true, QueueTypeStandard, "").Serialize(context.Background(), EmailJob{
To: "user@example.com",
Subject: "Hello",
Body: "World",
})
if err != nil {
t.Fatalf("serialize: %v", err)
}
result := worker.processDelivery(context.Background(), &Delivery{ID: message.ID, Body: message.Body, Attributes: message.Attributes, ReceiveCount: 2})
if result.decision.Action != RetryActionRetry {
t.Fatalf("expected retry decision, got %s", result.decision.Action)
}
if result.decision.Delay <= 0 {
t.Fatalf("expected positive retry delay")
}
}