init push
This commit is contained in:
45
internal/queue/context.go
Normal file
45
internal/queue/context.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type contextKey string
|
||||
|
||||
const (
|
||||
correlationIDKey contextKey = "queue-correlation-id"
|
||||
traceIDKey contextKey = "queue-trace-id"
|
||||
)
|
||||
|
||||
func WithCorrelationID(ctx context.Context, correlationID string) context.Context {
|
||||
correlationID = strings.TrimSpace(correlationID)
|
||||
if correlationID == "" {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, correlationIDKey, correlationID)
|
||||
}
|
||||
|
||||
func CorrelationIDFromContext(ctx context.Context) string {
|
||||
if ctx == nil {
|
||||
return ""
|
||||
}
|
||||
value, _ := ctx.Value(correlationIDKey).(string)
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func WithTraceID(ctx context.Context, traceID string) context.Context {
|
||||
traceID = strings.TrimSpace(traceID)
|
||||
if traceID == "" {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, traceIDKey, traceID)
|
||||
}
|
||||
|
||||
func TraceIDFromContext(ctx context.Context) string {
|
||||
if ctx == nil {
|
||||
return ""
|
||||
}
|
||||
value, _ := ctx.Value(traceIDKey).(string)
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
145
internal/queue/enqueuer.go
Normal file
145
internal/queue/enqueuer.go
Normal file
@@ -0,0 +1,145 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
type ProducerMetrics struct {
|
||||
published int64
|
||||
publishFailures int64
|
||||
secondaryFailures int64
|
||||
}
|
||||
|
||||
type EmailEnqueuer struct {
|
||||
serializer MessageSerializer
|
||||
producer QueueProducer
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewEmailEnqueuer(serializer MessageSerializer, producer QueueProducer, logger *slog.Logger) *EmailEnqueuer {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &EmailEnqueuer{
|
||||
serializer: serializer,
|
||||
producer: producer,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *EmailEnqueuer) Enqueue(ctx context.Context, job EmailJob) error {
|
||||
message, err := e.serializer.Serialize(ctx, job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.producer.Publish(ctx, *message); err != nil {
|
||||
e.logger.Error("queue publish failed",
|
||||
slog.String("backend", e.producer.Backend()),
|
||||
slog.String("message_id", message.ID),
|
||||
slog.String("job_type", job.NormalizedType()),
|
||||
slog.String("recipient_hash", job.RecipientHash()),
|
||||
slog.String("correlation_id", message.Attributes["correlation_id"]),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
return err
|
||||
}
|
||||
e.logger.Info("queue published",
|
||||
slog.String("backend", e.producer.Backend()),
|
||||
slog.String("message_id", message.ID),
|
||||
slog.String("job_type", job.NormalizedType()),
|
||||
slog.String("recipient_hash", job.RecipientHash()),
|
||||
slog.String("correlation_id", message.Attributes["correlation_id"]),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
type DualProducer struct {
|
||||
primary QueueProducer
|
||||
secondary QueueProducer
|
||||
secondaryRequired bool
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewDualProducer(primary, secondary QueueProducer, secondaryRequired bool, logger *slog.Logger) *DualProducer {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &DualProducer{
|
||||
primary: primary,
|
||||
secondary: secondary,
|
||||
secondaryRequired: secondaryRequired,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *DualProducer) Backend() string {
|
||||
return "dual"
|
||||
}
|
||||
|
||||
func (p *DualProducer) Publish(ctx context.Context, message OutboundMessage) error {
|
||||
if p.primary == nil {
|
||||
return errors.New("primary producer is required")
|
||||
}
|
||||
if err := p.primary.Publish(ctx, message); err != nil {
|
||||
return err
|
||||
}
|
||||
if p.secondary == nil {
|
||||
return nil
|
||||
}
|
||||
if err := p.secondary.Publish(ctx, message); err != nil {
|
||||
p.logger.Error("queue secondary publish failed",
|
||||
slog.String("primary_backend", p.primary.Backend()),
|
||||
slog.String("secondary_backend", p.secondary.Backend()),
|
||||
slog.String("message_id", message.ID),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
if p.secondaryRequired {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type DualEmailJobProducer struct {
|
||||
primary EmailJobProducer
|
||||
secondary EmailJobProducer
|
||||
secondaryRequired bool
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewDualEmailJobProducer(primary, secondary EmailJobProducer, secondaryRequired bool, logger *slog.Logger) *DualEmailJobProducer {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &DualEmailJobProducer{
|
||||
primary: primary,
|
||||
secondary: secondary,
|
||||
secondaryRequired: secondaryRequired,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *DualEmailJobProducer) Enqueue(ctx context.Context, job EmailJob) error {
|
||||
if p.primary == nil {
|
||||
return errors.New("primary email producer is required")
|
||||
}
|
||||
if err := p.primary.Enqueue(ctx, job); err != nil {
|
||||
return err
|
||||
}
|
||||
if p.secondary == nil {
|
||||
return nil
|
||||
}
|
||||
if err := p.secondary.Enqueue(ctx, job); err != nil {
|
||||
p.logger.Error("queue secondary enqueue failed",
|
||||
slog.String("job_type", job.NormalizedType()),
|
||||
slog.String("recipient_hash", job.RecipientHash()),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
if p.secondaryRequired {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
51
internal/queue/errors.go
Normal file
51
internal/queue/errors.go
Normal file
@@ -0,0 +1,51 @@
|
||||
package queue
|
||||
|
||||
import "errors"
|
||||
|
||||
type permanentError struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (e permanentError) Error() string {
|
||||
return e.err.Error()
|
||||
}
|
||||
|
||||
func (e permanentError) Unwrap() error {
|
||||
return e.err
|
||||
}
|
||||
|
||||
type transientError struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (e transientError) Error() string {
|
||||
return e.err.Error()
|
||||
}
|
||||
|
||||
func (e transientError) Unwrap() error {
|
||||
return e.err
|
||||
}
|
||||
|
||||
func Permanent(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return permanentError{err: err}
|
||||
}
|
||||
|
||||
func Transient(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return transientError{err: err}
|
||||
}
|
||||
|
||||
func IsPermanent(err error) bool {
|
||||
var target permanentError
|
||||
return errors.As(err, &target)
|
||||
}
|
||||
|
||||
func IsTransient(err error) bool {
|
||||
var target transientError
|
||||
return errors.As(err, &target)
|
||||
}
|
||||
47
internal/queue/file_processing_enqueuer.go
Normal file
47
internal/queue/file_processing_enqueuer.go
Normal file
@@ -0,0 +1,47 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
type FileProcessingEnqueuer struct {
|
||||
serializer FileProcessingMessageSerializer
|
||||
producer QueueProducer
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewFileProcessingEnqueuer(serializer FileProcessingMessageSerializer, producer QueueProducer, logger *slog.Logger) *FileProcessingEnqueuer {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &FileProcessingEnqueuer{
|
||||
serializer: serializer,
|
||||
producer: producer,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *FileProcessingEnqueuer) Enqueue(ctx context.Context, job FileProcessingJob) error {
|
||||
message, err := e.serializer.Serialize(ctx, job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.producer.Publish(ctx, *message); err != nil {
|
||||
e.logger.Error("file queue publish failed",
|
||||
slog.String("backend", e.producer.Backend()),
|
||||
slog.String("message_id", message.ID),
|
||||
slog.String("file_uuid", job.Canonical().FileUUID),
|
||||
slog.String("correlation_id", message.Attributes["correlation_id"]),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
return err
|
||||
}
|
||||
e.logger.Info("file queue published",
|
||||
slog.String("backend", e.producer.Backend()),
|
||||
slog.String("message_id", message.ID),
|
||||
slog.String("file_uuid", job.Canonical().FileUUID),
|
||||
slog.String("correlation_id", message.Attributes["correlation_id"]),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
62
internal/queue/file_processing_job_handler.go
Normal file
62
internal/queue/file_processing_job_handler.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
filemanager "wucher/internal/domain/file_manager"
|
||||
)
|
||||
|
||||
type FileProcessingJobProcessor interface {
|
||||
Process(ctx context.Context, job filemanager.FileProcessingJob, correlationID string) error
|
||||
}
|
||||
|
||||
type FileProcessingPermanentClassifier func(error) bool
|
||||
|
||||
type FileProcessingJobQueueHandler struct {
|
||||
processor FileProcessingJobProcessor
|
||||
isPermanent FileProcessingPermanentClassifier
|
||||
}
|
||||
|
||||
func NewFileProcessingJobQueueHandler(
|
||||
processor FileProcessingJobProcessor,
|
||||
isPermanent FileProcessingPermanentClassifier,
|
||||
) *FileProcessingJobQueueHandler {
|
||||
return &FileProcessingJobQueueHandler{
|
||||
processor: processor,
|
||||
isPermanent: isPermanent,
|
||||
}
|
||||
}
|
||||
|
||||
func (h *FileProcessingJobQueueHandler) Handle(ctx context.Context, envelope *FileProcessingEnvelope) error {
|
||||
if envelope == nil {
|
||||
return Permanent(errors.New("file processing envelope is required"))
|
||||
}
|
||||
if h.processor == nil {
|
||||
return Permanent(errors.New("file processing processor is required"))
|
||||
}
|
||||
|
||||
job := envelope.Payload.Canonical()
|
||||
if err := job.Validate(); err != nil {
|
||||
return Permanent(err)
|
||||
}
|
||||
|
||||
err := h.processor.Process(ctx, filemanager.FileProcessingJob{
|
||||
JobID: job.JobID,
|
||||
FileUUID: job.FileUUID,
|
||||
Bucket: job.Bucket,
|
||||
ObjectKey: job.ObjectKey,
|
||||
UploadedBy: job.UploadedBy,
|
||||
MimeType: job.MimeType,
|
||||
SizeBytes: job.SizeBytes,
|
||||
TraceID: job.TraceID,
|
||||
RequestID: job.RequestID,
|
||||
}, envelope.CorrelationID)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if h.isPermanent != nil && h.isPermanent(err) {
|
||||
return Permanent(err)
|
||||
}
|
||||
return Transient(err)
|
||||
}
|
||||
122
internal/queue/file_processing_job_handler_test.go
Normal file
122
internal/queue/file_processing_job_handler_test.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
filemanager "wucher/internal/domain/file_manager"
|
||||
"wucher/internal/shared/pkg/uuidv7"
|
||||
)
|
||||
|
||||
type stubFileProcessingProcessor struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (p stubFileProcessingProcessor) Process(context.Context, filemanager.FileProcessingJob, string) error {
|
||||
return p.err
|
||||
}
|
||||
|
||||
func TestFileProcessingJobQueueHandlerHandleSuccess(t *testing.T) {
|
||||
fileUUID, err := uuidv7.String(uuidv7.MustNew())
|
||||
if err != nil {
|
||||
t.Fatalf("uuid string: %v", err)
|
||||
}
|
||||
handler := NewFileProcessingJobQueueHandler(stubFileProcessingProcessor{}, nil)
|
||||
|
||||
err = handler.Handle(context.Background(), &FileProcessingEnvelope{
|
||||
MessageID: "msg-1",
|
||||
Payload: FileProcessingJob{
|
||||
FileUUID: fileUUID,
|
||||
Bucket: "bucket-a",
|
||||
ObjectKey: "files/a.pdf",
|
||||
SizeBytes: 12,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("handle: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileProcessingJobQueueHandlerHandleNilEnvelopePermanent(t *testing.T) {
|
||||
handler := NewFileProcessingJobQueueHandler(stubFileProcessingProcessor{}, nil)
|
||||
err := handler.Handle(context.Background(), nil)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error")
|
||||
}
|
||||
if !IsPermanent(err) {
|
||||
t.Fatalf("expected permanent error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileProcessingJobQueueHandlerHandleInvalidPayloadPermanent(t *testing.T) {
|
||||
handler := NewFileProcessingJobQueueHandler(stubFileProcessingProcessor{}, nil)
|
||||
err := handler.Handle(context.Background(), &FileProcessingEnvelope{
|
||||
MessageID: "msg-2",
|
||||
Payload: FileProcessingJob{
|
||||
FileUUID: "not-a-uuid",
|
||||
Bucket: "bucket-a",
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("expected error")
|
||||
}
|
||||
if !IsPermanent(err) {
|
||||
t.Fatalf("expected permanent error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileProcessingJobQueueHandlerHandleTransientError(t *testing.T) {
|
||||
fileUUID, err := uuidv7.String(uuidv7.MustNew())
|
||||
if err != nil {
|
||||
t.Fatalf("uuid string: %v", err)
|
||||
}
|
||||
handler := NewFileProcessingJobQueueHandler(stubFileProcessingProcessor{
|
||||
err: errors.New("temporary timeout"),
|
||||
}, nil)
|
||||
|
||||
err = handler.Handle(context.Background(), &FileProcessingEnvelope{
|
||||
MessageID: "msg-3",
|
||||
Payload: FileProcessingJob{
|
||||
FileUUID: fileUUID,
|
||||
Bucket: "bucket-a",
|
||||
ObjectKey: "files/a.pdf",
|
||||
SizeBytes: 12,
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("expected error")
|
||||
}
|
||||
if !IsTransient(err) {
|
||||
t.Fatalf("expected transient error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileProcessingJobQueueHandlerHandlePermanentClassification(t *testing.T) {
|
||||
fileUUID, err := uuidv7.String(uuidv7.MustNew())
|
||||
if err != nil {
|
||||
t.Fatalf("uuid string: %v", err)
|
||||
}
|
||||
permanentErr := errors.New("not found")
|
||||
handler := NewFileProcessingJobQueueHandler(stubFileProcessingProcessor{
|
||||
err: permanentErr,
|
||||
}, func(err error) bool {
|
||||
return errors.Is(err, permanentErr)
|
||||
})
|
||||
|
||||
err = handler.Handle(context.Background(), &FileProcessingEnvelope{
|
||||
MessageID: "msg-4",
|
||||
Payload: FileProcessingJob{
|
||||
FileUUID: fileUUID,
|
||||
Bucket: "bucket-a",
|
||||
ObjectKey: "files/a.pdf",
|
||||
SizeBytes: 12,
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("expected error")
|
||||
}
|
||||
if !IsPermanent(err) {
|
||||
t.Fatalf("expected permanent error")
|
||||
}
|
||||
}
|
||||
112
internal/queue/file_processing_serializer.go
Normal file
112
internal/queue/file_processing_serializer.go
Normal file
@@ -0,0 +1,112 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type FileProcessingJSONSerializer struct {
|
||||
version string
|
||||
queueType string
|
||||
messageGroupID string
|
||||
}
|
||||
|
||||
func NewFileProcessingJSONSerializer(version, queueType, messageGroupID string) *FileProcessingJSONSerializer {
|
||||
version = strings.TrimSpace(version)
|
||||
if version == "" {
|
||||
version = "v1"
|
||||
}
|
||||
queueType = strings.ToLower(strings.TrimSpace(queueType))
|
||||
if queueType == "" {
|
||||
queueType = QueueTypeStandard
|
||||
}
|
||||
return &FileProcessingJSONSerializer{
|
||||
version: version,
|
||||
queueType: queueType,
|
||||
messageGroupID: strings.TrimSpace(messageGroupID),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *FileProcessingJSONSerializer) Serialize(ctx context.Context, job FileProcessingJob) (*OutboundMessage, error) {
|
||||
envelope, err := NewFileProcessingEnvelope(ctx, s.version, job)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := json.Marshal(envelope)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
message := &OutboundMessage{
|
||||
ID: envelope.MessageID,
|
||||
Body: body,
|
||||
Attributes: map[string]string{
|
||||
"message_kind": envelope.Kind,
|
||||
"message_id": envelope.MessageID,
|
||||
"correlation_id": envelope.CorrelationID,
|
||||
"trace_id": envelope.TraceID,
|
||||
"schema_version": envelope.Version,
|
||||
"job_type": "file_processing",
|
||||
"file_uuid": envelope.Payload.FileUUID,
|
||||
},
|
||||
}
|
||||
if s.queueType == QueueTypeFIFO {
|
||||
message.MessageGroupID = s.messageGroupID
|
||||
if message.MessageGroupID == "" {
|
||||
message.MessageGroupID = "file-processing"
|
||||
}
|
||||
message.DeduplicationID = envelope.DeduplicationKey()
|
||||
}
|
||||
return message, nil
|
||||
}
|
||||
|
||||
func (s *FileProcessingJSONSerializer) Deserialize(body []byte, _ map[string]string) (*FileProcessingEnvelope, error) {
|
||||
var envelope FileProcessingEnvelope
|
||||
if err := json.Unmarshal(body, &envelope); err != nil {
|
||||
return nil, errors.Join(ErrInvalidFileProcessingEnvelope, err)
|
||||
}
|
||||
if err := validateFileProcessingEnvelope(&envelope); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &envelope, nil
|
||||
}
|
||||
|
||||
func validateFileProcessingEnvelope(envelope *FileProcessingEnvelope) error {
|
||||
if envelope == nil {
|
||||
return ErrInvalidFileProcessingEnvelope
|
||||
}
|
||||
if strings.TrimSpace(envelope.Version) == "" {
|
||||
return errors.Join(ErrInvalidFileProcessingEnvelope, errors.New("version is required"))
|
||||
}
|
||||
if strings.TrimSpace(envelope.MessageID) == "" {
|
||||
return errors.Join(ErrInvalidFileProcessingEnvelope, errors.New("message id is required"))
|
||||
}
|
||||
if strings.TrimSpace(envelope.Kind) == "" {
|
||||
return errors.Join(ErrInvalidFileProcessingEnvelope, errors.New("kind is required"))
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(envelope.Kind), FileProcessingKind) {
|
||||
return errors.Join(ErrInvalidFileProcessingEnvelope, errors.New("unsupported message kind"))
|
||||
}
|
||||
if envelope.OccurredAt.IsZero() {
|
||||
return errors.Join(ErrInvalidFileProcessingEnvelope, errors.New("occurred at is required"))
|
||||
}
|
||||
envelope.Payload = envelope.Payload.Canonical()
|
||||
if err := envelope.Payload.Validate(); err != nil {
|
||||
return errors.Join(ErrInvalidFileProcessingEnvelope, err)
|
||||
}
|
||||
if strings.TrimSpace(envelope.CorrelationID) == "" {
|
||||
envelope.CorrelationID = envelope.MessageID
|
||||
}
|
||||
if strings.TrimSpace(envelope.TraceID) == "" {
|
||||
envelope.TraceID = envelope.CorrelationID
|
||||
}
|
||||
if strings.TrimSpace(envelope.IdempotencyKey) == "" {
|
||||
envelope.IdempotencyKey = envelope.MessageID
|
||||
}
|
||||
if envelope.Payload.CreatedAt.IsZero() {
|
||||
envelope.Payload.CreatedAt = time.Now().UTC()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
53
internal/queue/file_processing_serializer_test.go
Normal file
53
internal/queue/file_processing_serializer_test.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"wucher/internal/shared/pkg/uuidv7"
|
||||
)
|
||||
|
||||
func TestFileProcessingJSONSerializerRoundTrip(t *testing.T) {
|
||||
serializer := NewFileProcessingJSONSerializer("v1", QueueTypeStandard, "")
|
||||
fileUUID, err := uuidv7.String(uuidv7.MustNew())
|
||||
if err != nil {
|
||||
t.Fatalf("uuid string: %v", err)
|
||||
}
|
||||
|
||||
ctx := WithCorrelationID(context.Background(), "upload-req-1")
|
||||
message, err := serializer.Serialize(ctx, FileProcessingJob{
|
||||
FileUUID: fileUUID,
|
||||
Bucket: "wucher-file-dev",
|
||||
ObjectKey: "files/2026/04/03/a.pdf",
|
||||
MimeType: "application/pdf",
|
||||
SizeBytes: 1234,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("serialize: %v", err)
|
||||
}
|
||||
|
||||
envelope, err := serializer.Deserialize(message.Body, message.Attributes)
|
||||
if err != nil {
|
||||
t.Fatalf("deserialize: %v", err)
|
||||
}
|
||||
if envelope.MessageID == "" {
|
||||
t.Fatalf("expected message id")
|
||||
}
|
||||
if envelope.CorrelationID != "upload-req-1" {
|
||||
t.Fatalf("unexpected correlation id: %q", envelope.CorrelationID)
|
||||
}
|
||||
if envelope.Payload.FileUUID != fileUUID {
|
||||
t.Fatalf("unexpected file uuid: %q", envelope.Payload.FileUUID)
|
||||
}
|
||||
if envelope.Payload.ObjectKey != "files/2026/04/03/a.pdf" {
|
||||
t.Fatalf("unexpected object key: %q", envelope.Payload.ObjectKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileProcessingJSONSerializerRejectsInvalidPayload(t *testing.T) {
|
||||
serializer := NewFileProcessingJSONSerializer("v1", QueueTypeStandard, "")
|
||||
_, err := serializer.Deserialize([]byte(`{"version":"v1","message_id":"m1","kind":"file_manager.processing","occurred_at":"2026-04-03T00:00:00Z","payload":{"bucket":"b","object_key":"k","size_bytes":1}}`), nil)
|
||||
if err == nil {
|
||||
t.Fatalf("expected invalid payload error")
|
||||
}
|
||||
}
|
||||
141
internal/queue/file_processing_types.go
Normal file
141
internal/queue/file_processing_types.go
Normal file
@@ -0,0 +1,141 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"wucher/internal/shared/pkg/uuidv7"
|
||||
)
|
||||
|
||||
const (
|
||||
FileProcessingKind = "file_manager.processing"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidFileProcessingJob = errors.New("invalid file processing job")
|
||||
ErrInvalidFileProcessingEnvelope = errors.New("invalid file processing envelope")
|
||||
)
|
||||
|
||||
type FileProcessingJob struct {
|
||||
JobID string `json:"job_id,omitempty"`
|
||||
FileUUID string `json:"file_uuid"`
|
||||
Bucket string `json:"bucket"`
|
||||
ObjectKey string `json:"object_key"`
|
||||
UploadedBy string `json:"uploaded_by,omitempty"`
|
||||
MimeType string `json:"mime_type,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
CreatedAt time.Time `json:"created_at,omitempty"`
|
||||
TraceID string `json:"trace_id,omitempty"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
}
|
||||
|
||||
func (j FileProcessingJob) Canonical() FileProcessingJob {
|
||||
job := j
|
||||
job.JobID = strings.TrimSpace(job.JobID)
|
||||
job.FileUUID = strings.TrimSpace(job.FileUUID)
|
||||
job.Bucket = strings.TrimSpace(job.Bucket)
|
||||
job.ObjectKey = strings.TrimSpace(job.ObjectKey)
|
||||
job.UploadedBy = strings.TrimSpace(job.UploadedBy)
|
||||
job.MimeType = strings.ToLower(strings.TrimSpace(job.MimeType))
|
||||
job.TraceID = strings.TrimSpace(job.TraceID)
|
||||
job.RequestID = strings.TrimSpace(job.RequestID)
|
||||
return job
|
||||
}
|
||||
|
||||
func (j FileProcessingJob) Validate() error {
|
||||
job := j.Canonical()
|
||||
if job.FileUUID == "" {
|
||||
return errors.Join(ErrInvalidFileProcessingJob, errors.New("file uuid is required"))
|
||||
}
|
||||
if _, err := uuidv7.ParseString(job.FileUUID); err != nil {
|
||||
return errors.Join(ErrInvalidFileProcessingJob, errors.New("file uuid is invalid"))
|
||||
}
|
||||
if job.Bucket == "" {
|
||||
return errors.Join(ErrInvalidFileProcessingJob, errors.New("bucket is required"))
|
||||
}
|
||||
if job.ObjectKey == "" {
|
||||
return errors.Join(ErrInvalidFileProcessingJob, errors.New("object key is required"))
|
||||
}
|
||||
if job.SizeBytes < 0 {
|
||||
return errors.Join(ErrInvalidFileProcessingJob, errors.New("size bytes must be non-negative"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type FileProcessingEnvelope struct {
|
||||
Version string `json:"version"`
|
||||
MessageID string `json:"message_id"`
|
||||
Kind string `json:"kind"`
|
||||
OccurredAt time.Time `json:"occurred_at"`
|
||||
CorrelationID string `json:"correlation_id,omitempty"`
|
||||
TraceID string `json:"trace_id,omitempty"`
|
||||
IdempotencyKey string `json:"idempotency_key,omitempty"`
|
||||
Payload FileProcessingJob `json:"payload"`
|
||||
}
|
||||
|
||||
func NewFileProcessingEnvelope(ctx context.Context, version string, job FileProcessingJob) (*FileProcessingEnvelope, error) {
|
||||
job = job.Canonical()
|
||||
if err := job.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
messageID, err := newMessageID()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
correlationID := CorrelationIDFromContext(ctx)
|
||||
if correlationID == "" {
|
||||
correlationID = messageID
|
||||
}
|
||||
traceID := TraceIDFromContext(ctx)
|
||||
if traceID == "" {
|
||||
traceID = correlationID
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if strings.TrimSpace(version) == "" {
|
||||
version = "v1"
|
||||
}
|
||||
if job.JobID == "" {
|
||||
job.JobID = messageID
|
||||
}
|
||||
if job.CreatedAt.IsZero() {
|
||||
job.CreatedAt = now
|
||||
}
|
||||
if job.TraceID == "" {
|
||||
job.TraceID = traceID
|
||||
}
|
||||
if job.RequestID == "" {
|
||||
job.RequestID = correlationID
|
||||
}
|
||||
return &FileProcessingEnvelope{
|
||||
Version: version,
|
||||
MessageID: messageID,
|
||||
Kind: FileProcessingKind,
|
||||
OccurredAt: now,
|
||||
CorrelationID: correlationID,
|
||||
TraceID: traceID,
|
||||
IdempotencyKey: job.JobID,
|
||||
Payload: job,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e FileProcessingEnvelope) DeduplicationKey() string {
|
||||
if v := strings.TrimSpace(e.IdempotencyKey); v != "" {
|
||||
return v
|
||||
}
|
||||
return strings.TrimSpace(e.MessageID)
|
||||
}
|
||||
|
||||
type FileProcessingMessageSerializer interface {
|
||||
Serialize(ctx context.Context, job FileProcessingJob) (*OutboundMessage, error)
|
||||
Deserialize(body []byte, attributes map[string]string) (*FileProcessingEnvelope, error)
|
||||
}
|
||||
|
||||
type FileProcessingJobProducer interface {
|
||||
Enqueue(ctx context.Context, job FileProcessingJob) error
|
||||
}
|
||||
|
||||
type FileProcessingJobHandler interface {
|
||||
Handle(ctx context.Context, envelope *FileProcessingEnvelope) error
|
||||
}
|
||||
299
internal/queue/file_processing_worker.go
Normal file
299
internal/queue/file_processing_worker.go
Normal file
@@ -0,0 +1,299 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"wucher/internal/shared/pkg/metrics"
|
||||
)
|
||||
|
||||
type FileProcessingWorker struct {
|
||||
cfg WorkerConfig
|
||||
consumer QueueConsumer
|
||||
serializer FileProcessingMessageSerializer
|
||||
handler FileProcessingJobHandler
|
||||
retry RetryPolicy
|
||||
idem IdempotencyStore
|
||||
metrics *Metrics
|
||||
health *HealthReporter
|
||||
logger *slog.Logger
|
||||
component string
|
||||
}
|
||||
|
||||
type fileProcessResult struct {
|
||||
delivery *Delivery
|
||||
envelope *FileProcessingEnvelope
|
||||
decision RetryDecision
|
||||
err error
|
||||
}
|
||||
|
||||
func NewFileProcessingWorker(
|
||||
cfg WorkerConfig,
|
||||
consumer QueueConsumer,
|
||||
serializer FileProcessingMessageSerializer,
|
||||
handler FileProcessingJobHandler,
|
||||
retry RetryPolicy,
|
||||
idem IdempotencyStore,
|
||||
metrics *Metrics,
|
||||
health *HealthReporter,
|
||||
logger *slog.Logger,
|
||||
) *FileProcessingWorker {
|
||||
if cfg.Concurrency <= 0 {
|
||||
cfg.Concurrency = 1
|
||||
}
|
||||
if cfg.MaxBatchSize <= 0 || cfg.MaxBatchSize > cfg.Concurrency {
|
||||
cfg.MaxBatchSize = cfg.Concurrency
|
||||
}
|
||||
if cfg.ProcessTimeout <= 0 {
|
||||
cfg.ProcessTimeout = 30 * time.Second
|
||||
}
|
||||
if cfg.ProcessingTTL <= 0 {
|
||||
cfg.ProcessingTTL = 15 * time.Minute
|
||||
}
|
||||
if cfg.IdempotencyTTL <= 0 {
|
||||
cfg.IdempotencyTTL = 24 * time.Hour
|
||||
}
|
||||
if cfg.PollErrorDelay <= 0 {
|
||||
cfg.PollErrorDelay = time.Second
|
||||
}
|
||||
if retry == nil {
|
||||
retry = NewExponentialBackoffPolicy(time.Second, 5*time.Minute, 30*time.Second)
|
||||
}
|
||||
if idem == nil {
|
||||
idem = NoopIdempotencyStore{}
|
||||
}
|
||||
if metrics == nil {
|
||||
metrics = NewMetrics()
|
||||
}
|
||||
if health == nil {
|
||||
health = NewHealthReporter(3)
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &FileProcessingWorker{
|
||||
cfg: cfg,
|
||||
consumer: consumer,
|
||||
serializer: serializer,
|
||||
handler: handler,
|
||||
retry: retry,
|
||||
idem: idem,
|
||||
metrics: metrics,
|
||||
health: health,
|
||||
logger: logger,
|
||||
component: normalizeWorkerMetricsComponent(cfg.MetricsComponent),
|
||||
}
|
||||
}
|
||||
|
||||
func (w *FileProcessingWorker) Run(ctx context.Context) error {
|
||||
if w.consumer == nil {
|
||||
return errors.New("queue consumer is required")
|
||||
}
|
||||
if w.serializer == nil {
|
||||
return errors.New("file message serializer is required")
|
||||
}
|
||||
if w.handler == nil {
|
||||
return errors.New("file message handler is required")
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
w.health.MarkDraining()
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
|
||||
deliveries, err := w.consumer.Receive(ctx, w.cfg.MaxBatchSize)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
w.health.MarkDraining()
|
||||
return nil
|
||||
}
|
||||
w.health.MarkPollFailure(err)
|
||||
w.logger.Error("file queue receive failed",
|
||||
slog.String("backend", w.consumer.Backend()),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
w.health.MarkDraining()
|
||||
return nil
|
||||
case <-time.After(w.cfg.PollErrorDelay):
|
||||
}
|
||||
continue
|
||||
}
|
||||
w.health.MarkPollSuccess()
|
||||
if len(deliveries) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
w.metrics.RecordReceive(len(deliveries))
|
||||
results := make(chan fileProcessResult, len(deliveries))
|
||||
for _, delivery := range deliveries {
|
||||
delivery := delivery
|
||||
go func() {
|
||||
results <- w.processDelivery(ctx, delivery)
|
||||
}()
|
||||
}
|
||||
|
||||
acks := make([]*Delivery, 0, len(deliveries))
|
||||
for range deliveries {
|
||||
result := <-results
|
||||
switch result.decision.Action {
|
||||
case RetryActionAck:
|
||||
acks = append(acks, result.delivery)
|
||||
case RetryActionRetry:
|
||||
if err := w.consumer.Retry(ctx, result.delivery, result.decision.Delay); err != nil {
|
||||
w.metrics.RecordRetryFailure()
|
||||
w.logger.Error("file queue retry scheduling failed",
|
||||
slog.String("backend", w.consumer.Backend()),
|
||||
slog.String("message_id", result.delivery.ID),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
} else {
|
||||
w.metrics.RecordRetry()
|
||||
}
|
||||
case RetryActionDeadLetter:
|
||||
if err := w.consumer.DeadLetter(ctx, result.delivery, result.decision.Reason, result.err); err != nil {
|
||||
w.metrics.RecordDeadLetterFailure()
|
||||
w.logger.Error("file queue dead-letter failed",
|
||||
slog.String("backend", w.consumer.Backend()),
|
||||
slog.String("message_id", result.delivery.ID),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
} else {
|
||||
w.metrics.RecordDeadLetter()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(acks) > 0 {
|
||||
if err := w.consumer.Ack(ctx, acks); err != nil {
|
||||
w.metrics.RecordAckFailure()
|
||||
w.logger.Error("file queue ack failed",
|
||||
slog.String("backend", w.consumer.Backend()),
|
||||
slog.Int("count", len(acks)),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *FileProcessingWorker) RefreshQueueDepth(ctx context.Context) error {
|
||||
if w.consumer == nil {
|
||||
return errors.New("queue consumer is required")
|
||||
}
|
||||
depth, err := w.consumer.ApproximateDepth(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.metrics.SetQueueDepth(depth)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *FileProcessingWorker) Metrics() *Metrics {
|
||||
return w.metrics
|
||||
}
|
||||
|
||||
func (w *FileProcessingWorker) Health() *HealthReporter {
|
||||
return w.health
|
||||
}
|
||||
|
||||
func (w *FileProcessingWorker) processDelivery(ctx context.Context, delivery *Delivery) (result fileProcessResult) {
|
||||
started := time.Now()
|
||||
var jobErr error
|
||||
defer func() {
|
||||
metrics.ObserveWorkerJob(w.component, "file_processing", jobErr, started)
|
||||
}()
|
||||
|
||||
envelope, err := w.serializer.Deserialize(delivery.Body, delivery.Attributes)
|
||||
if err != nil {
|
||||
w.metrics.RecordMalformed()
|
||||
w.metrics.RecordFailure()
|
||||
jobErr = err
|
||||
return fileProcessResult{
|
||||
delivery: delivery,
|
||||
decision: w.retry.Decide(delivery, nil, Permanent(err)),
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
|
||||
idempotencyKey := envelope.DeduplicationKey()
|
||||
state, err := w.idem.Acquire(ctx, idempotencyKey, w.cfg.ProcessingTTL)
|
||||
if err != nil {
|
||||
w.metrics.RecordFailure()
|
||||
jobErr = err
|
||||
return fileProcessResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: w.retry.Decide(delivery, nil, Transient(err)),
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
switch state {
|
||||
case IdempotencyStateDuplicate:
|
||||
w.metrics.RecordDuplicate()
|
||||
w.logger.Info("file queue duplicate skipped",
|
||||
slog.String("message_id", envelope.MessageID),
|
||||
slog.String("file_uuid", envelope.Payload.FileUUID),
|
||||
slog.String("correlation_id", envelope.CorrelationID),
|
||||
)
|
||||
return fileProcessResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: RetryDecision{Action: RetryActionAck},
|
||||
}
|
||||
case IdempotencyStateInProgress:
|
||||
inProgressErr := Transient(errors.New("message already in progress"))
|
||||
jobErr = inProgressErr
|
||||
return fileProcessResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: w.retry.Decide(delivery, nil, inProgressErr),
|
||||
err: inProgressErr,
|
||||
}
|
||||
}
|
||||
|
||||
processCtx, cancel := context.WithTimeout(ctx, w.cfg.ProcessTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := w.handler.Handle(processCtx, envelope); err != nil {
|
||||
_ = w.idem.Release(ctx, idempotencyKey)
|
||||
w.metrics.RecordFailure()
|
||||
jobErr = err
|
||||
return fileProcessResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: w.retry.Decide(delivery, nil, err),
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
|
||||
if err := w.idem.MarkCompleted(ctx, idempotencyKey, w.cfg.IdempotencyTTL); err != nil {
|
||||
w.metrics.RecordFailure()
|
||||
w.logger.Error("file queue idempotency completion failed",
|
||||
slog.String("message_id", envelope.MessageID),
|
||||
slog.String("file_uuid", envelope.Payload.FileUUID),
|
||||
slog.String("correlation_id", envelope.CorrelationID),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
jobErr = err
|
||||
return fileProcessResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: w.retry.Decide(delivery, nil, Transient(err)),
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
|
||||
w.metrics.RecordProcessed(time.Since(started))
|
||||
jobErr = nil
|
||||
return fileProcessResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: RetryDecision{Action: RetryActionAck},
|
||||
}
|
||||
}
|
||||
74
internal/queue/file_processing_worker_test.go
Normal file
74
internal/queue/file_processing_worker_test.go
Normal file
@@ -0,0 +1,74 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"wucher/internal/config"
|
||||
"wucher/internal/shared/pkg/logger"
|
||||
"wucher/internal/shared/pkg/uuidv7"
|
||||
)
|
||||
|
||||
type stubFileProcessingHandler struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (h stubFileProcessingHandler) Handle(context.Context, *FileProcessingEnvelope) error {
|
||||
return h.err
|
||||
}
|
||||
|
||||
func TestFileProcessingWorkerProcessDeliveryDuplicate(t *testing.T) {
|
||||
worker := NewFileProcessingWorker(WorkerConfig{}, nil, NewFileProcessingJSONSerializer("v1", QueueTypeStandard, ""), stubFileProcessingHandler{}, nil, stubIdempotencyStore{state: IdempotencyStateDuplicate}, nil, nil, logger.New(config.LoggingConfig{Format: "json"}))
|
||||
fileUUID, _ := uuidv7.String(uuidv7.MustNew())
|
||||
message, err := NewFileProcessingJSONSerializer("v1", QueueTypeStandard, "").Serialize(context.Background(), FileProcessingJob{
|
||||
FileUUID: fileUUID,
|
||||
Bucket: "bucket",
|
||||
ObjectKey: "files/doc.pdf",
|
||||
SizeBytes: 10,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("serialize: %v", err)
|
||||
}
|
||||
|
||||
result := worker.processDelivery(context.Background(), &Delivery{ID: message.ID, Body: message.Body, Attributes: message.Attributes, ReceiveCount: 1})
|
||||
if result.decision.Action != RetryActionAck {
|
||||
t.Fatalf("expected ack decision, got %s", result.decision.Action)
|
||||
}
|
||||
if worker.Metrics().Snapshot().Duplicates != 1 {
|
||||
t.Fatalf("expected duplicate metric to increment")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileProcessingWorkerProcessDeliveryMalformedPayload(t *testing.T) {
|
||||
worker := NewFileProcessingWorker(WorkerConfig{}, nil, NewFileProcessingJSONSerializer("v1", QueueTypeStandard, ""), stubFileProcessingHandler{}, nil, stubIdempotencyStore{}, nil, nil, logger.New(config.LoggingConfig{Format: "json"}))
|
||||
result := worker.processDelivery(context.Background(), &Delivery{ID: "msg-1", Body: []byte(`{"bad":true}`), ReceiveCount: 1})
|
||||
if result.decision.Action != RetryActionDeadLetter {
|
||||
t.Fatalf("expected dead-letter decision, got %s", result.decision.Action)
|
||||
}
|
||||
if worker.Metrics().Snapshot().Malformed != 1 {
|
||||
t.Fatalf("expected malformed metric to increment")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileProcessingWorkerProcessDeliveryTransientError(t *testing.T) {
|
||||
worker := NewFileProcessingWorker(WorkerConfig{}, nil, NewFileProcessingJSONSerializer("v1", QueueTypeStandard, ""), stubFileProcessingHandler{err: Transient(errors.New("db timeout"))}, nil, stubIdempotencyStore{}, nil, nil, logger.New(config.LoggingConfig{Format: "json"}))
|
||||
fileUUID, _ := uuidv7.String(uuidv7.MustNew())
|
||||
message, err := NewFileProcessingJSONSerializer("v1", QueueTypeStandard, "").Serialize(context.Background(), FileProcessingJob{
|
||||
FileUUID: fileUUID,
|
||||
Bucket: "bucket",
|
||||
ObjectKey: "files/doc.pdf",
|
||||
SizeBytes: 10,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("serialize: %v", err)
|
||||
}
|
||||
|
||||
result := worker.processDelivery(context.Background(), &Delivery{ID: message.ID, Body: message.Body, Attributes: message.Attributes, ReceiveCount: 2})
|
||||
if result.decision.Action != RetryActionRetry {
|
||||
t.Fatalf("expected retry decision, got %s", result.decision.Action)
|
||||
}
|
||||
if result.decision.Delay <= 0 {
|
||||
t.Fatalf("expected positive retry delay")
|
||||
}
|
||||
}
|
||||
90
internal/queue/health.go
Normal file
90
internal/queue/health.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type HealthReporter struct {
|
||||
mu sync.RWMutex
|
||||
startedAt time.Time
|
||||
lastPollAt time.Time
|
||||
lastSuccessAt time.Time
|
||||
lastFailureAt time.Time
|
||||
lastError string
|
||||
consecutivePollFailures int
|
||||
errorThreshold int
|
||||
draining bool
|
||||
}
|
||||
|
||||
type HealthSnapshot struct {
|
||||
Status string `json:"status"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
LastPollAt time.Time `json:"last_poll_at,omitempty"`
|
||||
LastSuccessAt time.Time `json:"last_success_at,omitempty"`
|
||||
LastFailureAt time.Time `json:"last_failure_at,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
ConsecutivePollFailure int `json:"consecutive_poll_failures"`
|
||||
Draining bool `json:"draining"`
|
||||
}
|
||||
|
||||
func NewHealthReporter(errorThreshold int) *HealthReporter {
|
||||
if errorThreshold <= 0 {
|
||||
errorThreshold = 3
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
return &HealthReporter{
|
||||
startedAt: now,
|
||||
lastPollAt: now,
|
||||
errorThreshold: errorThreshold,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *HealthReporter) MarkPollSuccess() {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
now := time.Now().UTC()
|
||||
r.lastPollAt = now
|
||||
r.lastSuccessAt = now
|
||||
r.consecutivePollFailures = 0
|
||||
r.lastError = ""
|
||||
}
|
||||
|
||||
func (r *HealthReporter) MarkPollFailure(err error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
now := time.Now().UTC()
|
||||
r.lastPollAt = now
|
||||
r.lastFailureAt = now
|
||||
r.consecutivePollFailures++
|
||||
if err != nil {
|
||||
r.lastError = err.Error()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *HealthReporter) MarkDraining() {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.draining = true
|
||||
}
|
||||
|
||||
func (r *HealthReporter) Snapshot() HealthSnapshot {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
status := "ok"
|
||||
if r.draining {
|
||||
status = "draining"
|
||||
} else if r.consecutivePollFailures >= r.errorThreshold {
|
||||
status = "degraded"
|
||||
}
|
||||
return HealthSnapshot{
|
||||
Status: status,
|
||||
StartedAt: r.startedAt,
|
||||
LastPollAt: r.lastPollAt,
|
||||
LastSuccessAt: r.lastSuccessAt,
|
||||
LastFailureAt: r.lastFailureAt,
|
||||
LastError: r.lastError,
|
||||
ConsecutivePollFailure: r.consecutivePollFailures,
|
||||
Draining: r.draining,
|
||||
}
|
||||
}
|
||||
95
internal/queue/idempotency.go
Normal file
95
internal/queue/idempotency.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type IdempotencyState string
|
||||
|
||||
const (
|
||||
IdempotencyStateAcquired IdempotencyState = "acquired"
|
||||
IdempotencyStateDuplicate IdempotencyState = "duplicate"
|
||||
IdempotencyStateInProgress IdempotencyState = "in_progress"
|
||||
)
|
||||
|
||||
type IdempotencyStore interface {
|
||||
Acquire(ctx context.Context, key string, processingTTL time.Duration) (IdempotencyState, error)
|
||||
MarkCompleted(ctx context.Context, key string, ttl time.Duration) error
|
||||
Release(ctx context.Context, key string) error
|
||||
}
|
||||
|
||||
type NoopIdempotencyStore struct{}
|
||||
|
||||
func (NoopIdempotencyStore) Acquire(context.Context, string, time.Duration) (IdempotencyState, error) {
|
||||
return IdempotencyStateAcquired, nil
|
||||
}
|
||||
|
||||
func (NoopIdempotencyStore) MarkCompleted(context.Context, string, time.Duration) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (NoopIdempotencyStore) Release(context.Context, string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type MemoryIdempotencyStore struct {
|
||||
mu sync.Mutex
|
||||
records map[string]memoryIdempotencyRecord
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type memoryIdempotencyRecord struct {
|
||||
state IdempotencyState
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
func NewMemoryIdempotencyStore() *MemoryIdempotencyStore {
|
||||
return &MemoryIdempotencyStore{
|
||||
records: make(map[string]memoryIdempotencyRecord),
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MemoryIdempotencyStore) Acquire(_ context.Context, key string, processingTTL time.Duration) (IdempotencyState, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.cleanupExpiredLocked()
|
||||
if record, ok := s.records[key]; ok {
|
||||
return record.state, nil
|
||||
}
|
||||
s.records[key] = memoryIdempotencyRecord{
|
||||
state: IdempotencyStateInProgress,
|
||||
expiresAt: s.now().Add(processingTTL),
|
||||
}
|
||||
return IdempotencyStateAcquired, nil
|
||||
}
|
||||
|
||||
func (s *MemoryIdempotencyStore) MarkCompleted(_ context.Context, key string, ttl time.Duration) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.records[key] = memoryIdempotencyRecord{
|
||||
state: IdempotencyStateDuplicate,
|
||||
expiresAt: s.now().Add(ttl),
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MemoryIdempotencyStore) Release(_ context.Context, key string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
delete(s.records, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MemoryIdempotencyStore) cleanupExpiredLocked() {
|
||||
now := s.now()
|
||||
for key, record := range s.records {
|
||||
if !record.expiresAt.IsZero() && now.After(record.expiresAt) {
|
||||
delete(s.records, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
112
internal/queue/metrics.go
Normal file
112
internal/queue/metrics.go
Normal file
@@ -0,0 +1,112 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Metrics struct {
|
||||
received atomic.Int64
|
||||
processed atomic.Int64
|
||||
retried atomic.Int64
|
||||
failed atomic.Int64
|
||||
deadLettered atomic.Int64
|
||||
duplicates atomic.Int64
|
||||
malformed atomic.Int64
|
||||
ackFailures atomic.Int64
|
||||
retryFailures atomic.Int64
|
||||
deadLetterFailures atomic.Int64
|
||||
lastQueueDepth atomic.Int64
|
||||
totalProcessingNanos atomic.Int64
|
||||
lastProcessingNanos atomic.Int64
|
||||
}
|
||||
|
||||
type MetricsSnapshot struct {
|
||||
Received int64 `json:"received"`
|
||||
Processed int64 `json:"processed"`
|
||||
Retried int64 `json:"retried"`
|
||||
Failed int64 `json:"failed"`
|
||||
DeadLettered int64 `json:"dead_lettered"`
|
||||
Duplicates int64 `json:"duplicates"`
|
||||
Malformed int64 `json:"malformed"`
|
||||
AckFailures int64 `json:"ack_failures"`
|
||||
RetryFailures int64 `json:"retry_failures"`
|
||||
DeadLetterFailures int64 `json:"dead_letter_failures"`
|
||||
ApproximateQueueDepth int64 `json:"approximate_queue_depth"`
|
||||
LastProcessingLatency time.Duration `json:"last_processing_latency"`
|
||||
AverageProcessingLatency time.Duration `json:"average_processing_latency"`
|
||||
}
|
||||
|
||||
func NewMetrics() *Metrics {
|
||||
return &Metrics{}
|
||||
}
|
||||
|
||||
func (m *Metrics) RecordReceive(n int) {
|
||||
m.received.Add(int64(n))
|
||||
}
|
||||
|
||||
func (m *Metrics) RecordProcessed(duration time.Duration) {
|
||||
m.processed.Add(1)
|
||||
m.lastProcessingNanos.Store(duration.Nanoseconds())
|
||||
m.totalProcessingNanos.Add(duration.Nanoseconds())
|
||||
}
|
||||
|
||||
func (m *Metrics) RecordRetry() {
|
||||
m.retried.Add(1)
|
||||
}
|
||||
|
||||
func (m *Metrics) RecordFailure() {
|
||||
m.failed.Add(1)
|
||||
}
|
||||
|
||||
func (m *Metrics) RecordDeadLetter() {
|
||||
m.deadLettered.Add(1)
|
||||
}
|
||||
|
||||
func (m *Metrics) RecordDuplicate() {
|
||||
m.duplicates.Add(1)
|
||||
}
|
||||
|
||||
func (m *Metrics) RecordMalformed() {
|
||||
m.malformed.Add(1)
|
||||
}
|
||||
|
||||
func (m *Metrics) RecordAckFailure() {
|
||||
m.ackFailures.Add(1)
|
||||
}
|
||||
|
||||
func (m *Metrics) RecordRetryFailure() {
|
||||
m.retryFailures.Add(1)
|
||||
}
|
||||
|
||||
func (m *Metrics) RecordDeadLetterFailure() {
|
||||
m.deadLetterFailures.Add(1)
|
||||
}
|
||||
|
||||
func (m *Metrics) SetQueueDepth(depth int64) {
|
||||
m.lastQueueDepth.Store(depth)
|
||||
}
|
||||
|
||||
func (m *Metrics) Snapshot() MetricsSnapshot {
|
||||
processed := m.processed.Load()
|
||||
total := m.totalProcessingNanos.Load()
|
||||
avg := time.Duration(0)
|
||||
if processed > 0 {
|
||||
avg = time.Duration(total / processed)
|
||||
}
|
||||
return MetricsSnapshot{
|
||||
Received: m.received.Load(),
|
||||
Processed: processed,
|
||||
Retried: m.retried.Load(),
|
||||
Failed: m.failed.Load(),
|
||||
DeadLettered: m.deadLettered.Load(),
|
||||
Duplicates: m.duplicates.Load(),
|
||||
Malformed: m.malformed.Load(),
|
||||
AckFailures: m.ackFailures.Load(),
|
||||
RetryFailures: m.retryFailures.Load(),
|
||||
DeadLetterFailures: m.deadLetterFailures.Load(),
|
||||
ApproximateQueueDepth: m.lastQueueDepth.Load(),
|
||||
LastProcessingLatency: time.Duration(m.lastProcessingNanos.Load()),
|
||||
AverageProcessingLatency: avg,
|
||||
}
|
||||
}
|
||||
106
internal/queue/retry.go
Normal file
106
internal/queue/retry.go
Normal file
@@ -0,0 +1,106 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"hash/fnv"
|
||||
"math"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RetryAction string
|
||||
|
||||
const (
|
||||
RetryActionRetry RetryAction = "retry"
|
||||
RetryActionDeadLetter RetryAction = "dead_letter"
|
||||
RetryActionAck RetryAction = "ack"
|
||||
)
|
||||
|
||||
type RetryDecision struct {
|
||||
Action RetryAction
|
||||
Delay time.Duration
|
||||
Reason string
|
||||
}
|
||||
|
||||
type RetryPolicy interface {
|
||||
Decide(delivery *Delivery, envelope *Envelope, err error) RetryDecision
|
||||
}
|
||||
|
||||
type ExponentialBackoffPolicy struct {
|
||||
minDelay time.Duration
|
||||
maxDelay time.Duration
|
||||
permanentFailureDelay time.Duration
|
||||
}
|
||||
|
||||
func NewExponentialBackoffPolicy(minDelay, maxDelay, permanentFailureDelay time.Duration) *ExponentialBackoffPolicy {
|
||||
if minDelay <= 0 {
|
||||
minDelay = time.Second
|
||||
}
|
||||
if maxDelay < minDelay {
|
||||
maxDelay = minDelay
|
||||
}
|
||||
if permanentFailureDelay <= 0 {
|
||||
permanentFailureDelay = 30 * time.Second
|
||||
}
|
||||
return &ExponentialBackoffPolicy{
|
||||
minDelay: minDelay,
|
||||
maxDelay: maxDelay,
|
||||
permanentFailureDelay: permanentFailureDelay,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ExponentialBackoffPolicy) Decide(delivery *Delivery, _ *Envelope, err error) RetryDecision {
|
||||
if err == nil {
|
||||
return RetryDecision{Action: RetryActionAck}
|
||||
}
|
||||
if IsPermanent(err) {
|
||||
return RetryDecision{
|
||||
Action: RetryActionDeadLetter,
|
||||
Delay: p.permanentFailureDelay,
|
||||
Reason: err.Error(),
|
||||
}
|
||||
}
|
||||
attempt := 1
|
||||
if delivery != nil && delivery.ReceiveCount > 0 {
|
||||
attempt = delivery.ReceiveCount
|
||||
}
|
||||
delay := p.backoffWithJitter(deliveryID(delivery), attempt)
|
||||
return RetryDecision{
|
||||
Action: RetryActionRetry,
|
||||
Delay: delay,
|
||||
Reason: err.Error(),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ExponentialBackoffPolicy) backoffWithJitter(seed string, attempt int) time.Duration {
|
||||
if attempt < 1 {
|
||||
attempt = 1
|
||||
}
|
||||
power := math.Pow(2, float64(attempt-1))
|
||||
delay := time.Duration(float64(p.minDelay) * power)
|
||||
if delay > p.maxDelay {
|
||||
delay = p.maxDelay
|
||||
}
|
||||
if delay <= 0 {
|
||||
return p.minDelay
|
||||
}
|
||||
jitter := deterministicJitter(seed, delay/2)
|
||||
return delay/2 + jitter
|
||||
}
|
||||
|
||||
func deterministicJitter(seed string, max time.Duration) time.Duration {
|
||||
if max <= 0 {
|
||||
return 0
|
||||
}
|
||||
h := fnv.New64a()
|
||||
_, _ = h.Write([]byte(seed))
|
||||
return time.Duration(h.Sum64() % uint64(max))
|
||||
}
|
||||
|
||||
func deliveryID(delivery *Delivery) string {
|
||||
if delivery == nil {
|
||||
return "unknown"
|
||||
}
|
||||
if delivery.ID != "" {
|
||||
return delivery.ID
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
185
internal/queue/serializer.go
Normal file
185
internal/queue/serializer.go
Normal file
@@ -0,0 +1,185 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrInvalidEnvelope = errors.New("invalid queue envelope")
|
||||
|
||||
type JSONSerializer struct {
|
||||
version string
|
||||
enableLegacyFallback bool
|
||||
queueType string
|
||||
messageGroupID string
|
||||
}
|
||||
|
||||
type LegacyJSONSerializer struct{}
|
||||
|
||||
func NewJSONSerializer(version string, enableLegacyFallback bool, queueType, messageGroupID string) *JSONSerializer {
|
||||
version = strings.TrimSpace(version)
|
||||
if version == "" {
|
||||
version = "v1"
|
||||
}
|
||||
queueType = strings.ToLower(strings.TrimSpace(queueType))
|
||||
if queueType == "" {
|
||||
queueType = QueueTypeStandard
|
||||
}
|
||||
return &JSONSerializer{
|
||||
version: version,
|
||||
enableLegacyFallback: enableLegacyFallback,
|
||||
queueType: queueType,
|
||||
messageGroupID: strings.TrimSpace(messageGroupID),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *JSONSerializer) Serialize(ctx context.Context, job EmailJob) (*OutboundMessage, error) {
|
||||
envelope, err := NewEnvelope(ctx, s.version, job)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := json.Marshal(envelope)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
msg := &OutboundMessage{
|
||||
ID: envelope.MessageID,
|
||||
Body: body,
|
||||
Attributes: map[string]string{
|
||||
"message_kind": envelope.Kind,
|
||||
"message_id": envelope.MessageID,
|
||||
"correlation_id": envelope.CorrelationID,
|
||||
"trace_id": envelope.TraceID,
|
||||
"schema_version": envelope.Version,
|
||||
"job_type": envelope.Payload.NormalizedType(),
|
||||
},
|
||||
}
|
||||
if s.queueType == QueueTypeFIFO {
|
||||
msg.MessageGroupID = s.messageGroupID
|
||||
if msg.MessageGroupID == "" {
|
||||
msg.MessageGroupID = "default"
|
||||
}
|
||||
msg.DeduplicationID = envelope.DeduplicationKey()
|
||||
}
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
func (s *JSONSerializer) Deserialize(body []byte, attributes map[string]string) (*Envelope, error) {
|
||||
var envelope Envelope
|
||||
if err := json.Unmarshal(body, &envelope); err == nil {
|
||||
if err := validateEnvelope(&envelope); err == nil {
|
||||
return &envelope, nil
|
||||
}
|
||||
}
|
||||
|
||||
if !s.enableLegacyFallback {
|
||||
return nil, ErrInvalidEnvelope
|
||||
}
|
||||
|
||||
var job EmailJob
|
||||
if err := json.Unmarshal(body, &job); err != nil {
|
||||
return nil, errors.Join(ErrInvalidEnvelope, err)
|
||||
}
|
||||
job = job.Canonical()
|
||||
if err := job.Validate(); err != nil {
|
||||
return nil, errors.Join(ErrInvalidEnvelope, err)
|
||||
}
|
||||
|
||||
id := messageHash(body)
|
||||
correlationID := strings.TrimSpace(attributes["correlation_id"])
|
||||
if correlationID == "" {
|
||||
correlationID = id
|
||||
}
|
||||
traceID := strings.TrimSpace(attributes["trace_id"])
|
||||
if traceID == "" {
|
||||
traceID = correlationID
|
||||
}
|
||||
return &Envelope{
|
||||
Version: "legacy",
|
||||
MessageID: id,
|
||||
Kind: EmailKind,
|
||||
OccurredAt: time.Now().UTC(),
|
||||
CorrelationID: correlationID,
|
||||
TraceID: traceID,
|
||||
IdempotencyKey: id,
|
||||
Payload: job,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (LegacyJSONSerializer) Serialize(ctx context.Context, job EmailJob) (*OutboundMessage, error) {
|
||||
job = job.Canonical()
|
||||
if err := job.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := json.Marshal(job)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
id := messageHash(body)
|
||||
correlationID := CorrelationIDFromContext(ctx)
|
||||
if correlationID == "" {
|
||||
correlationID = id
|
||||
}
|
||||
traceID := TraceIDFromContext(ctx)
|
||||
if traceID == "" {
|
||||
traceID = correlationID
|
||||
}
|
||||
return &OutboundMessage{
|
||||
ID: id,
|
||||
Body: body,
|
||||
Attributes: map[string]string{
|
||||
"message_kind": EmailKind,
|
||||
"message_id": id,
|
||||
"correlation_id": correlationID,
|
||||
"trace_id": traceID,
|
||||
"schema_version": "legacy",
|
||||
"job_type": job.NormalizedType(),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (LegacyJSONSerializer) Deserialize(body []byte, attributes map[string]string) (*Envelope, error) {
|
||||
return NewJSONSerializer("v1", true, QueueTypeStandard, "").Deserialize(body, attributes)
|
||||
}
|
||||
|
||||
func validateEnvelope(envelope *Envelope) error {
|
||||
if envelope == nil {
|
||||
return ErrInvalidEnvelope
|
||||
}
|
||||
if strings.TrimSpace(envelope.Version) == "" {
|
||||
return errors.Join(ErrInvalidEnvelope, errors.New("version is required"))
|
||||
}
|
||||
if strings.TrimSpace(envelope.MessageID) == "" {
|
||||
return errors.Join(ErrInvalidEnvelope, errors.New("message id is required"))
|
||||
}
|
||||
if strings.TrimSpace(envelope.Kind) == "" {
|
||||
return errors.Join(ErrInvalidEnvelope, errors.New("kind is required"))
|
||||
}
|
||||
if envelope.OccurredAt.IsZero() {
|
||||
return errors.Join(ErrInvalidEnvelope, errors.New("occurred at is required"))
|
||||
}
|
||||
if err := envelope.Payload.Validate(); err != nil {
|
||||
return errors.Join(ErrInvalidEnvelope, err)
|
||||
}
|
||||
envelope.Payload = envelope.Payload.Canonical()
|
||||
if strings.TrimSpace(envelope.CorrelationID) == "" {
|
||||
envelope.CorrelationID = envelope.MessageID
|
||||
}
|
||||
if strings.TrimSpace(envelope.TraceID) == "" {
|
||||
envelope.TraceID = envelope.CorrelationID
|
||||
}
|
||||
if strings.TrimSpace(envelope.IdempotencyKey) == "" {
|
||||
envelope.IdempotencyKey = envelope.MessageID
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func messageHash(body []byte) string {
|
||||
sum := sha256.Sum256(body)
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
54
internal/queue/serializer_test.go
Normal file
54
internal/queue/serializer_test.go
Normal file
@@ -0,0 +1,54 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestJSONSerializerRoundTrip(t *testing.T) {
|
||||
serializer := NewJSONSerializer("v1", true, QueueTypeStandard, "")
|
||||
ctx := WithCorrelationID(context.Background(), "req-123")
|
||||
message, err := serializer.Serialize(ctx, EmailJob{
|
||||
Type: "invite_set_password",
|
||||
To: "user@example.com",
|
||||
Subject: "Welcome",
|
||||
Body: "hello",
|
||||
HTMLBody: "<b>hello</b>",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("serialize: %v", err)
|
||||
}
|
||||
|
||||
envelope, err := serializer.Deserialize(message.Body, message.Attributes)
|
||||
if err != nil {
|
||||
t.Fatalf("deserialize: %v", err)
|
||||
}
|
||||
if envelope.MessageID == "" {
|
||||
t.Fatalf("expected message id")
|
||||
}
|
||||
if envelope.CorrelationID != "req-123" {
|
||||
t.Fatalf("expected correlation id req-123, got %q", envelope.CorrelationID)
|
||||
}
|
||||
if envelope.Payload.NormalizedType() != "invite_set_password" {
|
||||
t.Fatalf("unexpected job type: %q", envelope.Payload.NormalizedType())
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONSerializerLegacyFallback(t *testing.T) {
|
||||
serializer := NewJSONSerializer("v1", true, QueueTypeStandard, "")
|
||||
envelope, err := serializer.Deserialize([]byte(`{"to":"user@example.com","subject":"Reset","body":"hello"}`), map[string]string{
|
||||
"correlation_id": "legacy-1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("deserialize legacy: %v", err)
|
||||
}
|
||||
if envelope.Version != "legacy" {
|
||||
t.Fatalf("expected legacy version, got %q", envelope.Version)
|
||||
}
|
||||
if envelope.CorrelationID != "legacy-1" {
|
||||
t.Fatalf("unexpected correlation id: %q", envelope.CorrelationID)
|
||||
}
|
||||
if envelope.Payload.To != "user@example.com" {
|
||||
t.Fatalf("unexpected payload: %+v", envelope.Payload)
|
||||
}
|
||||
}
|
||||
330
internal/queue/types.go
Normal file
330
internal/queue/types.go
Normal file
@@ -0,0 +1,330 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"net/mail"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"wucher/internal/shared/pkg/uuidv7"
|
||||
)
|
||||
|
||||
const (
|
||||
BackendSQS = "sqs"
|
||||
|
||||
QueueTypeStandard = "standard"
|
||||
QueueTypeFIFO = "fifo"
|
||||
|
||||
EmailKind = "email.dispatch"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidEmailJob = errors.New("invalid email job")
|
||||
)
|
||||
|
||||
type EmailJob struct {
|
||||
JobID string `json:"job_id,omitempty"`
|
||||
MessageType string `json:"message_type,omitempty"`
|
||||
To string `json:"to"`
|
||||
CC []string `json:"cc,omitempty"`
|
||||
BCC []string `json:"bcc,omitempty"`
|
||||
Subject string `json:"subject"`
|
||||
TemplateName string `json:"template_name,omitempty"`
|
||||
TemplateData map[string]any `json:"template_data,omitempty"`
|
||||
TextBody string `json:"text_body,omitempty"`
|
||||
HTMLBody string `json:"html_body,omitempty"`
|
||||
Attachments []EmailAttachment `json:"attachments,omitempty"`
|
||||
ReplyTo []string `json:"reply_to,omitempty"`
|
||||
Tags map[string]string `json:"tags,omitempty"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at,omitempty"`
|
||||
TraceID string `json:"trace_id,omitempty"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
|
||||
// Legacy aliases kept for backward-compatible decoding and tests.
|
||||
Type string `json:"type,omitempty"`
|
||||
Body string `json:"body,omitempty"`
|
||||
}
|
||||
|
||||
type EmailAttachment struct {
|
||||
Filename string `json:"filename,omitempty"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
ContentID string `json:"content_id,omitempty"`
|
||||
Content []byte `json:"content,omitempty"`
|
||||
Inline bool `json:"inline,omitempty"`
|
||||
}
|
||||
|
||||
func (j EmailJob) Validate() error {
|
||||
job := j.Canonical()
|
||||
if strings.TrimSpace(job.To) == "" {
|
||||
return errors.Join(ErrInvalidEmailJob, errors.New("recipient is required"))
|
||||
}
|
||||
if err := validateEmailAddress(job.To); err != nil {
|
||||
return errors.Join(ErrInvalidEmailJob, err)
|
||||
}
|
||||
if err := validateEmailAddresses(job.CC); err != nil {
|
||||
return errors.Join(ErrInvalidEmailJob, err)
|
||||
}
|
||||
if err := validateEmailAddresses(job.BCC); err != nil {
|
||||
return errors.Join(ErrInvalidEmailJob, err)
|
||||
}
|
||||
if err := validateEmailAddresses(job.ReplyTo); err != nil {
|
||||
return errors.Join(ErrInvalidEmailJob, err)
|
||||
}
|
||||
hasBody := strings.TrimSpace(job.TextBody) != "" || strings.TrimSpace(job.HTMLBody) != ""
|
||||
if strings.TrimSpace(job.Subject) == "" && (strings.TrimSpace(job.TemplateName) == "" || hasBody) {
|
||||
return errors.Join(ErrInvalidEmailJob, errors.New("subject is required"))
|
||||
}
|
||||
if !hasBody && strings.TrimSpace(job.TemplateName) == "" {
|
||||
return errors.Join(ErrInvalidEmailJob, errors.New("body is required"))
|
||||
}
|
||||
for i := range job.Attachments {
|
||||
attachment := job.Attachments[i]
|
||||
if len(attachment.Content) == 0 {
|
||||
return errors.Join(ErrInvalidEmailJob, errors.New("attachment content is required"))
|
||||
}
|
||||
if strings.TrimSpace(attachment.ContentType) == "" {
|
||||
return errors.Join(ErrInvalidEmailJob, errors.New("attachment content type is required"))
|
||||
}
|
||||
if attachment.Inline && strings.TrimSpace(attachment.ContentID) == "" {
|
||||
return errors.Join(ErrInvalidEmailJob, errors.New("inline attachment content id is required"))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (j EmailJob) RecipientHash() string {
|
||||
sum := sha256.Sum256([]byte(strings.ToLower(strings.TrimSpace(j.Canonical().To))))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
func (j EmailJob) NormalizedType() string {
|
||||
if v := strings.TrimSpace(j.Canonical().MessageType); v != "" {
|
||||
return v
|
||||
}
|
||||
return "generic"
|
||||
}
|
||||
|
||||
func (j EmailJob) TextBodyValue() string {
|
||||
return j.Canonical().TextBody
|
||||
}
|
||||
|
||||
func (j EmailJob) Canonical() EmailJob {
|
||||
job := j
|
||||
job.To = strings.TrimSpace(job.To)
|
||||
job.Subject = strings.TrimSpace(job.Subject)
|
||||
job.TemplateName = strings.TrimSpace(job.TemplateName)
|
||||
job.JobID = strings.TrimSpace(job.JobID)
|
||||
job.TraceID = strings.TrimSpace(job.TraceID)
|
||||
job.RequestID = strings.TrimSpace(job.RequestID)
|
||||
if strings.TrimSpace(job.MessageType) == "" {
|
||||
job.MessageType = strings.TrimSpace(job.Type)
|
||||
}
|
||||
if strings.TrimSpace(job.TextBody) == "" {
|
||||
job.TextBody = job.Body
|
||||
}
|
||||
job.Type = ""
|
||||
job.Body = ""
|
||||
job.CC = cleanStrings(job.CC)
|
||||
job.BCC = cleanStrings(job.BCC)
|
||||
job.ReplyTo = cleanStrings(job.ReplyTo)
|
||||
job.Tags = cleanStringMap(job.Tags)
|
||||
job.Metadata = cleanStringMap(job.Metadata)
|
||||
job.Attachments = canonicalEmailAttachments(job.Attachments)
|
||||
return job
|
||||
}
|
||||
|
||||
func canonicalEmailAttachments(values []EmailAttachment) []EmailAttachment {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]EmailAttachment, 0, len(values))
|
||||
for i := range values {
|
||||
item := values[i]
|
||||
item.Filename = strings.TrimSpace(item.Filename)
|
||||
item.ContentType = strings.TrimSpace(item.ContentType)
|
||||
item.ContentID = strings.Trim(strings.TrimSpace(item.ContentID), "<>")
|
||||
if len(item.Content) == 0 {
|
||||
continue
|
||||
}
|
||||
item.Content = append([]byte(nil), item.Content...)
|
||||
out = append(out, item)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type Envelope struct {
|
||||
Version string `json:"version"`
|
||||
MessageID string `json:"message_id"`
|
||||
Kind string `json:"kind"`
|
||||
OccurredAt time.Time `json:"occurred_at"`
|
||||
CorrelationID string `json:"correlation_id,omitempty"`
|
||||
TraceID string `json:"trace_id,omitempty"`
|
||||
IdempotencyKey string `json:"idempotency_key,omitempty"`
|
||||
Payload EmailJob `json:"payload"`
|
||||
}
|
||||
|
||||
func NewEnvelope(ctx context.Context, version string, job EmailJob) (*Envelope, error) {
|
||||
job = job.Canonical()
|
||||
if err := job.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
id, err := newMessageID()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
correlationID := CorrelationIDFromContext(ctx)
|
||||
if correlationID == "" {
|
||||
correlationID = id
|
||||
}
|
||||
traceID := TraceIDFromContext(ctx)
|
||||
if traceID == "" {
|
||||
traceID = correlationID
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if job.JobID == "" {
|
||||
job.JobID = id
|
||||
}
|
||||
if job.CreatedAt.IsZero() {
|
||||
job.CreatedAt = now
|
||||
}
|
||||
if job.TraceID == "" {
|
||||
job.TraceID = traceID
|
||||
}
|
||||
if job.RequestID == "" {
|
||||
job.RequestID = correlationID
|
||||
}
|
||||
return &Envelope{
|
||||
Version: strings.TrimSpace(version),
|
||||
MessageID: id,
|
||||
Kind: EmailKind,
|
||||
OccurredAt: now,
|
||||
CorrelationID: correlationID,
|
||||
TraceID: traceID,
|
||||
IdempotencyKey: job.JobID,
|
||||
Payload: job,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e Envelope) DeduplicationKey() string {
|
||||
if v := strings.TrimSpace(e.IdempotencyKey); v != "" {
|
||||
return v
|
||||
}
|
||||
return strings.TrimSpace(e.MessageID)
|
||||
}
|
||||
|
||||
type OutboundMessage struct {
|
||||
ID string
|
||||
Body []byte
|
||||
Attributes map[string]string
|
||||
MessageGroupID string
|
||||
DeduplicationID string
|
||||
}
|
||||
|
||||
type Delivery struct {
|
||||
ID string
|
||||
ReceiptHandle string
|
||||
Body []byte
|
||||
Attributes map[string]string
|
||||
ReceiveCount int
|
||||
EnqueuedAt time.Time
|
||||
}
|
||||
|
||||
type MessageSerializer interface {
|
||||
Serialize(ctx context.Context, job EmailJob) (*OutboundMessage, error)
|
||||
Deserialize(body []byte, attributes map[string]string) (*Envelope, error)
|
||||
}
|
||||
|
||||
type QueueProducer interface {
|
||||
Backend() string
|
||||
Publish(ctx context.Context, message OutboundMessage) error
|
||||
}
|
||||
|
||||
type QueueConsumer interface {
|
||||
Backend() string
|
||||
Receive(ctx context.Context, maxMessages int) ([]*Delivery, error)
|
||||
Ack(ctx context.Context, deliveries []*Delivery) error
|
||||
Retry(ctx context.Context, delivery *Delivery, delay time.Duration) error
|
||||
DeadLetter(ctx context.Context, delivery *Delivery, reason string, cause error) error
|
||||
ApproximateDepth(ctx context.Context) (int64, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
type EmailJobProducer interface {
|
||||
Enqueue(ctx context.Context, job EmailJob) error
|
||||
}
|
||||
|
||||
type EmailJobHandler interface {
|
||||
Handle(ctx context.Context, envelope *Envelope) error
|
||||
}
|
||||
|
||||
func newMessageID() (string, error) {
|
||||
id, err := uuidv7.String(uuidv7.MustNew())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func cleanStrings(values []string) []string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, value)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cleanStringMap(values map[string]string) map[string]string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(values))
|
||||
for key, value := range values {
|
||||
key = strings.TrimSpace(key)
|
||||
value = strings.TrimSpace(value)
|
||||
if key == "" || value == "" {
|
||||
continue
|
||||
}
|
||||
out[key] = value
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func validateEmailAddresses(addresses []string) error {
|
||||
for _, address := range addresses {
|
||||
if err := validateEmailAddress(address); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateEmailAddress(address string) error {
|
||||
address = strings.TrimSpace(address)
|
||||
if address == "" {
|
||||
return nil
|
||||
}
|
||||
if _, err := mail.ParseAddress(address); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
320
internal/queue/worker.go
Normal file
320
internal/queue/worker.go
Normal file
@@ -0,0 +1,320 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"wucher/internal/shared/pkg/metrics"
|
||||
)
|
||||
|
||||
type WorkerConfig struct {
|
||||
Concurrency int
|
||||
MaxBatchSize int
|
||||
ProcessTimeout time.Duration
|
||||
ProcessingTTL time.Duration
|
||||
IdempotencyTTL time.Duration
|
||||
PollErrorDelay time.Duration
|
||||
MetricsComponent string
|
||||
}
|
||||
|
||||
type Worker struct {
|
||||
cfg WorkerConfig
|
||||
consumer QueueConsumer
|
||||
serializer MessageSerializer
|
||||
handler EmailJobHandler
|
||||
retry RetryPolicy
|
||||
idem IdempotencyStore
|
||||
metrics *Metrics
|
||||
health *HealthReporter
|
||||
logger *slog.Logger
|
||||
component string
|
||||
}
|
||||
|
||||
type processResult struct {
|
||||
delivery *Delivery
|
||||
envelope *Envelope
|
||||
decision RetryDecision
|
||||
err error
|
||||
}
|
||||
|
||||
func NewWorker(
|
||||
cfg WorkerConfig,
|
||||
consumer QueueConsumer,
|
||||
serializer MessageSerializer,
|
||||
handler EmailJobHandler,
|
||||
retry RetryPolicy,
|
||||
idem IdempotencyStore,
|
||||
metrics *Metrics,
|
||||
health *HealthReporter,
|
||||
logger *slog.Logger,
|
||||
) *Worker {
|
||||
if cfg.Concurrency <= 0 {
|
||||
cfg.Concurrency = 1
|
||||
}
|
||||
if cfg.MaxBatchSize <= 0 || cfg.MaxBatchSize > cfg.Concurrency {
|
||||
cfg.MaxBatchSize = cfg.Concurrency
|
||||
}
|
||||
if cfg.ProcessTimeout <= 0 {
|
||||
cfg.ProcessTimeout = 30 * time.Second
|
||||
}
|
||||
if cfg.ProcessingTTL <= 0 {
|
||||
cfg.ProcessingTTL = 15 * time.Minute
|
||||
}
|
||||
if cfg.IdempotencyTTL <= 0 {
|
||||
cfg.IdempotencyTTL = 24 * time.Hour
|
||||
}
|
||||
if cfg.PollErrorDelay <= 0 {
|
||||
cfg.PollErrorDelay = time.Second
|
||||
}
|
||||
if retry == nil {
|
||||
retry = NewExponentialBackoffPolicy(time.Second, 5*time.Minute, 30*time.Second)
|
||||
}
|
||||
if idem == nil {
|
||||
idem = NoopIdempotencyStore{}
|
||||
}
|
||||
if metrics == nil {
|
||||
metrics = NewMetrics()
|
||||
}
|
||||
if health == nil {
|
||||
health = NewHealthReporter(3)
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Worker{
|
||||
cfg: cfg,
|
||||
consumer: consumer,
|
||||
serializer: serializer,
|
||||
handler: handler,
|
||||
retry: retry,
|
||||
idem: idem,
|
||||
metrics: metrics,
|
||||
health: health,
|
||||
logger: logger,
|
||||
component: normalizeWorkerMetricsComponent(cfg.MetricsComponent),
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Run(ctx context.Context) error {
|
||||
if w.consumer == nil {
|
||||
return errors.New("queue consumer is required")
|
||||
}
|
||||
if w.serializer == nil {
|
||||
return errors.New("message serializer is required")
|
||||
}
|
||||
if w.handler == nil {
|
||||
return errors.New("message handler is required")
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
w.health.MarkDraining()
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
|
||||
deliveries, err := w.consumer.Receive(ctx, w.cfg.MaxBatchSize)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
w.health.MarkDraining()
|
||||
return nil
|
||||
}
|
||||
w.health.MarkPollFailure(err)
|
||||
w.logger.Error("queue receive failed",
|
||||
slog.String("backend", w.consumer.Backend()),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
w.health.MarkDraining()
|
||||
return nil
|
||||
case <-time.After(w.cfg.PollErrorDelay):
|
||||
}
|
||||
continue
|
||||
}
|
||||
w.health.MarkPollSuccess()
|
||||
if len(deliveries) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
w.metrics.RecordReceive(len(deliveries))
|
||||
results := make(chan processResult, len(deliveries))
|
||||
for _, delivery := range deliveries {
|
||||
delivery := delivery
|
||||
go func() {
|
||||
results <- w.processDelivery(ctx, delivery)
|
||||
}()
|
||||
}
|
||||
|
||||
acks := make([]*Delivery, 0, len(deliveries))
|
||||
for range deliveries {
|
||||
result := <-results
|
||||
switch result.decision.Action {
|
||||
case RetryActionAck:
|
||||
acks = append(acks, result.delivery)
|
||||
case RetryActionRetry:
|
||||
if err := w.consumer.Retry(ctx, result.delivery, result.decision.Delay); err != nil {
|
||||
w.metrics.RecordRetryFailure()
|
||||
w.logger.Error("queue retry scheduling failed",
|
||||
slog.String("backend", w.consumer.Backend()),
|
||||
slog.String("message_id", result.delivery.ID),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
} else {
|
||||
w.metrics.RecordRetry()
|
||||
}
|
||||
case RetryActionDeadLetter:
|
||||
if err := w.consumer.DeadLetter(ctx, result.delivery, result.decision.Reason, result.err); err != nil {
|
||||
w.metrics.RecordDeadLetterFailure()
|
||||
w.logger.Error("queue dead-letter failed",
|
||||
slog.String("backend", w.consumer.Backend()),
|
||||
slog.String("message_id", result.delivery.ID),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
} else {
|
||||
w.metrics.RecordDeadLetter()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(acks) > 0 {
|
||||
if err := w.consumer.Ack(ctx, acks); err != nil {
|
||||
w.metrics.RecordAckFailure()
|
||||
w.logger.Error("queue ack failed",
|
||||
slog.String("backend", w.consumer.Backend()),
|
||||
slog.Int("count", len(acks)),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) RefreshQueueDepth(ctx context.Context) error {
|
||||
if w.consumer == nil {
|
||||
return errors.New("queue consumer is required")
|
||||
}
|
||||
depth, err := w.consumer.ApproximateDepth(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.metrics.SetQueueDepth(depth)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) Metrics() *Metrics {
|
||||
return w.metrics
|
||||
}
|
||||
|
||||
func (w *Worker) Health() *HealthReporter {
|
||||
return w.health
|
||||
}
|
||||
|
||||
func (w *Worker) processDelivery(ctx context.Context, delivery *Delivery) (result processResult) {
|
||||
started := time.Now()
|
||||
jobType := "unknown"
|
||||
var jobErr error
|
||||
defer func() {
|
||||
metrics.ObserveWorkerJob(w.component, jobType, jobErr, started)
|
||||
}()
|
||||
|
||||
envelope, err := w.serializer.Deserialize(delivery.Body, delivery.Attributes)
|
||||
if err != nil {
|
||||
w.metrics.RecordMalformed()
|
||||
w.metrics.RecordFailure()
|
||||
jobErr = err
|
||||
return processResult{
|
||||
delivery: delivery,
|
||||
decision: w.retry.Decide(delivery, nil, Permanent(err)),
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
jobType = strings.TrimSpace(envelope.Payload.NormalizedType())
|
||||
|
||||
idempotencyKey := envelope.DeduplicationKey()
|
||||
state, err := w.idem.Acquire(ctx, idempotencyKey, w.cfg.ProcessingTTL)
|
||||
if err != nil {
|
||||
w.metrics.RecordFailure()
|
||||
jobErr = err
|
||||
return processResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: w.retry.Decide(delivery, envelope, Transient(err)),
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
switch state {
|
||||
case IdempotencyStateDuplicate:
|
||||
w.metrics.RecordDuplicate()
|
||||
w.logger.Info("queue duplicate skipped",
|
||||
slog.String("message_id", envelope.MessageID),
|
||||
slog.String("job_type", envelope.Payload.NormalizedType()),
|
||||
slog.String("recipient_hash", envelope.Payload.RecipientHash()),
|
||||
slog.String("correlation_id", envelope.CorrelationID),
|
||||
)
|
||||
return processResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: RetryDecision{Action: RetryActionAck},
|
||||
}
|
||||
case IdempotencyStateInProgress:
|
||||
inProgressErr := Transient(errors.New("message already in progress"))
|
||||
jobErr = inProgressErr
|
||||
return processResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: w.retry.Decide(delivery, envelope, inProgressErr),
|
||||
err: inProgressErr,
|
||||
}
|
||||
}
|
||||
|
||||
processCtx, cancel := context.WithTimeout(ctx, w.cfg.ProcessTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := w.handler.Handle(processCtx, envelope); err != nil {
|
||||
_ = w.idem.Release(ctx, idempotencyKey)
|
||||
w.metrics.RecordFailure()
|
||||
jobErr = err
|
||||
return processResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: w.retry.Decide(delivery, envelope, err),
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
|
||||
if err := w.idem.MarkCompleted(ctx, idempotencyKey, w.cfg.IdempotencyTTL); err != nil {
|
||||
w.metrics.RecordFailure()
|
||||
w.logger.Error("queue idempotency completion failed",
|
||||
slog.String("message_id", envelope.MessageID),
|
||||
slog.String("correlation_id", envelope.CorrelationID),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
jobErr = err
|
||||
return processResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: w.retry.Decide(delivery, envelope, Transient(err)),
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
|
||||
w.metrics.RecordProcessed(time.Since(started))
|
||||
jobErr = nil
|
||||
return processResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: RetryDecision{Action: RetryActionAck},
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeWorkerMetricsComponent(component string) string {
|
||||
component = strings.ToLower(strings.TrimSpace(component))
|
||||
if component == "" {
|
||||
return "worker"
|
||||
}
|
||||
return component
|
||||
}
|
||||
93
internal/queue/worker_test.go
Normal file
93
internal/queue/worker_test.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"wucher/internal/config"
|
||||
"wucher/internal/shared/pkg/logger"
|
||||
)
|
||||
|
||||
type stubHandler struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (h stubHandler) Handle(context.Context, *Envelope) error {
|
||||
return h.err
|
||||
}
|
||||
|
||||
type stubIdempotencyStore struct {
|
||||
state IdempotencyState
|
||||
err error
|
||||
}
|
||||
|
||||
func (s stubIdempotencyStore) Acquire(context.Context, string, time.Duration) (IdempotencyState, error) {
|
||||
if s.err != nil {
|
||||
return "", s.err
|
||||
}
|
||||
if s.state != "" {
|
||||
return s.state, nil
|
||||
}
|
||||
return IdempotencyStateAcquired, nil
|
||||
}
|
||||
|
||||
func (s stubIdempotencyStore) MarkCompleted(context.Context, string, time.Duration) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s stubIdempotencyStore) Release(context.Context, string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestWorkerProcessDeliveryDuplicate(t *testing.T) {
|
||||
worker := NewWorker(WorkerConfig{}, nil, NewJSONSerializer("v1", true, QueueTypeStandard, ""), stubHandler{}, nil, stubIdempotencyStore{state: IdempotencyStateDuplicate}, nil, nil, logger.New(config.LoggingConfig{Format: "json"}))
|
||||
message, err := NewJSONSerializer("v1", true, QueueTypeStandard, "").Serialize(context.Background(), EmailJob{
|
||||
To: "user@example.com",
|
||||
Subject: "Hello",
|
||||
Body: "World",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("serialize: %v", err)
|
||||
}
|
||||
|
||||
result := worker.processDelivery(context.Background(), &Delivery{ID: message.ID, Body: message.Body, Attributes: message.Attributes, ReceiveCount: 1})
|
||||
if result.decision.Action != RetryActionAck {
|
||||
t.Fatalf("expected ack decision, got %s", result.decision.Action)
|
||||
}
|
||||
if worker.Metrics().Snapshot().Duplicates != 1 {
|
||||
t.Fatalf("expected duplicate metric to increment")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerProcessDeliveryMalformedPayload(t *testing.T) {
|
||||
worker := NewWorker(WorkerConfig{}, nil, NewJSONSerializer("v1", false, QueueTypeStandard, ""), stubHandler{}, nil, stubIdempotencyStore{}, nil, nil, logger.New(config.LoggingConfig{Format: "json"}))
|
||||
result := worker.processDelivery(context.Background(), &Delivery{ID: "msg-1", Body: []byte(`{"bad":true}`), ReceiveCount: 1})
|
||||
if result.decision.Action != RetryActionDeadLetter {
|
||||
t.Fatalf("expected dead-letter decision, got %s", result.decision.Action)
|
||||
}
|
||||
if worker.Metrics().Snapshot().Malformed != 1 {
|
||||
t.Fatalf("expected malformed metric to increment")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerProcessDeliveryTransientError(t *testing.T) {
|
||||
worker := NewWorker(WorkerConfig{}, nil, NewJSONSerializer("v1", true, QueueTypeStandard, ""), stubHandler{err: Transient(errors.New("ses timeout"))}, nil, stubIdempotencyStore{}, nil, nil, logger.New(config.LoggingConfig{Format: "json"}))
|
||||
message, err := NewJSONSerializer("v1", true, QueueTypeStandard, "").Serialize(context.Background(), EmailJob{
|
||||
To: "user@example.com",
|
||||
Subject: "Hello",
|
||||
Body: "World",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("serialize: %v", err)
|
||||
}
|
||||
|
||||
result := worker.processDelivery(context.Background(), &Delivery{ID: message.ID, Body: message.Body, Attributes: message.Attributes, ReceiveCount: 2})
|
||||
if result.decision.Action != RetryActionRetry {
|
||||
t.Fatalf("expected retry decision, got %s", result.decision.Action)
|
||||
}
|
||||
if result.decision.Delay <= 0 {
|
||||
t.Fatalf("expected positive retry delay")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user