init push
This commit is contained in:
299
internal/queue/file_processing_worker.go
Normal file
299
internal/queue/file_processing_worker.go
Normal file
@@ -0,0 +1,299 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"wucher/internal/shared/pkg/metrics"
|
||||
)
|
||||
|
||||
type FileProcessingWorker struct {
|
||||
cfg WorkerConfig
|
||||
consumer QueueConsumer
|
||||
serializer FileProcessingMessageSerializer
|
||||
handler FileProcessingJobHandler
|
||||
retry RetryPolicy
|
||||
idem IdempotencyStore
|
||||
metrics *Metrics
|
||||
health *HealthReporter
|
||||
logger *slog.Logger
|
||||
component string
|
||||
}
|
||||
|
||||
type fileProcessResult struct {
|
||||
delivery *Delivery
|
||||
envelope *FileProcessingEnvelope
|
||||
decision RetryDecision
|
||||
err error
|
||||
}
|
||||
|
||||
func NewFileProcessingWorker(
|
||||
cfg WorkerConfig,
|
||||
consumer QueueConsumer,
|
||||
serializer FileProcessingMessageSerializer,
|
||||
handler FileProcessingJobHandler,
|
||||
retry RetryPolicy,
|
||||
idem IdempotencyStore,
|
||||
metrics *Metrics,
|
||||
health *HealthReporter,
|
||||
logger *slog.Logger,
|
||||
) *FileProcessingWorker {
|
||||
if cfg.Concurrency <= 0 {
|
||||
cfg.Concurrency = 1
|
||||
}
|
||||
if cfg.MaxBatchSize <= 0 || cfg.MaxBatchSize > cfg.Concurrency {
|
||||
cfg.MaxBatchSize = cfg.Concurrency
|
||||
}
|
||||
if cfg.ProcessTimeout <= 0 {
|
||||
cfg.ProcessTimeout = 30 * time.Second
|
||||
}
|
||||
if cfg.ProcessingTTL <= 0 {
|
||||
cfg.ProcessingTTL = 15 * time.Minute
|
||||
}
|
||||
if cfg.IdempotencyTTL <= 0 {
|
||||
cfg.IdempotencyTTL = 24 * time.Hour
|
||||
}
|
||||
if cfg.PollErrorDelay <= 0 {
|
||||
cfg.PollErrorDelay = time.Second
|
||||
}
|
||||
if retry == nil {
|
||||
retry = NewExponentialBackoffPolicy(time.Second, 5*time.Minute, 30*time.Second)
|
||||
}
|
||||
if idem == nil {
|
||||
idem = NoopIdempotencyStore{}
|
||||
}
|
||||
if metrics == nil {
|
||||
metrics = NewMetrics()
|
||||
}
|
||||
if health == nil {
|
||||
health = NewHealthReporter(3)
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &FileProcessingWorker{
|
||||
cfg: cfg,
|
||||
consumer: consumer,
|
||||
serializer: serializer,
|
||||
handler: handler,
|
||||
retry: retry,
|
||||
idem: idem,
|
||||
metrics: metrics,
|
||||
health: health,
|
||||
logger: logger,
|
||||
component: normalizeWorkerMetricsComponent(cfg.MetricsComponent),
|
||||
}
|
||||
}
|
||||
|
||||
func (w *FileProcessingWorker) Run(ctx context.Context) error {
|
||||
if w.consumer == nil {
|
||||
return errors.New("queue consumer is required")
|
||||
}
|
||||
if w.serializer == nil {
|
||||
return errors.New("file message serializer is required")
|
||||
}
|
||||
if w.handler == nil {
|
||||
return errors.New("file message handler is required")
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
w.health.MarkDraining()
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
|
||||
deliveries, err := w.consumer.Receive(ctx, w.cfg.MaxBatchSize)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
w.health.MarkDraining()
|
||||
return nil
|
||||
}
|
||||
w.health.MarkPollFailure(err)
|
||||
w.logger.Error("file queue receive failed",
|
||||
slog.String("backend", w.consumer.Backend()),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
w.health.MarkDraining()
|
||||
return nil
|
||||
case <-time.After(w.cfg.PollErrorDelay):
|
||||
}
|
||||
continue
|
||||
}
|
||||
w.health.MarkPollSuccess()
|
||||
if len(deliveries) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
w.metrics.RecordReceive(len(deliveries))
|
||||
results := make(chan fileProcessResult, len(deliveries))
|
||||
for _, delivery := range deliveries {
|
||||
delivery := delivery
|
||||
go func() {
|
||||
results <- w.processDelivery(ctx, delivery)
|
||||
}()
|
||||
}
|
||||
|
||||
acks := make([]*Delivery, 0, len(deliveries))
|
||||
for range deliveries {
|
||||
result := <-results
|
||||
switch result.decision.Action {
|
||||
case RetryActionAck:
|
||||
acks = append(acks, result.delivery)
|
||||
case RetryActionRetry:
|
||||
if err := w.consumer.Retry(ctx, result.delivery, result.decision.Delay); err != nil {
|
||||
w.metrics.RecordRetryFailure()
|
||||
w.logger.Error("file queue retry scheduling failed",
|
||||
slog.String("backend", w.consumer.Backend()),
|
||||
slog.String("message_id", result.delivery.ID),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
} else {
|
||||
w.metrics.RecordRetry()
|
||||
}
|
||||
case RetryActionDeadLetter:
|
||||
if err := w.consumer.DeadLetter(ctx, result.delivery, result.decision.Reason, result.err); err != nil {
|
||||
w.metrics.RecordDeadLetterFailure()
|
||||
w.logger.Error("file queue dead-letter failed",
|
||||
slog.String("backend", w.consumer.Backend()),
|
||||
slog.String("message_id", result.delivery.ID),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
} else {
|
||||
w.metrics.RecordDeadLetter()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(acks) > 0 {
|
||||
if err := w.consumer.Ack(ctx, acks); err != nil {
|
||||
w.metrics.RecordAckFailure()
|
||||
w.logger.Error("file queue ack failed",
|
||||
slog.String("backend", w.consumer.Backend()),
|
||||
slog.Int("count", len(acks)),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *FileProcessingWorker) RefreshQueueDepth(ctx context.Context) error {
|
||||
if w.consumer == nil {
|
||||
return errors.New("queue consumer is required")
|
||||
}
|
||||
depth, err := w.consumer.ApproximateDepth(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.metrics.SetQueueDepth(depth)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *FileProcessingWorker) Metrics() *Metrics {
|
||||
return w.metrics
|
||||
}
|
||||
|
||||
func (w *FileProcessingWorker) Health() *HealthReporter {
|
||||
return w.health
|
||||
}
|
||||
|
||||
func (w *FileProcessingWorker) processDelivery(ctx context.Context, delivery *Delivery) (result fileProcessResult) {
|
||||
started := time.Now()
|
||||
var jobErr error
|
||||
defer func() {
|
||||
metrics.ObserveWorkerJob(w.component, "file_processing", jobErr, started)
|
||||
}()
|
||||
|
||||
envelope, err := w.serializer.Deserialize(delivery.Body, delivery.Attributes)
|
||||
if err != nil {
|
||||
w.metrics.RecordMalformed()
|
||||
w.metrics.RecordFailure()
|
||||
jobErr = err
|
||||
return fileProcessResult{
|
||||
delivery: delivery,
|
||||
decision: w.retry.Decide(delivery, nil, Permanent(err)),
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
|
||||
idempotencyKey := envelope.DeduplicationKey()
|
||||
state, err := w.idem.Acquire(ctx, idempotencyKey, w.cfg.ProcessingTTL)
|
||||
if err != nil {
|
||||
w.metrics.RecordFailure()
|
||||
jobErr = err
|
||||
return fileProcessResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: w.retry.Decide(delivery, nil, Transient(err)),
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
switch state {
|
||||
case IdempotencyStateDuplicate:
|
||||
w.metrics.RecordDuplicate()
|
||||
w.logger.Info("file queue duplicate skipped",
|
||||
slog.String("message_id", envelope.MessageID),
|
||||
slog.String("file_uuid", envelope.Payload.FileUUID),
|
||||
slog.String("correlation_id", envelope.CorrelationID),
|
||||
)
|
||||
return fileProcessResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: RetryDecision{Action: RetryActionAck},
|
||||
}
|
||||
case IdempotencyStateInProgress:
|
||||
inProgressErr := Transient(errors.New("message already in progress"))
|
||||
jobErr = inProgressErr
|
||||
return fileProcessResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: w.retry.Decide(delivery, nil, inProgressErr),
|
||||
err: inProgressErr,
|
||||
}
|
||||
}
|
||||
|
||||
processCtx, cancel := context.WithTimeout(ctx, w.cfg.ProcessTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := w.handler.Handle(processCtx, envelope); err != nil {
|
||||
_ = w.idem.Release(ctx, idempotencyKey)
|
||||
w.metrics.RecordFailure()
|
||||
jobErr = err
|
||||
return fileProcessResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: w.retry.Decide(delivery, nil, err),
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
|
||||
if err := w.idem.MarkCompleted(ctx, idempotencyKey, w.cfg.IdempotencyTTL); err != nil {
|
||||
w.metrics.RecordFailure()
|
||||
w.logger.Error("file queue idempotency completion failed",
|
||||
slog.String("message_id", envelope.MessageID),
|
||||
slog.String("file_uuid", envelope.Payload.FileUUID),
|
||||
slog.String("correlation_id", envelope.CorrelationID),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
jobErr = err
|
||||
return fileProcessResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: w.retry.Decide(delivery, nil, Transient(err)),
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
|
||||
w.metrics.RecordProcessed(time.Since(started))
|
||||
jobErr = nil
|
||||
return fileProcessResult{
|
||||
delivery: delivery,
|
||||
envelope: envelope,
|
||||
decision: RetryDecision{Action: RetryActionAck},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user