init push
This commit is contained in:
384
internal/shared/pkg/metrics/prometheus.go
Normal file
384
internal/shared/pkg/metrics/prometheus.go
Normal file
@@ -0,0 +1,384 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
smithyhttp "github.com/aws/smithy-go/transport/http"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
var (
|
||||
registerOnce sync.Once
|
||||
|
||||
awsRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_aws_requests_total",
|
||||
Help: "Total number of AWS requests by component, service, operation, and result.",
|
||||
}, []string{"component", "service", "operation", "result"})
|
||||
awsRequestDurationSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "app_aws_request_duration_seconds",
|
||||
Help: "AWS request duration in seconds by component, service, and operation.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}, []string{"component", "service", "operation"})
|
||||
awsErrorsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_aws_errors_total",
|
||||
Help: "Total number of AWS errors by class.",
|
||||
}, []string{"component", "service", "operation", "error_class"})
|
||||
awsTransferredBytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_aws_transferred_bytes_total",
|
||||
Help: "Total AWS transferred bytes by component, service, operation, and direction.",
|
||||
}, []string{"component", "service", "operation", "direction"})
|
||||
|
||||
queueMessagesReceivedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_queue_messages_received_total",
|
||||
Help: "Total number of queue messages received.",
|
||||
}, []string{"component", "queue_name"})
|
||||
queueMessagesDeletedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_queue_messages_deleted_total",
|
||||
Help: "Total number of queue messages deleted.",
|
||||
}, []string{"component", "queue_name"})
|
||||
queueVisibilityChangesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_queue_visibility_changes_total",
|
||||
Help: "Total number of queue visibility timeout changes.",
|
||||
}, []string{"component", "queue_name", "result"})
|
||||
queueSendTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_queue_send_total",
|
||||
Help: "Total number of queue send attempts.",
|
||||
}, []string{"component", "queue_name", "result"})
|
||||
queueRetriesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_queue_retries_total",
|
||||
Help: "Total number of queue retry/requeue attempts.",
|
||||
}, []string{"component", "queue_name", "result"})
|
||||
queueMessageBytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_queue_message_bytes_total",
|
||||
Help: "Total queue message bytes observed on send and receive.",
|
||||
}, []string{"component", "queue_name", "direction"})
|
||||
queueDepth = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Name: "app_queue_depth",
|
||||
Help: "Approximate queue depth from SQS attributes.",
|
||||
}, []string{"component", "queue_name"})
|
||||
|
||||
workerJobsProcessedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_worker_jobs_processed_total",
|
||||
Help: "Total number of worker jobs processed.",
|
||||
}, []string{"component", "job_type", "result"})
|
||||
workerJobDurationSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "app_worker_job_duration_seconds",
|
||||
Help: "Worker job processing duration in seconds.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}, []string{"component", "job_type"})
|
||||
|
||||
emailSendTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_email_send_total",
|
||||
Help: "Total number of email send attempts.",
|
||||
}, []string{"component", "provider", "result"})
|
||||
emailSendDurationSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "app_email_send_duration_seconds",
|
||||
Help: "Email send duration in seconds.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}, []string{"component", "provider"})
|
||||
|
||||
fileUploadTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_file_upload_total",
|
||||
Help: "Total number of file upload operations.",
|
||||
}, []string{"component", "result"})
|
||||
fileDeleteTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_file_delete_total",
|
||||
Help: "Total number of file delete operations.",
|
||||
}, []string{"component", "result"})
|
||||
fileHeadTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_file_head_total",
|
||||
Help: "Total number of file head operations.",
|
||||
}, []string{"component", "result"})
|
||||
filePresignDownloadTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_file_presign_download_total",
|
||||
Help: "Total number of file presign download operations.",
|
||||
}, []string{"component", "result"})
|
||||
brandingLogoFetchTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_branding_logo_fetch_total",
|
||||
Help: "Total number of branding logo fetch operations.",
|
||||
}, []string{"component", "source", "result"})
|
||||
brandingLogoFetchDurationSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "app_branding_logo_fetch_duration_seconds",
|
||||
Help: "Branding logo fetch duration in seconds.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}, []string{"component", "source"})
|
||||
brandingRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "app_branding_requests_total",
|
||||
Help: "Total number of branding endpoint requests by endpoint, source, status, and result.",
|
||||
}, []string{"component", "endpoint", "source", "status_code", "result"})
|
||||
brandingRequestDurationSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "app_branding_request_duration_seconds",
|
||||
Help: "Branding endpoint request duration in seconds.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}, []string{"component", "endpoint"})
|
||||
)
|
||||
|
||||
func Register() {
|
||||
registerOnce.Do(func() {
|
||||
collectors := []prometheus.Collector{
|
||||
awsRequestsTotal,
|
||||
awsRequestDurationSeconds,
|
||||
awsErrorsTotal,
|
||||
awsTransferredBytesTotal,
|
||||
queueMessagesReceivedTotal,
|
||||
queueMessagesDeletedTotal,
|
||||
queueVisibilityChangesTotal,
|
||||
queueSendTotal,
|
||||
queueRetriesTotal,
|
||||
queueMessageBytesTotal,
|
||||
queueDepth,
|
||||
workerJobsProcessedTotal,
|
||||
workerJobDurationSeconds,
|
||||
emailSendTotal,
|
||||
emailSendDurationSeconds,
|
||||
fileUploadTotal,
|
||||
fileDeleteTotal,
|
||||
fileHeadTotal,
|
||||
filePresignDownloadTotal,
|
||||
brandingLogoFetchTotal,
|
||||
brandingLogoFetchDurationSeconds,
|
||||
brandingRequestsTotal,
|
||||
brandingRequestDurationSeconds,
|
||||
}
|
||||
for _, collector := range collectors {
|
||||
registerCollector(collector)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func PromHTTPHandler() http.Handler {
|
||||
Register()
|
||||
return promhttp.Handler()
|
||||
}
|
||||
|
||||
func ObserveAWSRequest(component, service, operation string, err error, started time.Time) {
|
||||
Register()
|
||||
component = normalizeLabel(component, "worker")
|
||||
service = normalizeLabel(service, "unknown")
|
||||
operation = normalizeLabel(operation, "unknown")
|
||||
result := resultLabel(err)
|
||||
|
||||
awsRequestsTotal.WithLabelValues(component, service, operation, result).Inc()
|
||||
awsRequestDurationSeconds.WithLabelValues(component, service, operation).Observe(time.Since(started).Seconds())
|
||||
if err != nil {
|
||||
awsErrorsTotal.WithLabelValues(component, service, operation, errorClass(err)).Inc()
|
||||
}
|
||||
}
|
||||
|
||||
func ObserveAWSTransferredBytes(component, service, operation, direction string, bytes int64) {
|
||||
Register()
|
||||
if bytes <= 0 {
|
||||
return
|
||||
}
|
||||
component = normalizeLabel(component, "worker")
|
||||
service = normalizeLabel(service, "unknown")
|
||||
operation = normalizeLabel(operation, "unknown")
|
||||
direction = normalizeLabel(direction, "unknown")
|
||||
awsTransferredBytesTotal.WithLabelValues(component, service, operation, direction).Add(float64(bytes))
|
||||
}
|
||||
|
||||
func ObserveQueueReceive(component, queueName string, messageCount int, messageBytes int64, err error, started time.Time) {
|
||||
ObserveAWSRequest(component, "sqs", "ReceiveMessage", err, started)
|
||||
if err != nil || messageCount <= 0 {
|
||||
return
|
||||
}
|
||||
component = normalizeLabel(component, "worker")
|
||||
queueName = normalizeLabel(queueName, "unknown")
|
||||
queueMessagesReceivedTotal.WithLabelValues(component, queueName).Add(float64(messageCount))
|
||||
if messageBytes > 0 {
|
||||
queueMessageBytesTotal.WithLabelValues(component, queueName, "receive").Add(float64(messageBytes))
|
||||
}
|
||||
}
|
||||
|
||||
func ObserveQueueDeleteBatch(component, queueName string, deletedCount int, err error, started time.Time) {
|
||||
ObserveAWSRequest(component, "sqs", "DeleteMessageBatch", err, started)
|
||||
if deletedCount <= 0 {
|
||||
return
|
||||
}
|
||||
component = normalizeLabel(component, "worker")
|
||||
queueName = normalizeLabel(queueName, "unknown")
|
||||
queueMessagesDeletedTotal.WithLabelValues(component, queueName).Add(float64(deletedCount))
|
||||
}
|
||||
|
||||
func ObserveQueueVisibilityChange(component, queueName string, err error, started time.Time) {
|
||||
ObserveAWSRequest(component, "sqs", "ChangeMessageVisibility", err, started)
|
||||
component = normalizeLabel(component, "worker")
|
||||
queueName = normalizeLabel(queueName, "unknown")
|
||||
result := resultLabel(err)
|
||||
queueVisibilityChangesTotal.WithLabelValues(component, queueName, result).Inc()
|
||||
queueRetriesTotal.WithLabelValues(component, queueName, result).Inc()
|
||||
}
|
||||
|
||||
func ObserveQueueSend(component, queueName string, messageBytes int64, err error, started time.Time) {
|
||||
ObserveAWSRequest(component, "sqs", "SendMessage", err, started)
|
||||
component = normalizeLabel(component, "worker")
|
||||
queueName = normalizeLabel(queueName, "unknown")
|
||||
result := resultLabel(err)
|
||||
queueSendTotal.WithLabelValues(component, queueName, result).Inc()
|
||||
if err == nil && messageBytes > 0 {
|
||||
queueMessageBytesTotal.WithLabelValues(component, queueName, "send").Add(float64(messageBytes))
|
||||
}
|
||||
}
|
||||
|
||||
func ObserveQueueDepth(component, queueName string, depth int64, err error, started time.Time) {
|
||||
ObserveAWSRequest(component, "sqs", "GetQueueAttributes", err, started)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
component = normalizeLabel(component, "worker")
|
||||
queueName = normalizeLabel(queueName, "unknown")
|
||||
queueDepth.WithLabelValues(component, queueName).Set(float64(depth))
|
||||
}
|
||||
|
||||
func InitializeQueueSeries(component, queueName string) {
|
||||
Register()
|
||||
component = normalizeLabel(component, "worker")
|
||||
queueName = normalizeLabel(queueName, "unknown")
|
||||
|
||||
queueMessagesReceivedTotal.WithLabelValues(component, queueName).Add(0)
|
||||
queueMessagesDeletedTotal.WithLabelValues(component, queueName).Add(0)
|
||||
queueVisibilityChangesTotal.WithLabelValues(component, queueName, "success").Add(0)
|
||||
queueVisibilityChangesTotal.WithLabelValues(component, queueName, "error").Add(0)
|
||||
queueSendTotal.WithLabelValues(component, queueName, "success").Add(0)
|
||||
queueSendTotal.WithLabelValues(component, queueName, "error").Add(0)
|
||||
queueRetriesTotal.WithLabelValues(component, queueName, "success").Add(0)
|
||||
queueRetriesTotal.WithLabelValues(component, queueName, "error").Add(0)
|
||||
queueMessageBytesTotal.WithLabelValues(component, queueName, "send").Add(0)
|
||||
queueMessageBytesTotal.WithLabelValues(component, queueName, "receive").Add(0)
|
||||
queueDepth.WithLabelValues(component, queueName).Set(0)
|
||||
}
|
||||
|
||||
func ObserveWorkerJob(component, jobType string, err error, started time.Time) {
|
||||
Register()
|
||||
component = normalizeLabel(component, "worker")
|
||||
jobType = normalizeLabel(jobType, "unknown")
|
||||
workerJobsProcessedTotal.WithLabelValues(component, jobType, resultLabel(err)).Inc()
|
||||
workerJobDurationSeconds.WithLabelValues(component, jobType).Observe(time.Since(started).Seconds())
|
||||
}
|
||||
|
||||
func ObserveEmailSend(component, provider string, err error, started time.Time) {
|
||||
Register()
|
||||
component = normalizeLabel(component, "worker")
|
||||
provider = normalizeLabel(provider, "unknown")
|
||||
emailSendTotal.WithLabelValues(component, provider, resultLabel(err)).Inc()
|
||||
emailSendDurationSeconds.WithLabelValues(component, provider).Observe(time.Since(started).Seconds())
|
||||
}
|
||||
|
||||
func ObserveFileUpload(component string, err error) {
|
||||
Register()
|
||||
fileUploadTotal.WithLabelValues(normalizeLabel(component, "api"), resultLabel(err)).Inc()
|
||||
}
|
||||
|
||||
func ObserveFileDelete(component string, err error) {
|
||||
Register()
|
||||
fileDeleteTotal.WithLabelValues(normalizeLabel(component, "api"), resultLabel(err)).Inc()
|
||||
}
|
||||
|
||||
func ObserveFileHead(component string, err error) {
|
||||
Register()
|
||||
fileHeadTotal.WithLabelValues(normalizeLabel(component, "api"), resultLabel(err)).Inc()
|
||||
}
|
||||
|
||||
func ObserveFilePresignDownload(component string, err error) {
|
||||
Register()
|
||||
filePresignDownloadTotal.WithLabelValues(normalizeLabel(component, "api"), resultLabel(err)).Inc()
|
||||
}
|
||||
|
||||
func ObserveBrandingLogoFetch(component, source string, err error, started time.Time) {
|
||||
Register()
|
||||
component = normalizeLabel(component, "api")
|
||||
source = normalizeLabel(source, "unknown")
|
||||
brandingLogoFetchTotal.WithLabelValues(component, source, resultLabel(err)).Inc()
|
||||
brandingLogoFetchDurationSeconds.WithLabelValues(component, source).Observe(time.Since(started).Seconds())
|
||||
}
|
||||
|
||||
func ObserveBrandingRequest(component, endpoint, source string, statusCode int, err error, started time.Time) {
|
||||
Register()
|
||||
component = normalizeLabel(component, "api")
|
||||
endpoint = normalizeLabel(endpoint, "unknown")
|
||||
source = normalizeLabel(source, "unknown")
|
||||
status := "0"
|
||||
if statusCode > 0 {
|
||||
status = normalizeLabel(fmt.Sprintf("%d", statusCode), "0")
|
||||
}
|
||||
brandingRequestsTotal.WithLabelValues(component, endpoint, source, status, resultLabel(err)).Inc()
|
||||
brandingRequestDurationSeconds.WithLabelValues(component, endpoint).Observe(time.Since(started).Seconds())
|
||||
}
|
||||
|
||||
func QueueNameFromURL(queueURL string) string {
|
||||
raw := strings.TrimSpace(queueURL)
|
||||
if raw == "" {
|
||||
return "unknown"
|
||||
}
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return "unknown"
|
||||
}
|
||||
name := strings.TrimSpace(path.Base(parsed.Path))
|
||||
if name == "" || name == "." || name == "/" {
|
||||
return "unknown"
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func registerCollector(collector prometheus.Collector) {
|
||||
if err := prometheus.Register(collector); err != nil {
|
||||
var alreadyRegistered prometheus.AlreadyRegisteredError
|
||||
if errors.As(err, &alreadyRegistered) {
|
||||
return
|
||||
}
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func errorClass(err error) string {
|
||||
statusCode := statusCodeFromError(err)
|
||||
if statusCode >= 400 && statusCode < 500 {
|
||||
return "4xx"
|
||||
}
|
||||
if statusCode >= 500 && statusCode < 600 {
|
||||
return "5xx"
|
||||
}
|
||||
return "other"
|
||||
}
|
||||
|
||||
func statusCodeFromError(err error) int {
|
||||
if err == nil {
|
||||
return 0
|
||||
}
|
||||
type statusCoder interface {
|
||||
HTTPStatusCode() int
|
||||
}
|
||||
var sc statusCoder
|
||||
if errors.As(err, &sc) {
|
||||
return sc.HTTPStatusCode()
|
||||
}
|
||||
var respErr *smithyhttp.ResponseError
|
||||
if errors.As(err, &respErr) {
|
||||
return respErr.HTTPStatusCode()
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func normalizeLabel(value, fallback string) string {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func resultLabel(err error) string {
|
||||
if err != nil {
|
||||
return "error"
|
||||
}
|
||||
return "success"
|
||||
}
|
||||
Reference in New Issue
Block a user