385 lines
14 KiB
Go
385 lines
14 KiB
Go
package metrics
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"path"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
smithyhttp "github.com/aws/smithy-go/transport/http"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
)
|
|
|
|
var (
|
|
registerOnce sync.Once
|
|
|
|
awsRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_aws_requests_total",
|
|
Help: "Total number of AWS requests by component, service, operation, and result.",
|
|
}, []string{"component", "service", "operation", "result"})
|
|
awsRequestDurationSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "app_aws_request_duration_seconds",
|
|
Help: "AWS request duration in seconds by component, service, and operation.",
|
|
Buckets: prometheus.DefBuckets,
|
|
}, []string{"component", "service", "operation"})
|
|
awsErrorsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_aws_errors_total",
|
|
Help: "Total number of AWS errors by class.",
|
|
}, []string{"component", "service", "operation", "error_class"})
|
|
awsTransferredBytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_aws_transferred_bytes_total",
|
|
Help: "Total AWS transferred bytes by component, service, operation, and direction.",
|
|
}, []string{"component", "service", "operation", "direction"})
|
|
|
|
queueMessagesReceivedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_queue_messages_received_total",
|
|
Help: "Total number of queue messages received.",
|
|
}, []string{"component", "queue_name"})
|
|
queueMessagesDeletedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_queue_messages_deleted_total",
|
|
Help: "Total number of queue messages deleted.",
|
|
}, []string{"component", "queue_name"})
|
|
queueVisibilityChangesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_queue_visibility_changes_total",
|
|
Help: "Total number of queue visibility timeout changes.",
|
|
}, []string{"component", "queue_name", "result"})
|
|
queueSendTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_queue_send_total",
|
|
Help: "Total number of queue send attempts.",
|
|
}, []string{"component", "queue_name", "result"})
|
|
queueRetriesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_queue_retries_total",
|
|
Help: "Total number of queue retry/requeue attempts.",
|
|
}, []string{"component", "queue_name", "result"})
|
|
queueMessageBytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_queue_message_bytes_total",
|
|
Help: "Total queue message bytes observed on send and receive.",
|
|
}, []string{"component", "queue_name", "direction"})
|
|
queueDepth = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
Name: "app_queue_depth",
|
|
Help: "Approximate queue depth from SQS attributes.",
|
|
}, []string{"component", "queue_name"})
|
|
|
|
workerJobsProcessedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_worker_jobs_processed_total",
|
|
Help: "Total number of worker jobs processed.",
|
|
}, []string{"component", "job_type", "result"})
|
|
workerJobDurationSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "app_worker_job_duration_seconds",
|
|
Help: "Worker job processing duration in seconds.",
|
|
Buckets: prometheus.DefBuckets,
|
|
}, []string{"component", "job_type"})
|
|
|
|
emailSendTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_email_send_total",
|
|
Help: "Total number of email send attempts.",
|
|
}, []string{"component", "provider", "result"})
|
|
emailSendDurationSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "app_email_send_duration_seconds",
|
|
Help: "Email send duration in seconds.",
|
|
Buckets: prometheus.DefBuckets,
|
|
}, []string{"component", "provider"})
|
|
|
|
fileUploadTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_file_upload_total",
|
|
Help: "Total number of file upload operations.",
|
|
}, []string{"component", "result"})
|
|
fileDeleteTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_file_delete_total",
|
|
Help: "Total number of file delete operations.",
|
|
}, []string{"component", "result"})
|
|
fileHeadTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_file_head_total",
|
|
Help: "Total number of file head operations.",
|
|
}, []string{"component", "result"})
|
|
filePresignDownloadTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_file_presign_download_total",
|
|
Help: "Total number of file presign download operations.",
|
|
}, []string{"component", "result"})
|
|
brandingLogoFetchTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_branding_logo_fetch_total",
|
|
Help: "Total number of branding logo fetch operations.",
|
|
}, []string{"component", "source", "result"})
|
|
brandingLogoFetchDurationSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "app_branding_logo_fetch_duration_seconds",
|
|
Help: "Branding logo fetch duration in seconds.",
|
|
Buckets: prometheus.DefBuckets,
|
|
}, []string{"component", "source"})
|
|
brandingRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "app_branding_requests_total",
|
|
Help: "Total number of branding endpoint requests by endpoint, source, status, and result.",
|
|
}, []string{"component", "endpoint", "source", "status_code", "result"})
|
|
brandingRequestDurationSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "app_branding_request_duration_seconds",
|
|
Help: "Branding endpoint request duration in seconds.",
|
|
Buckets: prometheus.DefBuckets,
|
|
}, []string{"component", "endpoint"})
|
|
)
|
|
|
|
func Register() {
|
|
registerOnce.Do(func() {
|
|
collectors := []prometheus.Collector{
|
|
awsRequestsTotal,
|
|
awsRequestDurationSeconds,
|
|
awsErrorsTotal,
|
|
awsTransferredBytesTotal,
|
|
queueMessagesReceivedTotal,
|
|
queueMessagesDeletedTotal,
|
|
queueVisibilityChangesTotal,
|
|
queueSendTotal,
|
|
queueRetriesTotal,
|
|
queueMessageBytesTotal,
|
|
queueDepth,
|
|
workerJobsProcessedTotal,
|
|
workerJobDurationSeconds,
|
|
emailSendTotal,
|
|
emailSendDurationSeconds,
|
|
fileUploadTotal,
|
|
fileDeleteTotal,
|
|
fileHeadTotal,
|
|
filePresignDownloadTotal,
|
|
brandingLogoFetchTotal,
|
|
brandingLogoFetchDurationSeconds,
|
|
brandingRequestsTotal,
|
|
brandingRequestDurationSeconds,
|
|
}
|
|
for _, collector := range collectors {
|
|
registerCollector(collector)
|
|
}
|
|
})
|
|
}
|
|
|
|
func PromHTTPHandler() http.Handler {
|
|
Register()
|
|
return promhttp.Handler()
|
|
}
|
|
|
|
func ObserveAWSRequest(component, service, operation string, err error, started time.Time) {
|
|
Register()
|
|
component = normalizeLabel(component, "worker")
|
|
service = normalizeLabel(service, "unknown")
|
|
operation = normalizeLabel(operation, "unknown")
|
|
result := resultLabel(err)
|
|
|
|
awsRequestsTotal.WithLabelValues(component, service, operation, result).Inc()
|
|
awsRequestDurationSeconds.WithLabelValues(component, service, operation).Observe(time.Since(started).Seconds())
|
|
if err != nil {
|
|
awsErrorsTotal.WithLabelValues(component, service, operation, errorClass(err)).Inc()
|
|
}
|
|
}
|
|
|
|
func ObserveAWSTransferredBytes(component, service, operation, direction string, bytes int64) {
|
|
Register()
|
|
if bytes <= 0 {
|
|
return
|
|
}
|
|
component = normalizeLabel(component, "worker")
|
|
service = normalizeLabel(service, "unknown")
|
|
operation = normalizeLabel(operation, "unknown")
|
|
direction = normalizeLabel(direction, "unknown")
|
|
awsTransferredBytesTotal.WithLabelValues(component, service, operation, direction).Add(float64(bytes))
|
|
}
|
|
|
|
func ObserveQueueReceive(component, queueName string, messageCount int, messageBytes int64, err error, started time.Time) {
|
|
ObserveAWSRequest(component, "sqs", "ReceiveMessage", err, started)
|
|
if err != nil || messageCount <= 0 {
|
|
return
|
|
}
|
|
component = normalizeLabel(component, "worker")
|
|
queueName = normalizeLabel(queueName, "unknown")
|
|
queueMessagesReceivedTotal.WithLabelValues(component, queueName).Add(float64(messageCount))
|
|
if messageBytes > 0 {
|
|
queueMessageBytesTotal.WithLabelValues(component, queueName, "receive").Add(float64(messageBytes))
|
|
}
|
|
}
|
|
|
|
func ObserveQueueDeleteBatch(component, queueName string, deletedCount int, err error, started time.Time) {
|
|
ObserveAWSRequest(component, "sqs", "DeleteMessageBatch", err, started)
|
|
if deletedCount <= 0 {
|
|
return
|
|
}
|
|
component = normalizeLabel(component, "worker")
|
|
queueName = normalizeLabel(queueName, "unknown")
|
|
queueMessagesDeletedTotal.WithLabelValues(component, queueName).Add(float64(deletedCount))
|
|
}
|
|
|
|
func ObserveQueueVisibilityChange(component, queueName string, err error, started time.Time) {
|
|
ObserveAWSRequest(component, "sqs", "ChangeMessageVisibility", err, started)
|
|
component = normalizeLabel(component, "worker")
|
|
queueName = normalizeLabel(queueName, "unknown")
|
|
result := resultLabel(err)
|
|
queueVisibilityChangesTotal.WithLabelValues(component, queueName, result).Inc()
|
|
queueRetriesTotal.WithLabelValues(component, queueName, result).Inc()
|
|
}
|
|
|
|
func ObserveQueueSend(component, queueName string, messageBytes int64, err error, started time.Time) {
|
|
ObserveAWSRequest(component, "sqs", "SendMessage", err, started)
|
|
component = normalizeLabel(component, "worker")
|
|
queueName = normalizeLabel(queueName, "unknown")
|
|
result := resultLabel(err)
|
|
queueSendTotal.WithLabelValues(component, queueName, result).Inc()
|
|
if err == nil && messageBytes > 0 {
|
|
queueMessageBytesTotal.WithLabelValues(component, queueName, "send").Add(float64(messageBytes))
|
|
}
|
|
}
|
|
|
|
func ObserveQueueDepth(component, queueName string, depth int64, err error, started time.Time) {
|
|
ObserveAWSRequest(component, "sqs", "GetQueueAttributes", err, started)
|
|
if err != nil {
|
|
return
|
|
}
|
|
component = normalizeLabel(component, "worker")
|
|
queueName = normalizeLabel(queueName, "unknown")
|
|
queueDepth.WithLabelValues(component, queueName).Set(float64(depth))
|
|
}
|
|
|
|
func InitializeQueueSeries(component, queueName string) {
|
|
Register()
|
|
component = normalizeLabel(component, "worker")
|
|
queueName = normalizeLabel(queueName, "unknown")
|
|
|
|
queueMessagesReceivedTotal.WithLabelValues(component, queueName).Add(0)
|
|
queueMessagesDeletedTotal.WithLabelValues(component, queueName).Add(0)
|
|
queueVisibilityChangesTotal.WithLabelValues(component, queueName, "success").Add(0)
|
|
queueVisibilityChangesTotal.WithLabelValues(component, queueName, "error").Add(0)
|
|
queueSendTotal.WithLabelValues(component, queueName, "success").Add(0)
|
|
queueSendTotal.WithLabelValues(component, queueName, "error").Add(0)
|
|
queueRetriesTotal.WithLabelValues(component, queueName, "success").Add(0)
|
|
queueRetriesTotal.WithLabelValues(component, queueName, "error").Add(0)
|
|
queueMessageBytesTotal.WithLabelValues(component, queueName, "send").Add(0)
|
|
queueMessageBytesTotal.WithLabelValues(component, queueName, "receive").Add(0)
|
|
queueDepth.WithLabelValues(component, queueName).Set(0)
|
|
}
|
|
|
|
func ObserveWorkerJob(component, jobType string, err error, started time.Time) {
|
|
Register()
|
|
component = normalizeLabel(component, "worker")
|
|
jobType = normalizeLabel(jobType, "unknown")
|
|
workerJobsProcessedTotal.WithLabelValues(component, jobType, resultLabel(err)).Inc()
|
|
workerJobDurationSeconds.WithLabelValues(component, jobType).Observe(time.Since(started).Seconds())
|
|
}
|
|
|
|
func ObserveEmailSend(component, provider string, err error, started time.Time) {
|
|
Register()
|
|
component = normalizeLabel(component, "worker")
|
|
provider = normalizeLabel(provider, "unknown")
|
|
emailSendTotal.WithLabelValues(component, provider, resultLabel(err)).Inc()
|
|
emailSendDurationSeconds.WithLabelValues(component, provider).Observe(time.Since(started).Seconds())
|
|
}
|
|
|
|
func ObserveFileUpload(component string, err error) {
|
|
Register()
|
|
fileUploadTotal.WithLabelValues(normalizeLabel(component, "api"), resultLabel(err)).Inc()
|
|
}
|
|
|
|
func ObserveFileDelete(component string, err error) {
|
|
Register()
|
|
fileDeleteTotal.WithLabelValues(normalizeLabel(component, "api"), resultLabel(err)).Inc()
|
|
}
|
|
|
|
func ObserveFileHead(component string, err error) {
|
|
Register()
|
|
fileHeadTotal.WithLabelValues(normalizeLabel(component, "api"), resultLabel(err)).Inc()
|
|
}
|
|
|
|
func ObserveFilePresignDownload(component string, err error) {
|
|
Register()
|
|
filePresignDownloadTotal.WithLabelValues(normalizeLabel(component, "api"), resultLabel(err)).Inc()
|
|
}
|
|
|
|
func ObserveBrandingLogoFetch(component, source string, err error, started time.Time) {
|
|
Register()
|
|
component = normalizeLabel(component, "api")
|
|
source = normalizeLabel(source, "unknown")
|
|
brandingLogoFetchTotal.WithLabelValues(component, source, resultLabel(err)).Inc()
|
|
brandingLogoFetchDurationSeconds.WithLabelValues(component, source).Observe(time.Since(started).Seconds())
|
|
}
|
|
|
|
func ObserveBrandingRequest(component, endpoint, source string, statusCode int, err error, started time.Time) {
|
|
Register()
|
|
component = normalizeLabel(component, "api")
|
|
endpoint = normalizeLabel(endpoint, "unknown")
|
|
source = normalizeLabel(source, "unknown")
|
|
status := "0"
|
|
if statusCode > 0 {
|
|
status = normalizeLabel(fmt.Sprintf("%d", statusCode), "0")
|
|
}
|
|
brandingRequestsTotal.WithLabelValues(component, endpoint, source, status, resultLabel(err)).Inc()
|
|
brandingRequestDurationSeconds.WithLabelValues(component, endpoint).Observe(time.Since(started).Seconds())
|
|
}
|
|
|
|
func QueueNameFromURL(queueURL string) string {
|
|
raw := strings.TrimSpace(queueURL)
|
|
if raw == "" {
|
|
return "unknown"
|
|
}
|
|
parsed, err := url.Parse(raw)
|
|
if err != nil {
|
|
return "unknown"
|
|
}
|
|
name := strings.TrimSpace(path.Base(parsed.Path))
|
|
if name == "" || name == "." || name == "/" {
|
|
return "unknown"
|
|
}
|
|
return name
|
|
}
|
|
|
|
func registerCollector(collector prometheus.Collector) {
|
|
if err := prometheus.Register(collector); err != nil {
|
|
var alreadyRegistered prometheus.AlreadyRegisteredError
|
|
if errors.As(err, &alreadyRegistered) {
|
|
return
|
|
}
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
func errorClass(err error) string {
|
|
statusCode := statusCodeFromError(err)
|
|
if statusCode >= 400 && statusCode < 500 {
|
|
return "4xx"
|
|
}
|
|
if statusCode >= 500 && statusCode < 600 {
|
|
return "5xx"
|
|
}
|
|
return "other"
|
|
}
|
|
|
|
func statusCodeFromError(err error) int {
|
|
if err == nil {
|
|
return 0
|
|
}
|
|
type statusCoder interface {
|
|
HTTPStatusCode() int
|
|
}
|
|
var sc statusCoder
|
|
if errors.As(err, &sc) {
|
|
return sc.HTTPStatusCode()
|
|
}
|
|
var respErr *smithyhttp.ResponseError
|
|
if errors.As(err, &respErr) {
|
|
return respErr.HTTPStatusCode()
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func normalizeLabel(value, fallback string) string {
|
|
value = strings.ToLower(strings.TrimSpace(value))
|
|
if value == "" {
|
|
return fallback
|
|
}
|
|
return value
|
|
}
|
|
|
|
func resultLabel(err error) string {
|
|
if err != nil {
|
|
return "error"
|
|
}
|
|
return "success"
|
|
}
|