Go语言监控告警生产环境运维1. 引言生产环境的稳定运行离不开完善的监控告警体系。监控不仅是发现问题的眼睛更是预防问题发生的预警系统。本文将深入讲解Go语言微服务中的监控指标设计、Grafana看板配置、日志聚合、告警规则和on-call流程。2. Prometheus指标设计2.1 Prometheus客户端使用package main import ( net/http time github.com/gin-gonic/gin github.com/prometheus/client_golang/prometheus github.com/prometheus/client_golang/prometheus/promhttp ) var ( // HTTP请求计数器 httpRequestsTotal prometheus.NewCounterVec( prometheus.CounterOpts{ Name: http_requests_total, Help: Total number of HTTP requests, }, []string{method, endpoint, status}, ) // HTTP请求延迟直方图 httpRequestDuration prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: http_request_duration_seconds, Help: HTTP request duration in seconds, Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}, }, []string{method, endpoint}, ) // HTTP请求正在处理的 Gauge httpRequestsInProgress prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: http_requests_in_progress, Help: Number of HTTP requests currently being processed, }, []string{method}, ) // 业务指标订单数量 ordersTotal prometheus.NewCounter( prometheus.CounterOpts{ Name: orders_total, Help: Total number of orders, }, ) // 业务指标订单金额 orderAmountTotal prometheus.NewCounterVec( prometheus.CounterOpts{ Name: order_amount_total, Help: Total amount of orders, }, []string{status}, ) // 数据库连接池指标 dbConnections prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: db_connections, Help: Number of database connections, }, []string{state}, ) // Redis连接池指标 redisConnections prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: redis_connections, Help: Number of Redis connections, }, []string{state}, ) ) func init() { // 注册所有指标 prometheus.MustRegister( httpRequestsTotal, httpRequestDuration, httpRequestsInProgress, ordersTotal, orderAmountTotal, dbConnections, redisConnections, ) } func main() { r : gin.Default() // Prometheus指标端点 r.GET(/metrics, gin.WrapH(promhttp.Handler())) // 使用中间件收集指标 r.Use(metricsMiddleware()) // 业务路由 r.GET(/api/v1/users, listUsers) r.POST(/api/v1/orders, createOrder) _ r.Run(:8080) } func metricsMiddleware() gin.HandlerFunc { return func(c *gin.Context) { if c.Request.URL.Path /metrics { c.Next() return } // 记录正在处理的请求 httpRequestsInProgress.WithLabelValues(c.Request.Method).Inc() defer httpRequestsInProgress.WithLabelValues(c.Request.Method).Dec() start : time.Now() // 处理请求 c.Next() // 记录请求指标 duration : time.Since(start).Seconds() status : fmt.Sprintf(%d, c.Writer.Status()) endpoint : c.FullPath() if endpoint { endpoint unknown } httpRequestsTotal.WithLabelValues( c.Request.Method, endpoint, status, ).Inc() httpRequestDuration.WithLabelValues( c.Request.Method, endpoint, ).Observe(duration) } }2.2 自定义业务指标package metrics import ( github.com/prometheus/client_golang/prometheus ) // 业务指标收集器 type BusinessMetrics struct { userRegistrations prometheus.Counter userLogins prometheus.Counter apiCallsTotal *prometheus.CounterVec apiLatency *prometheus.HistogramVec cacheHitRatio prometheus.Gauge queueSize prometheus.Gauge } func NewBusinessMetrics() *BusinessMetrics { m : BusinessMetrics{ userRegistrations: prometheus.NewCounter(prometheus.CounterOpts{ Name: user_registrations_total, Help: Total number of user registrations, }), userLogins: prometheus.NewCounter(prometheus.CounterOpts{ Name: user_logins_total, Help: Total number of user logins, }), apiCallsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: api_calls_total, Help: Total number of API calls, }, []string{service, method, status}), apiLatency: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: api_latency_seconds, Help: API latency in seconds, Buckets: prometheus.DefBuckets, }, []string{service, method}), cacheHitRatio: prometheus.NewGauge(prometheus.GaugeOpts{ Name: cache_hit_ratio, Help: Cache hit ratio, }), queueSize: prometheus.NewGauge(prometheus.GaugeOpts{ Name: queue_size, Help: Current queue size, }), } prometheus.MustRegister( m.userRegistrations, m.userLogins, m.apiCallsTotal, m.apiLatency, m.cacheHitRatio, m.queueSize, ) return m } // 记录用户注册 func (m *BusinessMetrics) RecordUserRegistration() { m.userRegistrations.Inc() } // 记录用户登录 func (m *BusinessMetrics) RecordUserLogin() { m.userLogins.Inc() } // 记录API调用 func (m *BusinessMetrics) RecordAPICall(service, method, status string) { m.apiCallsTotal.WithLabelValues(service, method, status).Inc() } // 记录API延迟 func (m *BusinessMetrics) RecordAPILatency(service, method string, duration float64) { m.apiLatency.WithLabelValues(service, method).Observe(duration) } // 设置缓存命中率 func (m *BusinessMetrics) SetCacheHitRatio(ratio float64) { m.cacheHitRatio.Set(ratio) } // 设置队列大小 func (m *BusinessMetrics) SetQueueSize(size float64) { m.queueSize.Set(size) }3. Grafana看板配置3.1 Grafana Dashboard JSON{ dashboard: { title: Go Microservice Dashboard, uid: go-microservice, version: 1, panels: [ { id: 1, title: Request Rate (QPS), type: graph, gridPos: {x: 0, y: 0, w: 12, h: 8}, targets: [ { expr: rate(http_requests_total[5m]), legendFormat: {{method}} {{endpoint}} {{status}} } ] }, { id: 2, title: Request Latency (p50, p95, p99), type: graph, gridPos: {x: 12, y: 0, w: 12, h: 8}, targets: [ { expr: histogram_quantile(0.50, rate(http_request_duration_seconds_bucket[5m])), legendFormat: p50 }, { expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])), legendFormat: p95 }, { expr: histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m])), legendFormat: p99 } ] }, { id: 3, title: Error Rate, type: graph, gridPos: {x: 0, y: 8, w: 12, h: 8}, targets: [ { expr: rate(http_requests_total{status~\5..\}[5m]), legendFormat: 5xx Error Rate } ] }, { id: 4, title: CPU Memory Usage, type: graph, gridPos: {x: 12, y: 8, w: 12, h: 8}, targets: [ { expr: process_cpu_seconds_total, legendFormat: CPU Seconds }, { expr: process_resident_memory_bytes, legendFormat: Memory Bytes } ] }, { id: 5, title: Goroutine Count, type: graph, gridPos: {x: 0, y: 16, w: 12, h: 8}, targets: [ { expr: go_goroutines, legendFormat: Goroutines } ] }, { id: 6, title: GC Stats, type: graph, gridPos: {x: 12, y: 16, w: 12, h: 8}, targets: [ { expr: rate(go_gc_duration_seconds_sum[5m]), legendFormat: GC Duration }, { expr: go_memstats_heap_alloc_bytes, legendFormat: Heap Allocated } ] } ] } }3.2 Grafana告警规则# grafana-alerts.yaml apiVersion: 1 groups: - name: go-microservice-alerts interval: 30s rules: # 高错误率告警 - alert: HighErrorRate expr: rate(http_requests_total{status~5..}[5m]) 0.05 for: 2m labels: severity: critical annotations: summary: High error rate detected description: Error rate is {{ $value | humanizePercentage }} (threshold: 5%) # 服务延迟过高告警 - alert: HighLatency expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) 1 for: 5m labels: severity: warning annotations: summary: High latency detected description: p95 latency is {{ $value | humanizeDuration }} (threshold: 1s) # 内存使用过高告警 - alert: HighMemoryUsage expr: process_resident_memory_bytes / 1024 / 1024 / 1024 2 for: 5m labels: severity: warning annotations: summary: High memory usage description: Memory usage is {{ $value | humanize }}GB (threshold: 2GB) # Goroutine泄漏告警 - alert: GoroutineLeak expr: go_goroutines 10000 for: 10m labels: severity: warning annotations: summary: Potential goroutine leak description: Goroutine count is {{ $value }} (threshold: 10000) # 服务不可用告警 - alert: ServiceDown expr: up{jobgo-microservice} 0 for: 1m labels: severity: critical annotations: summary: Service is down description: Service {{ $labels.instance }} is not responding4. 日志聚合方案4.1 结构化日志输出package logging import ( os time go.uber.org/zap go.uber.org/zap/zapcore ) // Logger 结构化日志器 var Logger *zap.Logger func InitLogger(env string) error { var config zap.Config if env production { config zap.NewProductionConfig() config.EncoderConfig.TimeKey timestamp config.EncoderConfig.EncodeTime zapcore.ISO8601TimeEncoder } else { config zap.NewDevelopmentConfig() config.EncoderConfig.EncodeLevel zapcore.CapitalColorLevelEncoder } var err error Logger, err config.Build( zap.AddCallerSkip(1), zap.AddStacktrace(zapcore.ErrorLevel), ) return err } // 记录带上下文的日志 func LogHTTPRequest(method, path string, status int, latency time.Duration, ip string) { Logger.Info(http_request, zap.String(method, method), zap.String(path, path), zap.Int(status, status), zap.Duration(latency, latency), zap.String(client_ip, ip), zap.String(service, api-gateway), ) } func LogError(operation string, err error, details map[string]interface{}) { fields : []zap.Field{ zap.String(operation, operation), zap.Error(err), } for k, v : range details { fields append(fields, zap.Any(k, v)) } Logger.Error(operation_failed, fields...) }4.2 日志收集配置# filebeat.yml filebeat.inputs: - type: container paths: - /var/log/containers/*.log processors: - add_kubernetes_metadata: host: ${NODE_NAME} matchers: - logs_path: logs_path: /var/log/containers/ - type: log paths: - /var/log/app/*.log fields: service: myapp environment: production fields_under_root: true output.logstash: hosts: [logstash:5044] processors: - add_host_metadata: when.not.contains.tags: forwarded - add_cloud_metadata: ~ - add_docker_metadata: ~4.3 Loki日志查询# 查询最近5分钟内的错误日志 {jobmyapp} | error | json | levelerror # 查询特定用户的请求日志 {jobmyapp} | user_id123 | json # 统计每分钟错误数量 sum by (level) (count_over_time({jobmyapp} | error [1m])) # 查询延迟最高的请求 topk(10, sum by (path) (rate(http_request_duration_seconds_sum[5m])) / sum by (path) (rate(http_request_duration_seconds_count[5m]))) # 关联指标和日志 sum by (path) (rate(http_requests_total{status500}[5m])) * 10005. 链路追踪5.1 OpenTelemetry集成package tracing import ( context fmt go.opentelemetry.io/otel go.opentelemetry.io/otel/attribute go.opentelemetry.io/otel/exporters/jaeger go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc go.opentelemetry.io/otel/propagation go.opentelemetry.io/otel/sdk/resource go.opentelemetry.io/otel/sdk/trace semconv go.opentelemetry.io/otel/semconv/v1.21.0 ) func InitTracer(serviceName, endpoint string) (func(context.Context) error, error) { // 创建Jaeger exporter exporter, err : jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(endpoint))) if err ! nil { return nil, err } // 或者使用OTLP exporter // exporter, err : otlptracegrpc.New(context.Background(), // otlptracegrpc.WithEndpoint(endpoint), // otlptracegrpc.WithInsecure(), // ) // 创建资源信息 res, err : resource.Merge( resource.Default(), resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceName(serviceName), semconv.ServiceVersion(1.0.0), attribute.String(environment, production), ), ) // 创建trace provider tp : trace.NewTracerProvider( trace.WithBatcher(exporter), trace.WithResource(res), trace.WithSampler(trace.ParentBased(trace.AlwaysSample())), ) otel.SetTracerProvider(tp) otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( propagation.TraceContext{}, propagation.Baggage{}, )) return tp.Shutdown, nil } // 追踪函数 func TraceFunc(ctx context.Context, tracer otel.Tracer, name string, fn func(context.Context) error) error { ctx, span : tracer.Start(ctx, name) defer span.End() if err : fn(ctx); err ! nil { span.RecordError(err) return err } return nil } // Gin中间件链路追踪 func TracingMiddleware(serviceName string) gin.HandlerFunc { tracer : otel.Tracer(serviceName) return func(c *gin.Context) { ctx : c.Request.Context() // 提取传播上下文 ctx otel.GetTextMapPropagator().Extract(ctx, propagation.HeaderCarrier(c.Request.Header)) ctx, span : tracer.Start(ctx, c.Request.Method c.FullPath()) defer span.End() span.SetAttributes( attribute.String(http.method, c.Request.Method), attribute.String(http.url, c.Request.URL.String()), attribute.String(http.host, c.Request.Host), attribute.String(http.user_agent, c.Request.UserAgent()), ) c.Request c.Request.WithContext(ctx) c.Next() span.SetAttributes(attribute.Int(http.status_code, c.Writer.Status())) } }6. 告警规则配置6.1 Prometheus告警规则# prometheus-alerts.yml groups: - name: microservice-alerts rules: # 服务可用性 - alert: InstanceDown expr: up{jobmyapp} 0 for: 1m labels: severity: critical annotations: summary: Instance {{ $labels.instance }} is down description: {{ $labels.job }} has been down for more than 1 minute # HTTP错误率 - alert: HighHTTPErrorRate expr: | sum(rate(http_requests_total{status~5..}[5m])) by (service) / sum(rate(http_requests_total[5m])) by (service) 0.05 for: 5m labels: severity: warning annotations: summary: High HTTP error rate in {{ $labels.service }} description: Error rate is {{ $value | humanizePercentage }} # 延迟SLO - alert: LatencySLOBreach expr: | histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[10m])) by (le, service)) 2 for: 10m labels: severity: warning annotations: summary: Latency SLO breach in {{ $labels.service }} description: p99 latency is {{ $value }}s, exceeds 2s threshold # 资源使用 - alert: HighCPUUsage expr: rate(process_cpu_seconds_total[5m]) * 100 80 for: 10m labels: severity: warning annotations: summary: High CPU usage description: CPU usage is {{ $value }}% - alert: HighMemoryUsage expr: process_resident_memory_bytes / 1024 / 1024 / 1024 3 for: 5m labels: severity: warning annotations: summary: High memory usage description: Memory usage is {{ $value | humanize }}GB # 数据库连接池 - alert: DatabaseConnectionPoolExhausted expr: | db_connections{statein_use} / db_connections{stateidle} 0.8 for: 5m labels: severity: critical annotations: summary: Database connection pool nearly exhausted # 业务指标异常 - alert: OrderAmountDropped expr: | sum(rate(order_amount_total[5m])) 100 for: 30m labels: severity: warning annotations: summary: Order amount dropped significantly description: Order rate is {{ $value }} per second6.2 AlertManager配置# alertmanager.yml global: resolve_timeout: 5m smtp_smarthost: smtp.example.com:587 smtp_from: alertmanagerexample.com route: group_by: [alertname, severity] group_wait: 30s group_interval: 5m repeat_interval: 4h receiver: default-receiver routes: - match: severity: critical receiver: critical-receiver group_wait: 10s - match: severity: warning receiver: warning-receiver receivers: - name: default-receiver email_configs: - to: oncallexample.com headers: subject: Alert: {{ .GroupLabels.alertname }} - name: critical-receiver pagerduty_configs: - service_key: YOUR_PAGERDUTY_KEY severity: critical slack_configs: - channel: #critical-alerts api_url: YOUR_SLACK_WEBHOOK - name: warning-receiver slack_configs: - channel: #warnings api_url: YOUR_SLACK_WEBHOOK7. On-Call流程7.1 On-Call管理package oncall import ( fmt time ) // OnCallSchedule 值班表 type OnCallSchedule struct { CurrentOnCall *OnCallEngineer NextOnCall *OnCallEngineer EscalationList []OnCallEngineer } // OnCallEngineer 值班工程师 type OnCallEngineer struct { ID string Name string Email string Phone string StartTime time.Time EndTime time.Time } // RotationManager 值班轮换管理器 type RotationManager struct { engineers []OnCallEngineer currentIdx int } func NewRotationManager(engineers []OnCallEngineer) *RotationManager { return RotationManager{ engineers: engineers, currentIdx: 0, } } func (m *RotationManager) GetCurrentOnCall() *OnCallEngineer { return m.engineers[m.currentIdx] } func (m *RotationManager) GetNextOnCall() *OnCallEngineer { nextIdx : (m.currentIdx 1) % len(m.engineers) return m.engineers[nextIdx] } func (m *RotationManager) Rotate() { m.currentIdx (m.currentIdx 1) % len(m.engineers) } // Incident 事件 type Incident struct { ID string Title string Description string Severity string // critical, high, medium, low Status string // open, acknowledged, resolved CreatedAt time.Time ResolvedAt *time.Time Assignee *OnCallEngineer } // IncidentManager 事件管理器 type IncidentManager struct { incidents map[string]*Incident } func NewIncidentManager() *IncidentManager { return IncidentManager{ incidents: make(map[string]*Incident), } } func (m *IncidentManager) CreateIncident(title, desc, severity string, assignee *OnCallEngineer) *Incident { inc : Incident{ ID: fmt.Sprintf(INC-%d, time.Now().Unix()), Title: title, Description: desc, Severity: severity, Status: open, CreatedAt: time.Now(), Assignee: assignee, } m.incidents[inc.ID] inc return inc } func (m *IncidentManager) Acknowledge(id string) error { inc, ok : m.incidents[id] if !ok { return fmt.Errorf(incident not found) } inc.Status acknowledged return nil } func (m *IncidentManager) Resolve(id string) error { inc, ok : m.incidents[id] if !ok { return fmt.Errorf(incident not found) } inc.Status resolved now : time.Now() inc.ResolvedAt now return nil }7.2 值班脚本#!/bin/bash # check_oncall.sh - 检查当前值班人并发送通知 ONCALL_APIhttps://oncall-api.example.com SLACK_WEBHOOKhttps://hooks.slack.com/services/XXX # 获取当前值班人 CURRENT_ONCALL$(curl -s ${ONCALL_API}/current) # 检查是否有未解决的告警 ALERT_COUNT$(curl -s ${ONCALL_API}/alerts?statusopen | jq length) if [ $ALERT_COUNT -gt 0 ]; then MESSAGE⚠️ *On-Call Alert*: You have ${ALERT_COUNT} unresolved alerts. Please check ${ONCALL_API} # 发送Slack通知 curl -X POST -H Content-type: application/json \ --data {\text\: \${MESSAGE}\} \ ${SLACK_WEBHOOK} fi8. 总结完善的监控告警体系是生产环境运维的基石Prometheus指标设计从基础设施、业务、性能三个维度设计指标覆盖系统运行全貌Grafana看板通过可视化看板快速了解系统状态设置合理的图表布局日志聚合使用结构化日志支持多维度查询便于问题追溯链路追踪通过OpenTelemetry实现分布式追踪快速定位跨服务问题告警规则设置合理的阈值和告警级别避免告警疲劳On-Call流程建立完善的值班和事件升级机制确保问题及时处理监控告警需要持续优化根据业务发展和问题经验不断完善告警规则和处理流程。
Go语言监控告警：生产环境运维
Go语言监控告警生产环境运维1. 引言生产环境的稳定运行离不开完善的监控告警体系。监控不仅是发现问题的眼睛更是预防问题发生的预警系统。本文将深入讲解Go语言微服务中的监控指标设计、Grafana看板配置、日志聚合、告警规则和on-call流程。2. Prometheus指标设计2.1 Prometheus客户端使用package main import ( net/http time github.com/gin-gonic/gin github.com/prometheus/client_golang/prometheus github.com/prometheus/client_golang/prometheus/promhttp ) var ( // HTTP请求计数器 httpRequestsTotal prometheus.NewCounterVec( prometheus.CounterOpts{ Name: http_requests_total, Help: Total number of HTTP requests, }, []string{method, endpoint, status}, ) // HTTP请求延迟直方图 httpRequestDuration prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: http_request_duration_seconds, Help: HTTP request duration in seconds, Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}, }, []string{method, endpoint}, ) // HTTP请求正在处理的 Gauge httpRequestsInProgress prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: http_requests_in_progress, Help: Number of HTTP requests currently being processed, }, []string{method}, ) // 业务指标订单数量 ordersTotal prometheus.NewCounter( prometheus.CounterOpts{ Name: orders_total, Help: Total number of orders, }, ) // 业务指标订单金额 orderAmountTotal prometheus.NewCounterVec( prometheus.CounterOpts{ Name: order_amount_total, Help: Total amount of orders, }, []string{status}, ) // 数据库连接池指标 dbConnections prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: db_connections, Help: Number of database connections, }, []string{state}, ) // Redis连接池指标 redisConnections prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: redis_connections, Help: Number of Redis connections, }, []string{state}, ) ) func init() { // 注册所有指标 prometheus.MustRegister( httpRequestsTotal, httpRequestDuration, httpRequestsInProgress, ordersTotal, orderAmountTotal, dbConnections, redisConnections, ) } func main() { r : gin.Default() // Prometheus指标端点 r.GET(/metrics, gin.WrapH(promhttp.Handler())) // 使用中间件收集指标 r.Use(metricsMiddleware()) // 业务路由 r.GET(/api/v1/users, listUsers) r.POST(/api/v1/orders, createOrder) _ r.Run(:8080) } func metricsMiddleware() gin.HandlerFunc { return func(c *gin.Context) { if c.Request.URL.Path /metrics { c.Next() return } // 记录正在处理的请求 httpRequestsInProgress.WithLabelValues(c.Request.Method).Inc() defer httpRequestsInProgress.WithLabelValues(c.Request.Method).Dec() start : time.Now() // 处理请求 c.Next() // 记录请求指标 duration : time.Since(start).Seconds() status : fmt.Sprintf(%d, c.Writer.Status()) endpoint : c.FullPath() if endpoint { endpoint unknown } httpRequestsTotal.WithLabelValues( c.Request.Method, endpoint, status, ).Inc() httpRequestDuration.WithLabelValues( c.Request.Method, endpoint, ).Observe(duration) } }2.2 自定义业务指标package metrics import ( github.com/prometheus/client_golang/prometheus ) // 业务指标收集器 type BusinessMetrics struct { userRegistrations prometheus.Counter userLogins prometheus.Counter apiCallsTotal *prometheus.CounterVec apiLatency *prometheus.HistogramVec cacheHitRatio prometheus.Gauge queueSize prometheus.Gauge } func NewBusinessMetrics() *BusinessMetrics { m : BusinessMetrics{ userRegistrations: prometheus.NewCounter(prometheus.CounterOpts{ Name: user_registrations_total, Help: Total number of user registrations, }), userLogins: prometheus.NewCounter(prometheus.CounterOpts{ Name: user_logins_total, Help: Total number of user logins, }), apiCallsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: api_calls_total, Help: Total number of API calls, }, []string{service, method, status}), apiLatency: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: api_latency_seconds, Help: API latency in seconds, Buckets: prometheus.DefBuckets, }, []string{service, method}), cacheHitRatio: prometheus.NewGauge(prometheus.GaugeOpts{ Name: cache_hit_ratio, Help: Cache hit ratio, }), queueSize: prometheus.NewGauge(prometheus.GaugeOpts{ Name: queue_size, Help: Current queue size, }), } prometheus.MustRegister( m.userRegistrations, m.userLogins, m.apiCallsTotal, m.apiLatency, m.cacheHitRatio, m.queueSize, ) return m } // 记录用户注册 func (m *BusinessMetrics) RecordUserRegistration() { m.userRegistrations.Inc() } // 记录用户登录 func (m *BusinessMetrics) RecordUserLogin() { m.userLogins.Inc() } // 记录API调用 func (m *BusinessMetrics) RecordAPICall(service, method, status string) { m.apiCallsTotal.WithLabelValues(service, method, status).Inc() } // 记录API延迟 func (m *BusinessMetrics) RecordAPILatency(service, method string, duration float64) { m.apiLatency.WithLabelValues(service, method).Observe(duration) } // 设置缓存命中率 func (m *BusinessMetrics) SetCacheHitRatio(ratio float64) { m.cacheHitRatio.Set(ratio) } // 设置队列大小 func (m *BusinessMetrics) SetQueueSize(size float64) { m.queueSize.Set(size) }3. Grafana看板配置3.1 Grafana Dashboard JSON{ dashboard: { title: Go Microservice Dashboard, uid: go-microservice, version: 1, panels: [ { id: 1, title: Request Rate (QPS), type: graph, gridPos: {x: 0, y: 0, w: 12, h: 8}, targets: [ { expr: rate(http_requests_total[5m]), legendFormat: {{method}} {{endpoint}} {{status}} } ] }, { id: 2, title: Request Latency (p50, p95, p99), type: graph, gridPos: {x: 12, y: 0, w: 12, h: 8}, targets: [ { expr: histogram_quantile(0.50, rate(http_request_duration_seconds_bucket[5m])), legendFormat: p50 }, { expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])), legendFormat: p95 }, { expr: histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m])), legendFormat: p99 } ] }, { id: 3, title: Error Rate, type: graph, gridPos: {x: 0, y: 8, w: 12, h: 8}, targets: [ { expr: rate(http_requests_total{status~\5..\}[5m]), legendFormat: 5xx Error Rate } ] }, { id: 4, title: CPU Memory Usage, type: graph, gridPos: {x: 12, y: 8, w: 12, h: 8}, targets: [ { expr: process_cpu_seconds_total, legendFormat: CPU Seconds }, { expr: process_resident_memory_bytes, legendFormat: Memory Bytes } ] }, { id: 5, title: Goroutine Count, type: graph, gridPos: {x: 0, y: 16, w: 12, h: 8}, targets: [ { expr: go_goroutines, legendFormat: Goroutines } ] }, { id: 6, title: GC Stats, type: graph, gridPos: {x: 12, y: 16, w: 12, h: 8}, targets: [ { expr: rate(go_gc_duration_seconds_sum[5m]), legendFormat: GC Duration }, { expr: go_memstats_heap_alloc_bytes, legendFormat: Heap Allocated } ] } ] } }3.2 Grafana告警规则# grafana-alerts.yaml apiVersion: 1 groups: - name: go-microservice-alerts interval: 30s rules: # 高错误率告警 - alert: HighErrorRate expr: rate(http_requests_total{status~5..}[5m]) 0.05 for: 2m labels: severity: critical annotations: summary: High error rate detected description: Error rate is {{ $value | humanizePercentage }} (threshold: 5%) # 服务延迟过高告警 - alert: HighLatency expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) 1 for: 5m labels: severity: warning annotations: summary: High latency detected description: p95 latency is {{ $value | humanizeDuration }} (threshold: 1s) # 内存使用过高告警 - alert: HighMemoryUsage expr: process_resident_memory_bytes / 1024 / 1024 / 1024 2 for: 5m labels: severity: warning annotations: summary: High memory usage description: Memory usage is {{ $value | humanize }}GB (threshold: 2GB) # Goroutine泄漏告警 - alert: GoroutineLeak expr: go_goroutines 10000 for: 10m labels: severity: warning annotations: summary: Potential goroutine leak description: Goroutine count is {{ $value }} (threshold: 10000) # 服务不可用告警 - alert: ServiceDown expr: up{jobgo-microservice} 0 for: 1m labels: severity: critical annotations: summary: Service is down description: Service {{ $labels.instance }} is not responding4. 日志聚合方案4.1 结构化日志输出package logging import ( os time go.uber.org/zap go.uber.org/zap/zapcore ) // Logger 结构化日志器 var Logger *zap.Logger func InitLogger(env string) error { var config zap.Config if env production { config zap.NewProductionConfig() config.EncoderConfig.TimeKey timestamp config.EncoderConfig.EncodeTime zapcore.ISO8601TimeEncoder } else { config zap.NewDevelopmentConfig() config.EncoderConfig.EncodeLevel zapcore.CapitalColorLevelEncoder } var err error Logger, err config.Build( zap.AddCallerSkip(1), zap.AddStacktrace(zapcore.ErrorLevel), ) return err } // 记录带上下文的日志 func LogHTTPRequest(method, path string, status int, latency time.Duration, ip string) { Logger.Info(http_request, zap.String(method, method), zap.String(path, path), zap.Int(status, status), zap.Duration(latency, latency), zap.String(client_ip, ip), zap.String(service, api-gateway), ) } func LogError(operation string, err error, details map[string]interface{}) { fields : []zap.Field{ zap.String(operation, operation), zap.Error(err), } for k, v : range details { fields append(fields, zap.Any(k, v)) } Logger.Error(operation_failed, fields...) }4.2 日志收集配置# filebeat.yml filebeat.inputs: - type: container paths: - /var/log/containers/*.log processors: - add_kubernetes_metadata: host: ${NODE_NAME} matchers: - logs_path: logs_path: /var/log/containers/ - type: log paths: - /var/log/app/*.log fields: service: myapp environment: production fields_under_root: true output.logstash: hosts: [logstash:5044] processors: - add_host_metadata: when.not.contains.tags: forwarded - add_cloud_metadata: ~ - add_docker_metadata: ~4.3 Loki日志查询# 查询最近5分钟内的错误日志 {jobmyapp} | error | json | levelerror # 查询特定用户的请求日志 {jobmyapp} | user_id123 | json # 统计每分钟错误数量 sum by (level) (count_over_time({jobmyapp} | error [1m])) # 查询延迟最高的请求 topk(10, sum by (path) (rate(http_request_duration_seconds_sum[5m])) / sum by (path) (rate(http_request_duration_seconds_count[5m]))) # 关联指标和日志 sum by (path) (rate(http_requests_total{status500}[5m])) * 10005. 链路追踪5.1 OpenTelemetry集成package tracing import ( context fmt go.opentelemetry.io/otel go.opentelemetry.io/otel/attribute go.opentelemetry.io/otel/exporters/jaeger go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc go.opentelemetry.io/otel/propagation go.opentelemetry.io/otel/sdk/resource go.opentelemetry.io/otel/sdk/trace semconv go.opentelemetry.io/otel/semconv/v1.21.0 ) func InitTracer(serviceName, endpoint string) (func(context.Context) error, error) { // 创建Jaeger exporter exporter, err : jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(endpoint))) if err ! nil { return nil, err } // 或者使用OTLP exporter // exporter, err : otlptracegrpc.New(context.Background(), // otlptracegrpc.WithEndpoint(endpoint), // otlptracegrpc.WithInsecure(), // ) // 创建资源信息 res, err : resource.Merge( resource.Default(), resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceName(serviceName), semconv.ServiceVersion(1.0.0), attribute.String(environment, production), ), ) // 创建trace provider tp : trace.NewTracerProvider( trace.WithBatcher(exporter), trace.WithResource(res), trace.WithSampler(trace.ParentBased(trace.AlwaysSample())), ) otel.SetTracerProvider(tp) otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( propagation.TraceContext{}, propagation.Baggage{}, )) return tp.Shutdown, nil } // 追踪函数 func TraceFunc(ctx context.Context, tracer otel.Tracer, name string, fn func(context.Context) error) error { ctx, span : tracer.Start(ctx, name) defer span.End() if err : fn(ctx); err ! nil { span.RecordError(err) return err } return nil } // Gin中间件链路追踪 func TracingMiddleware(serviceName string) gin.HandlerFunc { tracer : otel.Tracer(serviceName) return func(c *gin.Context) { ctx : c.Request.Context() // 提取传播上下文 ctx otel.GetTextMapPropagator().Extract(ctx, propagation.HeaderCarrier(c.Request.Header)) ctx, span : tracer.Start(ctx, c.Request.Method c.FullPath()) defer span.End() span.SetAttributes( attribute.String(http.method, c.Request.Method), attribute.String(http.url, c.Request.URL.String()), attribute.String(http.host, c.Request.Host), attribute.String(http.user_agent, c.Request.UserAgent()), ) c.Request c.Request.WithContext(ctx) c.Next() span.SetAttributes(attribute.Int(http.status_code, c.Writer.Status())) } }6. 告警规则配置6.1 Prometheus告警规则# prometheus-alerts.yml groups: - name: microservice-alerts rules: # 服务可用性 - alert: InstanceDown expr: up{jobmyapp} 0 for: 1m labels: severity: critical annotations: summary: Instance {{ $labels.instance }} is down description: {{ $labels.job }} has been down for more than 1 minute # HTTP错误率 - alert: HighHTTPErrorRate expr: | sum(rate(http_requests_total{status~5..}[5m])) by (service) / sum(rate(http_requests_total[5m])) by (service) 0.05 for: 5m labels: severity: warning annotations: summary: High HTTP error rate in {{ $labels.service }} description: Error rate is {{ $value | humanizePercentage }} # 延迟SLO - alert: LatencySLOBreach expr: | histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[10m])) by (le, service)) 2 for: 10m labels: severity: warning annotations: summary: Latency SLO breach in {{ $labels.service }} description: p99 latency is {{ $value }}s, exceeds 2s threshold # 资源使用 - alert: HighCPUUsage expr: rate(process_cpu_seconds_total[5m]) * 100 80 for: 10m labels: severity: warning annotations: summary: High CPU usage description: CPU usage is {{ $value }}% - alert: HighMemoryUsage expr: process_resident_memory_bytes / 1024 / 1024 / 1024 3 for: 5m labels: severity: warning annotations: summary: High memory usage description: Memory usage is {{ $value | humanize }}GB # 数据库连接池 - alert: DatabaseConnectionPoolExhausted expr: | db_connections{statein_use} / db_connections{stateidle} 0.8 for: 5m labels: severity: critical annotations: summary: Database connection pool nearly exhausted # 业务指标异常 - alert: OrderAmountDropped expr: | sum(rate(order_amount_total[5m])) 100 for: 30m labels: severity: warning annotations: summary: Order amount dropped significantly description: Order rate is {{ $value }} per second6.2 AlertManager配置# alertmanager.yml global: resolve_timeout: 5m smtp_smarthost: smtp.example.com:587 smtp_from: alertmanagerexample.com route: group_by: [alertname, severity] group_wait: 30s group_interval: 5m repeat_interval: 4h receiver: default-receiver routes: - match: severity: critical receiver: critical-receiver group_wait: 10s - match: severity: warning receiver: warning-receiver receivers: - name: default-receiver email_configs: - to: oncallexample.com headers: subject: Alert: {{ .GroupLabels.alertname }} - name: critical-receiver pagerduty_configs: - service_key: YOUR_PAGERDUTY_KEY severity: critical slack_configs: - channel: #critical-alerts api_url: YOUR_SLACK_WEBHOOK - name: warning-receiver slack_configs: - channel: #warnings api_url: YOUR_SLACK_WEBHOOK7. On-Call流程7.1 On-Call管理package oncall import ( fmt time ) // OnCallSchedule 值班表 type OnCallSchedule struct { CurrentOnCall *OnCallEngineer NextOnCall *OnCallEngineer EscalationList []OnCallEngineer } // OnCallEngineer 值班工程师 type OnCallEngineer struct { ID string Name string Email string Phone string StartTime time.Time EndTime time.Time } // RotationManager 值班轮换管理器 type RotationManager struct { engineers []OnCallEngineer currentIdx int } func NewRotationManager(engineers []OnCallEngineer) *RotationManager { return RotationManager{ engineers: engineers, currentIdx: 0, } } func (m *RotationManager) GetCurrentOnCall() *OnCallEngineer { return m.engineers[m.currentIdx] } func (m *RotationManager) GetNextOnCall() *OnCallEngineer { nextIdx : (m.currentIdx 1) % len(m.engineers) return m.engineers[nextIdx] } func (m *RotationManager) Rotate() { m.currentIdx (m.currentIdx 1) % len(m.engineers) } // Incident 事件 type Incident struct { ID string Title string Description string Severity string // critical, high, medium, low Status string // open, acknowledged, resolved CreatedAt time.Time ResolvedAt *time.Time Assignee *OnCallEngineer } // IncidentManager 事件管理器 type IncidentManager struct { incidents map[string]*Incident } func NewIncidentManager() *IncidentManager { return IncidentManager{ incidents: make(map[string]*Incident), } } func (m *IncidentManager) CreateIncident(title, desc, severity string, assignee *OnCallEngineer) *Incident { inc : Incident{ ID: fmt.Sprintf(INC-%d, time.Now().Unix()), Title: title, Description: desc, Severity: severity, Status: open, CreatedAt: time.Now(), Assignee: assignee, } m.incidents[inc.ID] inc return inc } func (m *IncidentManager) Acknowledge(id string) error { inc, ok : m.incidents[id] if !ok { return fmt.Errorf(incident not found) } inc.Status acknowledged return nil } func (m *IncidentManager) Resolve(id string) error { inc, ok : m.incidents[id] if !ok { return fmt.Errorf(incident not found) } inc.Status resolved now : time.Now() inc.ResolvedAt now return nil }7.2 值班脚本#!/bin/bash # check_oncall.sh - 检查当前值班人并发送通知 ONCALL_APIhttps://oncall-api.example.com SLACK_WEBHOOKhttps://hooks.slack.com/services/XXX # 获取当前值班人 CURRENT_ONCALL$(curl -s ${ONCALL_API}/current) # 检查是否有未解决的告警 ALERT_COUNT$(curl -s ${ONCALL_API}/alerts?statusopen | jq length) if [ $ALERT_COUNT -gt 0 ]; then MESSAGE⚠️ *On-Call Alert*: You have ${ALERT_COUNT} unresolved alerts. Please check ${ONCALL_API} # 发送Slack通知 curl -X POST -H Content-type: application/json \ --data {\text\: \${MESSAGE}\} \ ${SLACK_WEBHOOK} fi8. 总结完善的监控告警体系是生产环境运维的基石Prometheus指标设计从基础设施、业务、性能三个维度设计指标覆盖系统运行全貌Grafana看板通过可视化看板快速了解系统状态设置合理的图表布局日志聚合使用结构化日志支持多维度查询便于问题追溯链路追踪通过OpenTelemetry实现分布式追踪快速定位跨服务问题告警规则设置合理的阈值和告警级别避免告警疲劳On-Call流程建立完善的值班和事件升级机制确保问题及时处理监控告警需要持续优化根据业务发展和问题经验不断完善告警规则和处理流程。
