前言上篇文章我们实现了Grafana的可视化但监控不只是看图表——更重要的是出问题时能及时通知到人。Alertmanager是Prometheus生态的告警组件负责接收告警、分组、抑制、静默和通知。今天我们从零实现一个完整的告警系统· 告警接收Webhook· 告警分组Group· 告警抑制Inhibit· 告警静默Silence· 告警通知Email/钉钉/微信· 告警路由Route· 告警升级Escalation---一、告警系统核心原理1. 架构图┌─────────────────────────────────────────────────────────────┐│ Prometheus告警规则 ││ 触发告警 │└─────────────────────────────────────────────────────────────┘│▼┌─────────────────────────────────────────────────────────────┐│ Alertmanager ││ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ││ │ 告警接收 │→│ 告警分组 │→│ 告警路由 │ ││ │ (Webhook) │ │ (Group) │ │ (Route) │ ││ └─────────────┘ └─────────────┘ └─────────────┘ ││ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ││ │ 告警抑制 │ │ 告警静默 │ │ 告警通知 │ ││ │ (Inhibit) │ │ (Silence) │ │ (Notify) │ ││ └─────────────┘ └─────────────┘ └─────────────┘ │└─────────────────────────────────────────────────────────────┘│▼┌─────────────────────────────────────────────────────────────┐│ 通知渠道 ││ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ││ │ 邮件 │ │ 钉钉 │ │ 微信 │ ││ └─────────────┘ └─────────────┘ └─────────────┘ │└─────────────────────────────────────────────────────────────┘2. 核心概念概念 说明Group 按标签分组避免告警风暴Inhibit 高优先级告警抑制低优先级Silence 手动静默告警Route 告警路由到不同接收者Receiver 通知接收者邮件/钉钉等---二、完整代码实现1. 基础数据结构c#include stdio.h#include stdlib.h#include string.h#include unistd.h#include pthread.h#include time.h#include errno.h#define MAX_LABELS 16#define MAX_LABEL_KEY 64#define MAX_LABEL_VALUE 128#define MAX_ANNOTATIONS 16#define MAX_RECEIVERS 16#define MAX_ROUTES 16#define MAX_SILENCES 100// 标签typedef struct label {char key[MAX_LABEL_KEY];char value[MAX_LABEL_VALUE];} label_t;// 告警typedef struct alert {char name[128];label_t labels[MAX_LABELS];int label_count;label_t annotations[MAX_ANNOTATIONS];int annotation_count;time_t starts_at;time_t ends_at;char status[16]; // firing, resolvedchar generator_url[256];int repeated;struct alert *next;} alert_t;// 告警组typedef struct alert_group {char group_key[256];alert_t *alerts;int alert_count;time_t first_alert_at;time_t last_alert_at;int active;struct alert_group *next;} alert_group_t;// 告警抑制规则typedef struct inhibit_rule {char source_matchers[256];char target_matchers[256];char equal[64];struct inhibit_rule *next;} inhibit_rule_t;// 静默typedef struct silence {char id[64];label_t matchers[MAX_LABELS];int matcher_count;time_t starts_at;time_t ends_at;char created_by[64];char comment[256];int active;struct silence *next;} silence_t;// 接收者配置typedef struct receiver_config {char name[64];char type[16]; // email, dingtalk, wechat, webhookchar address[256];char template[256];} receiver_config_t;// 路由typedef struct route_config {char match_labels[256];char match_regex[256];char receiver[64];int continue_route;int group_wait;int group_interval;int repeat_interval;struct route_config *next;} route_config_t;// Alertmanagertypedef struct alertmanager {alert_group_t *groups;inhibit_rule_t *inhibit_rules;silence_t *silences;receiver_config_t receivers[MAX_RECEIVERS];int receiver_count;route_config_t *routes;pthread_mutex_t mutex;int running;int port;pthread_t notify_thread;} alertmanager_t;2. Alertmanager创建c// 创建Alertmanageralertmanager_t *am_create(int port) {alertmanager_t *am malloc(sizeof(alertmanager_t));memset(am, 0, sizeof(alertmanager_t));am-port port;am-running 1;am-receiver_count 0;pthread_mutex_init(am-mutex, NULL);printf([Alertmanager] 启动端口: %d\n, port);return am;}// 添加接收者void am_add_receiver(alertmanager_t *am, const char *name,const char *type, const char *address) {if (am-receiver_count MAX_RECEIVERS) return;receiver_config_t *r am-receivers[am-receiver_count];strcpy(r-name, name);strcpy(r-type, type);strcpy(r-address, address);printf([Alertmanager] 添加接收者: %s (%s)\n, name, type);}// 添加路由void am_add_route(alertmanager_t *am, const char *match_labels,const char *receiver, int group_wait,int group_interval, int repeat_interval) {pthread_mutex_lock(am-mutex);route_config_t *route malloc(sizeof(route_config_t));strcpy(route-match_labels, match_labels);strcpy(route-receiver, receiver);route-group_wait group_wait;route-group_interval group_interval;route-repeat_interval repeat_interval;route-continue_route 0;route-next am-routes;am-routes route;pthread_mutex_unlock(am-mutex);printf([Alertmanager] 添加路由: %s → %s\n, match_labels, receiver);}// 添加抑制规则void am_add_inhibit_rule(alertmanager_t *am, const char *source,const char *target, const char *equal) {pthread_mutex_lock(am-mutex);inhibit_rule_t *rule malloc(sizeof(inhibit_rule_t));strcpy(rule-source_matchers, source);strcpy(rule-target_matchers, target);strcpy(rule-equal, equal);rule-next am-inhibit_rules;am-inhibit_rules rule;pthread_mutex_unlock(am-mutex);}3. 告警接收与分组c// 发送通知void am_send_notification(alertmanager_t *am, alert_group_t *group,receiver_config_t *receiver) {char message[4096];alert_t *alert group-alerts;if (strcmp(receiver-type, email) 0) {snprintf(message, sizeof(message),Subject: [Alert] %d alerts firing\n\n, group-alert_count);while (alert) {strcat(message, alert-name);strcat(message, \n);alert alert-next;}printf([Email] 发送到 %s: %s\n, receiver-address, message);} else if (strcmp(receiver-type, dingtalk) 0) {snprintf(message, sizeof(message),{\msgtype\:\text\,\text\:{\content\:\[Alert] %d alerts firing\}},group-alert_count);printf([钉钉] 发送到 %s\n, receiver-address);} else if (strcmp(receiver-type, wechat) 0) {printf([微信] 发送到 %s\n, receiver-address);} else {printf([Webhook] 发送到 %s\n, receiver-address);}}// 查找匹配的路由route_config_t *am_find_route(alertmanager_t *am, alert_t *alert) {route_config_t *route am-routes;while (route) {if (strlen(route-match_labels) 0) {// 简化匹配检查是否包含char label_str[256] {0};for (int i 0; i alert-label_count; i) {if (i 0) strcat(label_str, ,);strcat(label_str, alert-labels[i].key);strcat(label_str, );strcat(label_str, alert-labels[i].value);}if (strstr(label_str, route-match_labels) ! NULL) {return route;}}route route-next;}return NULL;}// 接收告警void am_receive_alert(alertmanager_t *am, alert_t *alert) {pthread_mutex_lock(am-mutex);// 1. 检查静默silence_t *silence am-silences;time_t now time(NULL);while (silence) {if (silence-active now silence-starts_at now silence-ends_at) {int matched 1;for (int i 0; i silence-matcher_count; i) {int found 0;for (int j 0; j alert-label_count; j) {if (strcmp(alert-labels[j].key, silence-matchers[i].key) 0 strcmp(alert-labels[j].value, silence-matchers[i].value) 0) {found 1;break;}}if (!found) {matched 0;break;}}if (matched) {printf([Silence] 告警被静默: %s\n, alert-name);pthread_mutex_unlock(am-mutex);return;}}silence silence-next;}// 2. 查找或创建告警组char group_key[256];snprintf(group_key, sizeof(group_key), %s, alert-name);// 按标签分组for (int i 0; i alert-label_count; i) {if (strcmp(alert-labels[i].key, severity) 0 ||strcmp(alert-labels[i].key, service) 0) {char tmp[64];snprintf(tmp, sizeof(tmp), |%s%s,alert-labels[i].key, alert-labels[i].value);strcat(group_key, tmp);}}alert_group_t *group am-groups;while (group) {if (strcmp(group-group_key, group_key) 0) {break;}group group-next;}if (!group) {group malloc(sizeof(alert_group_t));strcpy(group-group_key, group_key);group-alerts NULL;group-alert_count 0;group-first_alert_at now;group-last_alert_at now;group-active 1;group-next am-groups;am-groups group;}// 添加告警到组alert-next group-alerts;group-alerts alert;group-alert_count;group-last_alert_at now;// 3. 路由查找route_config_t *route am_find_route(am, alert);if (route) {receiver_config_t *receiver NULL;for (int i 0; i am-receiver_count; i) {if (strcmp(am-receivers[i].name, route-receiver) 0) {receiver am-receivers[i];break;}}if (receiver) {am_send_notification(am, group, receiver);}}pthread_mutex_unlock(am-mutex);}4. 静默管理c// 创建静默void am_create_silence(alertmanager_t *am, const char *matchers,int duration_seconds, const char *created_by,const char *comment) {pthread_mutex_lock(am-mutex);silence_t *s malloc(sizeof(silence_t));snprintf(s-id, sizeof(s-id), silence-%d, (int)time(NULL));// 解析匹配器: severitycritical,serviceuser-servicechar *p (char*)matchers;s-matcher_count 0;char *token strtok(p, ,);while (token s-matcher_count MAX_LABELS) {char *eq strchr(token, );if (eq) {int key_len eq - token;int val_len strlen(token) - key_len - 1;strncpy(s-matchers[s-matcher_count].key, token, key_len);s-matchers[s-matcher_count].key[key_len] \0;strcpy(s-matchers[s-matcher_count].value, eq 1);s-matcher_count;}token strtok(NULL, ,);}s-starts_at time(NULL);s-ends_at s-starts_at duration_seconds;strcpy(s-created_by, created_by);strcpy(s-comment, comment);s-active 1;s-next am-silences;am-silences s;pthread_mutex_unlock(am-mutex);printf([Silence] 创建静默: %s\n, s-id);}// 结束静默void am_expire_silence(alertmanager_t *am, const char *id) {pthread_mutex_lock(am-mutex);silence_t *s am-silences;while (s) {if (strcmp(s-id, id) 0) {s-active 0;printf([Silence] 结束静默: %s\n, id);break;}s s-next;}pthread_mutex_unlock(am-mutex);}5. 告警升级c// 告警升级配置typedef struct escalation {char severity[16];int after_minutes;char receiver[64];char message[256];struct escalation *next;} escalation_t;// 检查并升级告警void am_check_escalation(alertmanager_t *am, alert_group_t *group) {time_t now time(NULL);alert_t *alert group-alerts;if (!alert) return;// 检查告警持续时间time_t duration now - group-first_alert_at;// 查找最高严重性的告警char severity[16] info;while (alert) {for (int i 0; i alert-label_count; i) {if (strcmp(alert-labels[i].key, severity) 0) {if (strcmp(alert-labels[i].value, critical) 0) {strcpy(severity, critical);} else if (strcmp(alert-labels[i].value, warning) 0 strcmp(severity, critical) ! 0) {strcpy(severity, warning);}break;}}alert alert-next;}// 根据严重性和持续时间决定升级if (duration 300 strcmp(severity, critical) 0) {printf([Escalation] 告警持续 %ld 秒升级到高优先级通知\n, duration);// 发送升级通知for (int i 0; i am-receiver_count; i) {if (strcmp(am-receivers[i].name, oncall) 0) {am_send_notification(am, group, am-receivers[i]);break;}}}}6. 测试代码cvoid test_alertmanager() {printf( Alertmanager告警系统测试 \n\n);alertmanager_t *am am_create(9093);// 添加接收者am_add_receiver(am, ops, email, opsexample.com);am_add_receiver(am, dev, dingtalk, https://oapi.dingtalk.com/robot/send);am_add_receiver(am, oncall, wechat, oncall);// 添加路由am_add_route(am, severitycritical, ops, 10, 300, 3600);am_add_route(am, serviceuser-service, dev, 30, 600, 7200);// 添加抑制规则am_add_inhibit_rule(am, severitycritical, severitywarning, service);// 创建静默am_create_silence(am, serviceuser-service,severitywarning, 3600,admin, 维护窗口);// 模拟告警printf(发送告警...\n);// 告警1: Criticalalert_t *alert1 malloc(sizeof(alert_t));strcpy(alert1-name, HighCPUUsage);alert1-label_count 2;strcpy(alert1-labels[0].key, severity);strcpy(alert1-labels[0].value, critical);strcpy(alert1-labels[1].key, service);strcpy(alert1-labels[1].value, user-service);alert1-starts_at time(NULL);strcpy(alert1-status, firing);am_receive_alert(am, alert1);// 告警2: Warning (将被抑制)alert_t *alert2 malloc(sizeof(alert_t));strcpy(alert2-name, HighMemoryUsage);alert2-label_count 2;strcpy(alert2-labels[0].key, severity);strcpy(alert2-labels[0].value, warning);strcpy(alert2-labels[1].key, service);strcpy(alert2-labels[1].value, user-service);alert2-starts_at time(NULL);strcpy(alert2-status, firing);am_receive_alert(am, alert2);// 告警3: 不同服务alert_t *alert3 malloc(sizeof(alert_t));strcpy(alert3-name, DatabaseSlow);alert3-label_count 2;strcpy(alert3-labels[0].key, severity);strcpy(alert3-labels[0].value, warning);strcpy(alert3-labels[1].key, service);strcpy(alert3-labels[1].value, order-service);alert3-starts_at time(NULL);strcpy(alert3-status, firing);am_receive_alert(am, alert3);// 显示状态printf(\n Alertmanager状态 \n);printf(告警组: );alert_group_t *g am-groups;int count 0;while (g) { count; g g-next; }printf(%d\n, count);printf(静默: );silence_t *s am-silences;int sc 0;while (s) { if (s-active) sc; s s-next; }printf(%d\n, sc);free(am);}int main() {srand(time(NULL));test_alertmanager();return 0;}---三、编译和运行bashgcc -o alertmanager alertmanager.c -lpthread./alertmanager---四、Alertmanager vs 本实现特性 本实现 Alertmanager告警分组 ✅ ✅告警路由 ✅ ✅告警抑制 ✅ ✅告警静默 ✅ ✅告警通知 ✅ 基础 ✅ 丰富告警升级 ✅ ✅告警聚合 ❌ ✅---五、总结通过这篇文章你学会了· Alertmanager的核心架构· 告警接收与分组· 告警路由与接收者· 告警抑制规则· 告警静默管理· 告警升级处理Alertmanager是云原生告警的核心组件。掌握它你就理解了告警系统的完整设计。下一篇预告《从零实现一个分布式服务网格Istio的核心设计》---评论区分享一下你用Alertmanager处理过什么告警场景
从零实现一个分布式告警：Alertmanager的核心设计
前言上篇文章我们实现了Grafana的可视化但监控不只是看图表——更重要的是出问题时能及时通知到人。Alertmanager是Prometheus生态的告警组件负责接收告警、分组、抑制、静默和通知。今天我们从零实现一个完整的告警系统· 告警接收Webhook· 告警分组Group· 告警抑制Inhibit· 告警静默Silence· 告警通知Email/钉钉/微信· 告警路由Route· 告警升级Escalation---一、告警系统核心原理1. 架构图┌─────────────────────────────────────────────────────────────┐│ Prometheus告警规则 ││ 触发告警 │└─────────────────────────────────────────────────────────────┘│▼┌─────────────────────────────────────────────────────────────┐│ Alertmanager ││ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ││ │ 告警接收 │→│ 告警分组 │→│ 告警路由 │ ││ │ (Webhook) │ │ (Group) │ │ (Route) │ ││ └─────────────┘ └─────────────┘ └─────────────┘ ││ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ││ │ 告警抑制 │ │ 告警静默 │ │ 告警通知 │ ││ │ (Inhibit) │ │ (Silence) │ │ (Notify) │ ││ └─────────────┘ └─────────────┘ └─────────────┘ │└─────────────────────────────────────────────────────────────┘│▼┌─────────────────────────────────────────────────────────────┐│ 通知渠道 ││ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ││ │ 邮件 │ │ 钉钉 │ │ 微信 │ ││ └─────────────┘ └─────────────┘ └─────────────┘ │└─────────────────────────────────────────────────────────────┘2. 核心概念概念 说明Group 按标签分组避免告警风暴Inhibit 高优先级告警抑制低优先级Silence 手动静默告警Route 告警路由到不同接收者Receiver 通知接收者邮件/钉钉等---二、完整代码实现1. 基础数据结构c#include stdio.h#include stdlib.h#include string.h#include unistd.h#include pthread.h#include time.h#include errno.h#define MAX_LABELS 16#define MAX_LABEL_KEY 64#define MAX_LABEL_VALUE 128#define MAX_ANNOTATIONS 16#define MAX_RECEIVERS 16#define MAX_ROUTES 16#define MAX_SILENCES 100// 标签typedef struct label {char key[MAX_LABEL_KEY];char value[MAX_LABEL_VALUE];} label_t;// 告警typedef struct alert {char name[128];label_t labels[MAX_LABELS];int label_count;label_t annotations[MAX_ANNOTATIONS];int annotation_count;time_t starts_at;time_t ends_at;char status[16]; // firing, resolvedchar generator_url[256];int repeated;struct alert *next;} alert_t;// 告警组typedef struct alert_group {char group_key[256];alert_t *alerts;int alert_count;time_t first_alert_at;time_t last_alert_at;int active;struct alert_group *next;} alert_group_t;// 告警抑制规则typedef struct inhibit_rule {char source_matchers[256];char target_matchers[256];char equal[64];struct inhibit_rule *next;} inhibit_rule_t;// 静默typedef struct silence {char id[64];label_t matchers[MAX_LABELS];int matcher_count;time_t starts_at;time_t ends_at;char created_by[64];char comment[256];int active;struct silence *next;} silence_t;// 接收者配置typedef struct receiver_config {char name[64];char type[16]; // email, dingtalk, wechat, webhookchar address[256];char template[256];} receiver_config_t;// 路由typedef struct route_config {char match_labels[256];char match_regex[256];char receiver[64];int continue_route;int group_wait;int group_interval;int repeat_interval;struct route_config *next;} route_config_t;// Alertmanagertypedef struct alertmanager {alert_group_t *groups;inhibit_rule_t *inhibit_rules;silence_t *silences;receiver_config_t receivers[MAX_RECEIVERS];int receiver_count;route_config_t *routes;pthread_mutex_t mutex;int running;int port;pthread_t notify_thread;} alertmanager_t;2. Alertmanager创建c// 创建Alertmanageralertmanager_t *am_create(int port) {alertmanager_t *am malloc(sizeof(alertmanager_t));memset(am, 0, sizeof(alertmanager_t));am-port port;am-running 1;am-receiver_count 0;pthread_mutex_init(am-mutex, NULL);printf([Alertmanager] 启动端口: %d\n, port);return am;}// 添加接收者void am_add_receiver(alertmanager_t *am, const char *name,const char *type, const char *address) {if (am-receiver_count MAX_RECEIVERS) return;receiver_config_t *r am-receivers[am-receiver_count];strcpy(r-name, name);strcpy(r-type, type);strcpy(r-address, address);printf([Alertmanager] 添加接收者: %s (%s)\n, name, type);}// 添加路由void am_add_route(alertmanager_t *am, const char *match_labels,const char *receiver, int group_wait,int group_interval, int repeat_interval) {pthread_mutex_lock(am-mutex);route_config_t *route malloc(sizeof(route_config_t));strcpy(route-match_labels, match_labels);strcpy(route-receiver, receiver);route-group_wait group_wait;route-group_interval group_interval;route-repeat_interval repeat_interval;route-continue_route 0;route-next am-routes;am-routes route;pthread_mutex_unlock(am-mutex);printf([Alertmanager] 添加路由: %s → %s\n, match_labels, receiver);}// 添加抑制规则void am_add_inhibit_rule(alertmanager_t *am, const char *source,const char *target, const char *equal) {pthread_mutex_lock(am-mutex);inhibit_rule_t *rule malloc(sizeof(inhibit_rule_t));strcpy(rule-source_matchers, source);strcpy(rule-target_matchers, target);strcpy(rule-equal, equal);rule-next am-inhibit_rules;am-inhibit_rules rule;pthread_mutex_unlock(am-mutex);}3. 告警接收与分组c// 发送通知void am_send_notification(alertmanager_t *am, alert_group_t *group,receiver_config_t *receiver) {char message[4096];alert_t *alert group-alerts;if (strcmp(receiver-type, email) 0) {snprintf(message, sizeof(message),Subject: [Alert] %d alerts firing\n\n, group-alert_count);while (alert) {strcat(message, alert-name);strcat(message, \n);alert alert-next;}printf([Email] 发送到 %s: %s\n, receiver-address, message);} else if (strcmp(receiver-type, dingtalk) 0) {snprintf(message, sizeof(message),{\msgtype\:\text\,\text\:{\content\:\[Alert] %d alerts firing\}},group-alert_count);printf([钉钉] 发送到 %s\n, receiver-address);} else if (strcmp(receiver-type, wechat) 0) {printf([微信] 发送到 %s\n, receiver-address);} else {printf([Webhook] 发送到 %s\n, receiver-address);}}// 查找匹配的路由route_config_t *am_find_route(alertmanager_t *am, alert_t *alert) {route_config_t *route am-routes;while (route) {if (strlen(route-match_labels) 0) {// 简化匹配检查是否包含char label_str[256] {0};for (int i 0; i alert-label_count; i) {if (i 0) strcat(label_str, ,);strcat(label_str, alert-labels[i].key);strcat(label_str, );strcat(label_str, alert-labels[i].value);}if (strstr(label_str, route-match_labels) ! NULL) {return route;}}route route-next;}return NULL;}// 接收告警void am_receive_alert(alertmanager_t *am, alert_t *alert) {pthread_mutex_lock(am-mutex);// 1. 检查静默silence_t *silence am-silences;time_t now time(NULL);while (silence) {if (silence-active now silence-starts_at now silence-ends_at) {int matched 1;for (int i 0; i silence-matcher_count; i) {int found 0;for (int j 0; j alert-label_count; j) {if (strcmp(alert-labels[j].key, silence-matchers[i].key) 0 strcmp(alert-labels[j].value, silence-matchers[i].value) 0) {found 1;break;}}if (!found) {matched 0;break;}}if (matched) {printf([Silence] 告警被静默: %s\n, alert-name);pthread_mutex_unlock(am-mutex);return;}}silence silence-next;}// 2. 查找或创建告警组char group_key[256];snprintf(group_key, sizeof(group_key), %s, alert-name);// 按标签分组for (int i 0; i alert-label_count; i) {if (strcmp(alert-labels[i].key, severity) 0 ||strcmp(alert-labels[i].key, service) 0) {char tmp[64];snprintf(tmp, sizeof(tmp), |%s%s,alert-labels[i].key, alert-labels[i].value);strcat(group_key, tmp);}}alert_group_t *group am-groups;while (group) {if (strcmp(group-group_key, group_key) 0) {break;}group group-next;}if (!group) {group malloc(sizeof(alert_group_t));strcpy(group-group_key, group_key);group-alerts NULL;group-alert_count 0;group-first_alert_at now;group-last_alert_at now;group-active 1;group-next am-groups;am-groups group;}// 添加告警到组alert-next group-alerts;group-alerts alert;group-alert_count;group-last_alert_at now;// 3. 路由查找route_config_t *route am_find_route(am, alert);if (route) {receiver_config_t *receiver NULL;for (int i 0; i am-receiver_count; i) {if (strcmp(am-receivers[i].name, route-receiver) 0) {receiver am-receivers[i];break;}}if (receiver) {am_send_notification(am, group, receiver);}}pthread_mutex_unlock(am-mutex);}4. 静默管理c// 创建静默void am_create_silence(alertmanager_t *am, const char *matchers,int duration_seconds, const char *created_by,const char *comment) {pthread_mutex_lock(am-mutex);silence_t *s malloc(sizeof(silence_t));snprintf(s-id, sizeof(s-id), silence-%d, (int)time(NULL));// 解析匹配器: severitycritical,serviceuser-servicechar *p (char*)matchers;s-matcher_count 0;char *token strtok(p, ,);while (token s-matcher_count MAX_LABELS) {char *eq strchr(token, );if (eq) {int key_len eq - token;int val_len strlen(token) - key_len - 1;strncpy(s-matchers[s-matcher_count].key, token, key_len);s-matchers[s-matcher_count].key[key_len] \0;strcpy(s-matchers[s-matcher_count].value, eq 1);s-matcher_count;}token strtok(NULL, ,);}s-starts_at time(NULL);s-ends_at s-starts_at duration_seconds;strcpy(s-created_by, created_by);strcpy(s-comment, comment);s-active 1;s-next am-silences;am-silences s;pthread_mutex_unlock(am-mutex);printf([Silence] 创建静默: %s\n, s-id);}// 结束静默void am_expire_silence(alertmanager_t *am, const char *id) {pthread_mutex_lock(am-mutex);silence_t *s am-silences;while (s) {if (strcmp(s-id, id) 0) {s-active 0;printf([Silence] 结束静默: %s\n, id);break;}s s-next;}pthread_mutex_unlock(am-mutex);}5. 告警升级c// 告警升级配置typedef struct escalation {char severity[16];int after_minutes;char receiver[64];char message[256];struct escalation *next;} escalation_t;// 检查并升级告警void am_check_escalation(alertmanager_t *am, alert_group_t *group) {time_t now time(NULL);alert_t *alert group-alerts;if (!alert) return;// 检查告警持续时间time_t duration now - group-first_alert_at;// 查找最高严重性的告警char severity[16] info;while (alert) {for (int i 0; i alert-label_count; i) {if (strcmp(alert-labels[i].key, severity) 0) {if (strcmp(alert-labels[i].value, critical) 0) {strcpy(severity, critical);} else if (strcmp(alert-labels[i].value, warning) 0 strcmp(severity, critical) ! 0) {strcpy(severity, warning);}break;}}alert alert-next;}// 根据严重性和持续时间决定升级if (duration 300 strcmp(severity, critical) 0) {printf([Escalation] 告警持续 %ld 秒升级到高优先级通知\n, duration);// 发送升级通知for (int i 0; i am-receiver_count; i) {if (strcmp(am-receivers[i].name, oncall) 0) {am_send_notification(am, group, am-receivers[i]);break;}}}}6. 测试代码cvoid test_alertmanager() {printf( Alertmanager告警系统测试 \n\n);alertmanager_t *am am_create(9093);// 添加接收者am_add_receiver(am, ops, email, opsexample.com);am_add_receiver(am, dev, dingtalk, https://oapi.dingtalk.com/robot/send);am_add_receiver(am, oncall, wechat, oncall);// 添加路由am_add_route(am, severitycritical, ops, 10, 300, 3600);am_add_route(am, serviceuser-service, dev, 30, 600, 7200);// 添加抑制规则am_add_inhibit_rule(am, severitycritical, severitywarning, service);// 创建静默am_create_silence(am, serviceuser-service,severitywarning, 3600,admin, 维护窗口);// 模拟告警printf(发送告警...\n);// 告警1: Criticalalert_t *alert1 malloc(sizeof(alert_t));strcpy(alert1-name, HighCPUUsage);alert1-label_count 2;strcpy(alert1-labels[0].key, severity);strcpy(alert1-labels[0].value, critical);strcpy(alert1-labels[1].key, service);strcpy(alert1-labels[1].value, user-service);alert1-starts_at time(NULL);strcpy(alert1-status, firing);am_receive_alert(am, alert1);// 告警2: Warning (将被抑制)alert_t *alert2 malloc(sizeof(alert_t));strcpy(alert2-name, HighMemoryUsage);alert2-label_count 2;strcpy(alert2-labels[0].key, severity);strcpy(alert2-labels[0].value, warning);strcpy(alert2-labels[1].key, service);strcpy(alert2-labels[1].value, user-service);alert2-starts_at time(NULL);strcpy(alert2-status, firing);am_receive_alert(am, alert2);// 告警3: 不同服务alert_t *alert3 malloc(sizeof(alert_t));strcpy(alert3-name, DatabaseSlow);alert3-label_count 2;strcpy(alert3-labels[0].key, severity);strcpy(alert3-labels[0].value, warning);strcpy(alert3-labels[1].key, service);strcpy(alert3-labels[1].value, order-service);alert3-starts_at time(NULL);strcpy(alert3-status, firing);am_receive_alert(am, alert3);// 显示状态printf(\n Alertmanager状态 \n);printf(告警组: );alert_group_t *g am-groups;int count 0;while (g) { count; g g-next; }printf(%d\n, count);printf(静默: );silence_t *s am-silences;int sc 0;while (s) { if (s-active) sc; s s-next; }printf(%d\n, sc);free(am);}int main() {srand(time(NULL));test_alertmanager();return 0;}---三、编译和运行bashgcc -o alertmanager alertmanager.c -lpthread./alertmanager---四、Alertmanager vs 本实现特性 本实现 Alertmanager告警分组 ✅ ✅告警路由 ✅ ✅告警抑制 ✅ ✅告警静默 ✅ ✅告警通知 ✅ 基础 ✅ 丰富告警升级 ✅ ✅告警聚合 ❌ ✅---五、总结通过这篇文章你学会了· Alertmanager的核心架构· 告警接收与分组· 告警路由与接收者· 告警抑制规则· 告警静默管理· 告警升级处理Alertmanager是云原生告警的核心组件。掌握它你就理解了告警系统的完整设计。下一篇预告《从零实现一个分布式服务网格Istio的核心设计》---评论区分享一下你用Alertmanager处理过什么告警场景
