Go语言日志系统与Zap实战

Go语言日志系统与Zap实战 Go语言日志系统与Zap实战引言日志系统是任何应用程序的重要组成部分。本文将深入探讨Go语言中的日志系统设计并重点介绍高性能日志库Zap的使用方法和最佳实践。一、日志系统基础1.1 日志级别const ( DebugLevel iota // 调试信息详细的程序运行信息 InfoLevel // 一般信息记录程序正常运行状态 WarnLevel // 警告信息可能出现问题但不影响运行 ErrorLevel // 错误信息程序出现错误但仍可运行 DPanicLevel // 严重错误开发环境会panic PanicLevel // 恐慌级别会触发panic FatalLevel // 致命错误程序会立即退出 )1.2 日志格式// 文本格式 // 2024-01-15 10:30:45.123 INFO [main.go:123] User login: user_id123 // JSON格式 // {level:info,time:2024-01-15T10:30:45.123Z,caller:main.go:123,msg:User login,user_id:123}1.3 日志记录原则原则说明结构化使用结构化日志便于分析和查询级别分明根据重要性选择合适的日志级别上下文丰富记录必要的上下文信息性能优先避免频繁的日志操作影响性能安全合规避免记录敏感信息二、标准库log包2.1 基本使用import log func main() { // 基本日志 log.Println(Hello, World!) // 带前缀的日志 log.SetPrefix([APP] ) log.Println(Application started) // 设置输出到文件 file, err : os.OpenFile(app.log, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err ! nil { log.Fatal(Failed to open log file) } defer file.Close() log.SetOutput(file) // 格式化日志 log.Printf(User %s logged in at %s, john, time.Now()) }2.2 标准库的局限性限制说明无日志级别无法区分日志重要性无结构化仅支持文本格式性能一般不适合高并发场景功能有限缺乏丰富的配置选项三、Zap日志库3.1 安装与初始化import ( go.uber.org/zap ) func NewLogger() *zap.Logger { // 开发环境控制台友好的输出格式 logger, err : zap.NewDevelopment() if err ! nil { panic(err) } return logger } func NewProductionLogger() *zap.Logger { // 生产环境JSON格式输出 logger, err : zap.NewProduction() if err ! nil { panic(err) } return logger }3.2 日志记录func main() { logger : zap.NewExample() defer logger.Sync() // 基本日志 logger.Debug(Debug message) logger.Info(Info message) logger.Warn(Warn message) logger.Error(Error message) // 带字段的日志 logger.Info(User login, zap.String(user_id, 123), zap.String(username, john), zap.Time(login_time, time.Now()), ) // 结构化日志 logger.Error(Database connection failed, zap.Error(err), zap.String(host, localhost), zap.Int(port, 5432), ) }3.3 日志字段类型// 常用字段类型 zap.String(key, value) zap.Int(key, 42) zap.Int64(key, 42) zap.Uint(key, 42) zap.Float64(key, 3.14) zap.Bool(key, true) zap.Time(key, time.Now()) zap.Duration(key, time.Second) zap.Error(err) zap.Any(key, complexValue) // 数组字段 zap.Strings(tags, []string{tag1, tag2}) zap.Ints(ids, []int{1, 2, 3}) // 对象字段 zap.Object(user, User{ID: 123, Name: John}) // 跳过调用者信息 zap.Skip()四、Zap高级配置4.1 自定义配置func NewCustomLogger() *zap.Logger { cfg : zap.Config{ Level: zap.NewAtomicLevelAt(zap.InfoLevel), Development: false, Sampling: zap.SamplingConfig{ Initial: 100, Thereafter: 100, }, Encoding: json, EncoderConfig: zap.NewProductionEncoderConfig(), OutputPaths: []string{stdout, ./logs/app.log}, ErrorOutputPaths: []string{stderr, ./logs/error.log}, } // 自定义编码器配置 cfg.EncoderConfig zap.EncoderConfig{ TimeKey: time, LevelKey: level, NameKey: logger, CallerKey: caller, MessageKey: msg, StacktraceKey: stacktrace, LineEnding: zap.DefaultLineEnding, EncodeLevel: zap.LowercaseLevelEncoder, EncodeTime: zap.RFC3339TimeEncoder, EncodeDuration: zap.SecondsDurationEncoder, EncodeCaller: zap.ShortCallerEncoder, } logger, err : cfg.Build() if err ! nil { panic(err) } return logger }4.2 日志采样func NewSampledLogger() *zap.Logger { sampler : zap.NewSamplerWithOptions( zap.NewProduction(), time.Second, 100, // 初始采样数 10, // 之后每秒采样数 ) return sampler }4.3 日志分级输出func NewMultiLevelLogger() *zap.Logger { // 创建两个核心logger分别处理不同级别 highPriority : zap.LevelEnablerFunc(func(lvl zap.Level) bool { return lvl zap.ErrorLevel }) lowPriority : zap.LevelEnablerFunc(func(lvl zap.Level) bool { return lvl zap.ErrorLevel }) // 创建输出写入器 highWriter, _ : zap.Open(./logs/error.log) lowWriter, _ : zap.Open(./logs/app.log) // 创建core core : zap.NewTee( zap.NewCore( zap.NewJSONEncoder(zap.NewProductionEncoderConfig()), highWriter, highPriority, ), zap.NewCore( zap.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()), lowWriter, lowPriority, ), ) return zap.New(core) }五、日志封装与最佳实践5.1 自定义日志包装器type Logger struct { *zap.Logger } func NewLogger() *Logger { logger, _ : zap.NewProduction() return Logger{Logger: logger} } func (l *Logger) InfoWithUser(msg string, userID string, fields ...zap.Field) { l.Info(msg, append(fields, zap.String(user_id, userID))...) } func (l *Logger) ErrorWithRequest(msg string, r *http.Request, err error) { l.Error(msg, zap.Error(err), zap.String(method, r.Method), zap.String(path, r.URL.Path), zap.String(remote_addr, r.RemoteAddr), ) } func (l *Logger) DBQuery(query string, args ...interface{}) { l.Debug(SQL query, zap.String(query, query), zap.Any(args, args), ) }5.2 请求日志中间件func LoggingMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { start : time.Now() // 使用自定义ResponseWriter记录状态码 lw : loggingResponseWriter{ResponseWriter: w, statusCode: http.StatusOK} next.ServeHTTP(lw, r) duration : time.Since(start) logger.Info(HTTP request, zap.String(method, r.Method), zap.String(path, r.URL.Path), zap.Int(status, lw.statusCode), zap.Duration(duration, duration), zap.String(remote_addr, r.RemoteAddr), ) }) } type loggingResponseWriter struct { http.ResponseWriter statusCode int } func (lw *loggingResponseWriter) WriteHeader(code int) { lw.statusCode code lw.ResponseWriter.WriteHeader(code) }5.3 上下文日志func LoggerMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // 为每个请求创建带trace ID的logger traceID : uuid.New().String() logger : logger.With(zap.String(trace_id, traceID)) // 将logger存入context ctx : context.WithValue(r.Context(), logger, logger) next.ServeHTTP(w, r.WithContext(ctx)) }) } func GetLogger(ctx context.Context) *zap.Logger { if logger, ok : ctx.Value(logger).(*zap.Logger); ok { return logger } return zap.L() }六、日志轮换与归档6.1 使用lumberjack进行日志轮换import ( github.com/natefinch/lumberjack go.uber.org/zap ) func NewRotatingLogger() *zap.Logger { writer : lumberjack.Logger{ Filename: ./logs/app.log, MaxSize: 100, // 每个文件最大100MB MaxBackups: 3, // 保留3个备份 MaxAge: 28, // 保留28天 Compress: true, // 压缩归档文件 } core : zap.NewCore( zap.NewJSONEncoder(zap.NewProductionEncoderConfig()), zap.AddSync(writer), zap.InfoLevel, ) return zap.New(core) }6.2 定时清理日志func StartLogCleanup(logger *zap.Logger) { ticker : time.NewTicker(24 * time.Hour) defer ticker.Stop() for range ticker.C { err : cleanupOldLogs(./logs/, 30) if err ! nil { logger.Error(Failed to cleanup old logs, zap.Error(err)) } } } func cleanupOldLogs(dir string, maxDays int) error { cutoff : time.Now().AddDate(0, 0, -maxDays) files, err : os.ReadDir(dir) if err ! nil { return err } for _, file : range files { info, err : file.Info() if err ! nil { return err } if info.ModTime().Before(cutoff) { err : os.Remove(filepath.Join(dir, file.Name())) if err ! nil { return err } } } return nil }七、日志分析与监控7.1 指标收集type LogMetrics struct { requestCount prometheus.Counter errorCount prometheus.Counter latencyHistogram prometheus.Histogram } func NewLogMetrics() *LogMetrics { return LogMetrics{ requestCount: prometheus.NewCounter(prometheus.CounterOpts{ Name: http_requests_total, Help: Total number of HTTP requests, }), errorCount: prometheus.NewCounter(prometheus.CounterOpts{ Name: http_errors_total, Help: Total number of HTTP errors, }), latencyHistogram: prometheus.NewHistogram(prometheus.HistogramOpts{ Name: http_request_duration_seconds, Help: HTTP request duration in seconds, Buckets: prometheus.DefBuckets, }), } } func (m *LogMetrics) RecordRequest(statusCode int, duration time.Duration) { m.requestCount.Inc() m.latencyHistogram.Observe(duration.Seconds()) if statusCode 400 { m.errorCount.Inc() } }7.2 结构化日志查询// 使用jq查询日志示例 // cat app.log | jq .level error // cat app.log | jq .user_id 123 // cat app.log | jq .duration 1 func QueryLogsByLevel(logFile, level string) ([]map[string]interface{}, error) { file, err : os.Open(logFile) if err ! nil { return nil, err } defer file.Close() var results []map[string]interface{} scanner : bufio.NewScanner(file) for scanner.Scan() { var entry map[string]interface{} if err : json.Unmarshal(scanner.Bytes(), entry); err ! nil { continue } if entry[level] level { results append(results, entry) } } return results, nil }八、实战案例完整日志系统8.1 日志系统架构type LogSystem struct { logger *zap.Logger metrics *LogMetrics sampler *zap.Logger errorWriter *lumberjack.Logger } func NewLogSystem(config *Config) *LogSystem { // 创建错误写入器 errorWriter : lumberjack.Logger{ Filename: config.Log.ErrorFile, MaxSize: config.Log.MaxSize, MaxBackups: config.Log.MaxBackups, MaxAge: config.Log.MaxAge, Compress: config.Log.Compress, } // 创建主写入器 mainWriter : lumberjack.Logger{ Filename: config.Log.File, MaxSize: config.Log.MaxSize, MaxBackups: config.Log.MaxBackups, MaxAge: config.Log.MaxAge, Compress: config.Log.Compress, } // 创建核心 core : zap.NewTee( zap.NewCore( zap.NewJSONEncoder(zap.NewProductionEncoderConfig()), zap.AddSync(mainWriter), zap.LevelEnablerFunc(func(lvl zap.Level) bool { return lvl zap.DebugLevel }), ), zap.NewCore( zap.NewJSONEncoder(zap.NewProductionEncoderConfig()), zap.AddSync(errorWriter), zap.LevelEnablerFunc(func(lvl zap.Level) bool { return lvl zap.ErrorLevel }), ), ) logger : zap.New(core) return LogSystem{ logger: logger, metrics: NewLogMetrics(), errorWriter: errorWriter, } } func (ls *LogSystem) HandleRequest(w http.ResponseWriter, r *http.Request, handler func()) { start : time.Now() traceID : uuid.New().String() logger : ls.logger.With(zap.String(trace_id, traceID)) ctx : context.WithValue(r.Context(), logger, logger) defer func() { duration : time.Since(start) ls.metrics.RecordRequest(http.StatusOK, duration) logger.Info(Request completed, zap.String(method, r.Method), zap.String(path, r.URL.Path), zap.Duration(duration, duration), ) }() handler() }8.2 日志配置结构type LogConfig struct { File string mapstructure:file ErrorFile string mapstructure:error_file Level string mapstructure:level MaxSize int mapstructure:max_size MaxBackups int mapstructure:max_backups MaxAge int mapstructure:max_age Compress bool mapstructure:compress } type Config struct { Log LogConfig mapstructure:log // ... }结论日志系统是应用程序的重要组成部分。通过使用Zap这样的高性能日志库我们可以构建出高效、可扩展的日志系统。在实际项目中需要根据应用规模和需求选择合适的日志策略包括日志级别、输出格式、轮换策略等以确保日志系统既满足调试需求又不会影响应用性能。