feat: 建立可观测性、容量与恢复验证基线

2026-08-14 14:54:56 +08:00
parent 126c215a6b
commit 1e6a5dbfb8
16 changed files with 398 additions and 3 deletions
@@ -0,0 +1,28 @@
+package handler
+
+import (
+	"net/http"
+	"time"
+
+	"silk-server-go/internal/middleware"
+	"silk-server-go/internal/service"
+
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+)
+
+var opsStartedAt = time.Now()
+
+// RegisterOpsRoutes 注册运维指标路由。
+func RegisterOpsRoutes(rg *gin.RouterGroup, db *gorm.DB) {
+	rg.GET("/ops/metrics", middleware.RequirePermission(db, "log:read"), opsMetrics())
+}
+
+func opsMetrics() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.JSON(http.StatusOK, gin.H{
+			"uptimeSeconds": int(time.Since(opsStartedAt).Seconds()),
+			"dependencies":  service.DependencySnapshot(),
+		})
+	}
+}
@@ -2,6 +2,7 @@ package middleware

 import (
 	"log/slog"
+	"strings"
 	"time"

 	"github.com/gin-gonic/gin"
@@ -17,13 +18,22 @@ func Logger() gin.HandlerFunc {

 		latency := time.Since(start)
 		status := c.Writer.Status()
+		query := RedactSensitiveQuery(c.Request.URL.RawQuery)
+		auth := c.GetHeader("Authorization")
+		authPrefix := ""
+		if auth != "" {
+			authPrefix = strings.SplitN(auth, " ", 2)[0]
+		}

 		slog.Info("请求",
+			"requestId", RequestID(c),
 			"method", c.Request.Method,
 			"path", path,
+			"query", query,
 			"status", status,
 			"latency", latency.String(),
 			"ip", c.ClientIP(),
+			"authType", authPrefix,
 		)
 	}
 }
@@ -0,0 +1,65 @@
+package middleware
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+)
+
+const RequestIDHeader = "X-Request-ID"
+
+// RequestID 返回当前请求 ID。
+func RequestID(c *gin.Context) string {
+	value, _ := c.Get("requestId")
+	if id, ok := value.(string); ok {
+		return id
+	}
+	return ""
+}
+
+// RequestIDMiddleware 生成或透传 X-Request-ID。
+func RequestIDMiddleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		id := c.GetHeader(RequestIDHeader)
+		if id == "" {
+			id = randomRequestID()
+		}
+		c.Set("requestId", id)
+		c.Header(RequestIDHeader, id)
+		c.Next()
+	}
+}
+
+func randomRequestID() string {
+	b := make([]byte, 16)
+	if _, err := rand.Read(b); err != nil {
+		return "req-" + hex.EncodeToString([]byte(time.Now().Format(time.RFC3339Nano)))
+	}
+	return hex.EncodeToString(b)
+}
+
+// RedactSensitiveQuery 脱敏 URL query 中的 token/password/secret。
+func RedactSensitiveQuery(rawQuery string) string {
+	if rawQuery == "" {
+		return ""
+	}
+	values, err := url.ParseQuery(rawQuery)
+	if err != nil {
+		return "[invalid-query]"
+	}
+	keys := make([]string, 0, len(values))
+	for key := range values {
+		keys = append(keys, key)
+	}
+	for _, key := range keys {
+		lower := strings.ToLower(key)
+		if strings.Contains(lower, "token") || strings.Contains(lower, "password") || strings.Contains(lower, "secret") || strings.Contains(lower, "authorization") {
+			values.Set(key, "[redacted]")
+		}
+	}
+	return values.Encode()
+}
@@ -0,0 +1,42 @@
+package middleware
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+)
+
+func TestRequestIDMiddlewarePassesThrough(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	rec := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(rec)
+	c.Request = httptest.NewRequest(http.MethodGet, "/", nil)
+	c.Request.Header.Set(RequestIDHeader, "request-1")
+	RequestIDMiddleware()(c)
+	if rec.Header().Get(RequestIDHeader) != "request-1" {
+		t.Fatalf("header = %s", rec.Header().Get(RequestIDHeader))
+	}
+	if RequestID(c) != "request-1" {
+		t.Fatalf("request id = %s", RequestID(c))
+	}
+}
+
+func TestRequestIDMiddlewareGenerates(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	rec := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(rec)
+	c.Request = httptest.NewRequest(http.MethodGet, "/", nil)
+	RequestIDMiddleware()(c)
+	if RequestID(c) == "" {
+		t.Fatal("should generate request id")
+	}
+}
+
+func TestRedactSensitiveQuery(t *testing.T) {
+	got := RedactSensitiveQuery("access_token=abc&password=secret&roomId=1")
+	if got != "access_token=%5Bredacted%5D&password=%5Bredacted%5D&roomId=1" {
+		t.Fatalf("redacted query = %s", got)
+	}
+}
@@ -77,6 +77,7 @@ func NewAIClient(baseURL string) *AIClient {

 // Detect 上传图片到 ai-service /detect，返回检测结果（无状态接口，天然幂等）
 func (c *AIClient) Detect(ctx context.Context, imageBytes []byte, filename string) (*AIDetectResponse, error) {
+	start := time.Now()
 	var body bytes.Buffer
 	writer := multipart.NewWriter(&body)
 	part, err := writer.CreateFormFile("file", filename)
@@ -97,7 +98,9 @@ func (c *AIClient) Detect(ctx context.Context, imageBytes []byte, filename strin
 	req.Header.Set("Content-Type", writer.FormDataContentType())

 	resp, err := c.client.Do(req)
+	latency := time.Since(start)
 	if err != nil {
+		RecordDependency(DependencyAI, err, latency)
 		return nil, err
 	}
 	defer resp.Body.Close()
@@ -107,12 +110,16 @@ func (c *AIClient) Detect(ctx context.Context, imageBytes []byte, filename strin
 		return nil, err
 	}
 	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("ai-service /detect 返回 %d: %s", resp.StatusCode, string(data))
+		err := fmt.Errorf("ai-service /detect 返回 %d: %s", resp.StatusCode, string(data))
+		RecordDependency(DependencyAI, err, latency)
+		return nil, err
 	}

 	var out AIDetectResponse
 	if err := json.Unmarshal(data, &out); err != nil {
+		RecordDependency(DependencyAI, err, latency)
 		return nil, err
 	}
+	RecordDependency(DependencyAI, nil, latency)
 	return &out, nil
 }
@@ -0,0 +1,86 @@
+package service
+
+import (
+	"sort"
+	"sync"
+	"time"
+)
+
+// Dependency 依赖名称。
+type Dependency string
+
+const (
+	DependencyPostgres Dependency = "postgresql"
+	DependencyRedis    Dependency = "redis"
+	DependencyMQTT     Dependency = "mqtt"
+	DependencyIoTDB    Dependency = "iotdb"
+	DependencyS3       Dependency = "s3"
+	DependencyAI       Dependency = "ai"
+	DependencyWVP      Dependency = "wvp"
+	DependencyWechat   Dependency = "wechat"
+	DependencyWeather  Dependency = "weather"
+)
+
+// DependencyMetric 依赖调用指标。
+type DependencyMetric struct {
+	Requests      int64      `json:"requests"`
+	Failures      int64      `json:"failures"`
+	LastLatencyMs float64    `json:"lastLatencyMs"`
+	P50LatencyMs  float64    `json:"p50LatencyMs"`
+	P95LatencyMs  float64    `json:"p95LatencyMs"`
+	LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
+	LastFailureAt *time.Time `json:"lastFailureAt,omitempty"`
+	LastError     string     `json:"lastError,omitempty"`
+}
+
+var dependencyMetrics sync.Map
+
+// RecordDependency 记录一次依赖调用。
+func RecordDependency(dep Dependency, err error, latency time.Duration) {
+	now := time.Now()
+	value, _ := dependencyMetrics.LoadOrStore(dep, &DependencyMetric{})
+	metric := value.(*DependencyMetric)
+	metric.Requests++
+	metric.LastLatencyMs = float64(latency.Microseconds()) / 1000
+	if err != nil {
+		metric.Failures++
+		metric.LastFailureAt = &now
+		metric.LastError = err.Error()
+	} else {
+		metric.LastSuccessAt = &now
+		metric.LastError = ""
+	}
+}
+
+// DependencySnapshot 返回当前依赖指标快照。
+func DependencySnapshot() map[Dependency]DependencyMetric {
+	result := map[Dependency]DependencyMetric{}
+	dependencyMetrics.Range(func(key, value interface{}) bool {
+		dep := key.(Dependency)
+		metric := *value.(*DependencyMetric)
+		result[dep] = metric
+		return true
+	})
+	return result
+}
+
+// RecordDependencyHistogram 预留直方图能力；当前仅更新最近一次指标。
+func RecordDependencyHistogram(dep Dependency, err error, latency time.Duration, samples []float64) {
+	RecordDependency(dep, err, latency)
+	if len(samples) == 0 {
+		return
+	}
+	value, _ := dependencyMetrics.LoadOrStore(dep, &DependencyMetric{})
+	metric := value.(*DependencyMetric)
+	sort.Float64s(samples)
+	metric.P50LatencyMs = percentile(samples, 0.5)
+	metric.P95LatencyMs = percentile(samples, 0.95)
+}
+
+func percentile(sorted []float64, p float64) float64 {
+	if len(sorted) == 0 {
+		return 0
+	}
+	idx := int(float64(len(sorted)-1) * p)
+	return sorted[idx]
+}