feat: 建立可观测性、容量与恢复验证基线

This commit is contained in:
weijuesen
2026-08-14 14:54:56 +08:00
parent 126c215a6b
commit 1e6a5dbfb8
16 changed files with 398 additions and 3 deletions
+8 -1
View File
@@ -77,6 +77,7 @@ func NewAIClient(baseURL string) *AIClient {
// Detect 上传图片到 ai-service /detect,返回检测结果(无状态接口,天然幂等)
func (c *AIClient) Detect(ctx context.Context, imageBytes []byte, filename string) (*AIDetectResponse, error) {
start := time.Now()
var body bytes.Buffer
writer := multipart.NewWriter(&body)
part, err := writer.CreateFormFile("file", filename)
@@ -97,7 +98,9 @@ func (c *AIClient) Detect(ctx context.Context, imageBytes []byte, filename strin
req.Header.Set("Content-Type", writer.FormDataContentType())
resp, err := c.client.Do(req)
latency := time.Since(start)
if err != nil {
RecordDependency(DependencyAI, err, latency)
return nil, err
}
defer resp.Body.Close()
@@ -107,12 +110,16 @@ func (c *AIClient) Detect(ctx context.Context, imageBytes []byte, filename strin
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("ai-service /detect 返回 %d: %s", resp.StatusCode, string(data))
err := fmt.Errorf("ai-service /detect 返回 %d: %s", resp.StatusCode, string(data))
RecordDependency(DependencyAI, err, latency)
return nil, err
}
var out AIDetectResponse
if err := json.Unmarshal(data, &out); err != nil {
RecordDependency(DependencyAI, err, latency)
return nil, err
}
RecordDependency(DependencyAI, nil, latency)
return &out, nil
}
@@ -0,0 +1,86 @@
package service
import (
"sort"
"sync"
"time"
)
// Dependency 依赖名称。
type Dependency string
const (
DependencyPostgres Dependency = "postgresql"
DependencyRedis Dependency = "redis"
DependencyMQTT Dependency = "mqtt"
DependencyIoTDB Dependency = "iotdb"
DependencyS3 Dependency = "s3"
DependencyAI Dependency = "ai"
DependencyWVP Dependency = "wvp"
DependencyWechat Dependency = "wechat"
DependencyWeather Dependency = "weather"
)
// DependencyMetric 依赖调用指标。
type DependencyMetric struct {
Requests int64 `json:"requests"`
Failures int64 `json:"failures"`
LastLatencyMs float64 `json:"lastLatencyMs"`
P50LatencyMs float64 `json:"p50LatencyMs"`
P95LatencyMs float64 `json:"p95LatencyMs"`
LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
LastFailureAt *time.Time `json:"lastFailureAt,omitempty"`
LastError string `json:"lastError,omitempty"`
}
var dependencyMetrics sync.Map
// RecordDependency 记录一次依赖调用。
func RecordDependency(dep Dependency, err error, latency time.Duration) {
now := time.Now()
value, _ := dependencyMetrics.LoadOrStore(dep, &DependencyMetric{})
metric := value.(*DependencyMetric)
metric.Requests++
metric.LastLatencyMs = float64(latency.Microseconds()) / 1000
if err != nil {
metric.Failures++
metric.LastFailureAt = &now
metric.LastError = err.Error()
} else {
metric.LastSuccessAt = &now
metric.LastError = ""
}
}
// DependencySnapshot 返回当前依赖指标快照。
func DependencySnapshot() map[Dependency]DependencyMetric {
result := map[Dependency]DependencyMetric{}
dependencyMetrics.Range(func(key, value interface{}) bool {
dep := key.(Dependency)
metric := *value.(*DependencyMetric)
result[dep] = metric
return true
})
return result
}
// RecordDependencyHistogram 预留直方图能力;当前仅更新最近一次指标。
func RecordDependencyHistogram(dep Dependency, err error, latency time.Duration, samples []float64) {
RecordDependency(dep, err, latency)
if len(samples) == 0 {
return
}
value, _ := dependencyMetrics.LoadOrStore(dep, &DependencyMetric{})
metric := value.(*DependencyMetric)
sort.Float64s(samples)
metric.P50LatencyMs = percentile(samples, 0.5)
metric.P95LatencyMs = percentile(samples, 0.95)
}
func percentile(sorted []float64, p float64) float64 {
if len(sorted) == 0 {
return 0
}
idx := int(float64(len(sorted)-1) * p)
return sorted[idx]
}