zhouchengcheng/ctms
1
1
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

优化系统健康评分与缓存监测

Browse Source
This commit is contained in:
Cheng Zhou
2026-06-30 10:43:24 +08:00
parent b25055775e
commit 668e719b6d
10 changed files with 412 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/api/v1/permission_monitoring.py
+75 -17
View File
@@ -15,7 +15,11 @@ from sqlalchemy import func, select, desc
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.deps import get_current_user, get_db_session, is_system_admin
from app.core.permission_monitor import get_permission_monitor
from app.core.permission_monitor import (
CACHE_HIT_RATE_HEALTH_MIN_ACCESSES,
CACHE_METRICS_WINDOW_SECONDS,
get_permission_monitor,
)
from app.models.permission_access_log import PermissionAccessLog
from app.models.permission_metric_snapshot import PermissionMetricSnapshot
from app.models.security_access_log import SecurityAccessLog
@@ -78,7 +82,7 @@ async def get_permission_metrics(
row = result.one()
total = row.total_checks or 0
monitor = get_permission_monitor()
cache_metrics = monitor.metrics.cache_metrics.to_dict()
cache_metrics = monitor.metrics.cache_metrics.window_stats()
return {
"window_hours": hours,
"check_metrics": {
@@ -178,33 +182,87 @@ async def permission_system_health(
deny_rate = round(row.denied / total * 100, 2) if total else 0
monitor = get_permission_monitor()
cache_metrics = monitor.metrics.cache_metrics
cache_metrics = monitor.metrics.cache_metrics.window_stats()
now_timestamp = datetime.now(timezone.utc).timestamp()
recent_alerts = [
alert
for alert in monitor.get_alerts(limit=1000)
if alert["timestamp"] >= now_timestamp - 3600
]
health_score = 100
issues = []
deductions: dict[str, int] = {
"performance": 0,
"deny_rate": 0,
"cache": 0,
"alerts": 0,
}
issues: list[str] = []
if avg_ms > 200:
deductions["performance"] = 40
elif avg_ms > 50:
deductions["performance"] = 25
elif avg_ms > 10:
deductions["performance"] = 10
if deductions["performance"]:
issues.append(f"权限检查平均响应时间过长（{avg_ms:.2f}ms）")
if avg_ms > 10:
health_score -= 10
issues.append("权限检查响应时间过长")
if deny_rate > 50:
health_score -= 5
issues.append("权限拒绝率过高")
if (
cache_metrics.total_accesses >= 10
and cache_metrics.hit_rate < 50
):
health_score -= 10
issues.append("缓存命中率过低")
deductions["deny_rate"] = 20
elif deny_rate > 20:
deductions["deny_rate"] = 10
if deductions["deny_rate"]:
issues.append(f"权限拒绝率偏高（{deny_rate:.2f}%）")
cache_sample_sufficient = (
cache_metrics["total_accesses"] >= CACHE_HIT_RATE_HEALTH_MIN_ACCESSES
)
if cache_sample_sufficient:
if cache_metrics["hit_rate"] < 20:
deductions["cache"] = 25
elif cache_metrics["hit_rate"] < 50:
deductions["cache"] = 20
elif cache_metrics["hit_rate"] < 80:
deductions["cache"] = 10
if deductions["cache"]:
issues.append(f"近 1 小时缓存命中率偏低（{cache_metrics['hit_rate']:.2f}%）")
error_alert_count = sum(alert["level"] == "error" for alert in recent_alerts)
warning_alert_count = sum(alert["level"] == "warning" for alert in recent_alerts)
if error_alert_count:
deductions["alerts"] = 15
issues.append(f"近 1 小时发生 {error_alert_count} 条错误告警")
elif warning_alert_count:
deductions["alerts"] = 5
issues.append(f"近 1 小时发生 {warning_alert_count} 条警告")
health_score = max(0, 100 - sum(deductions.values()))
sample_sufficient = total >= 10
return {
"status": "healthy" if health_score >= 80 else "degraded" if health_score >= 50 else "unhealthy",
"health_score": max(0, health_score),
"health_score": health_score,
"issues": issues,
"sample_sufficient": sample_sufficient,
"score_breakdown": {
"performance": {"weight": 40, "deduction": deductions["performance"]},
"deny_rate": {"weight": 20, "deduction": deductions["deny_rate"]},
"cache": {
"weight": 25,
"deduction": deductions["cache"],
"sample_sufficient": cache_sample_sufficient,
"scope": "sliding_window",
"window_seconds": CACHE_METRICS_WINDOW_SECONDS,
},
"alerts": {"weight": 15, "deduction": deductions["alerts"]},
},
"last_hour": {
"total_checks": total,
"denied_checks": row.denied or 0,
"deny_rate": deny_rate,
"avg_elapsed_ms": round(avg_ms, 2),
"error_alerts": error_alert_count,
"warning_alerts": warning_alert_count,
},
"cache_stats": monitor.get_cache_stats(),
}
backend/app/core/permission_cache.py
+15
View File
@@ -44,8 +44,12 @@ class PermissionCache:
if cache_key in self._member_role_cache:
cached_role, timestamp = self._member_role_cache[cache_key]
if not self._is_expired(timestamp, ttl):
from app.core.permission_monitor import get_permission_monitor
get_permission_monitor().record_cache_hit("member_roles")
return cached_role
from app.core.permission_monitor import get_permission_monitor
get_permission_monitor().record_cache_miss("member_roles")
membership = await member_crud.get_member(db, study_id, user_id)
role = membership.role_in_study if membership and membership.is_active else None
@@ -102,6 +106,17 @@ class PermissionCache:
self._member_role_cache.clear()
def get_cache_stats(self) -> dict[str, Any]:
now = time.time()
self._project_permissions_cache = {
key: value
for key, value in self._project_permissions_cache.items()
if now - value[1] <= self.default_ttl
}
self._member_role_cache = {
key: value
for key, value in self._member_role_cache.items()
if now - value[1] <= self.default_ttl
}
return {
"project_permissions_count": len(self._project_permissions_cache),
"member_role_count": len(self._member_role_cache),
backend/app/core/permission_monitor.py
+64 -10
View File
@@ -8,13 +8,15 @@ permission_access_logs 表中，通过 /metrics 端点实时聚合查询。
from __future__ import annotations
import time
from collections import deque
from dataclasses import dataclass, field
from typing import Any
from app.core.permission_cache import get_permission_cache
CACHE_HIT_RATE_HEALTH_MIN_ACCESSES = 10
CACHE_HIT_RATE_HEALTH_MIN_ACCESSES = 50
CACHE_METRICS_WINDOW_SECONDS = 3600
@dataclass
@@ -25,6 +27,11 @@ class CacheMetrics:
cache_hits: int = 0
cache_misses: int = 0
cache_invalidations: int = 0
_access_events: deque[tuple[float, str, bool]] = field(
default_factory=deque,
repr=False,
)
_invalidation_events: deque[float] = field(default_factory=deque, repr=False)
@property
def hit_rate(self) -> float:
@@ -48,6 +55,53 @@ class CacheMetrics:
"miss_rate": round(self.miss_rate, 2),
}
def record_access(self, cache_name: str, *, hit: bool) -> None:
self.total_accesses += 1
if hit:
self.cache_hits += 1
else:
self.cache_misses += 1
self._access_events.append((time.time(), cache_name, hit))
def record_invalidation(self) -> None:
self.cache_invalidations += 1
self._invalidation_events.append(time.time())
def window_stats(self, seconds: int = CACHE_METRICS_WINDOW_SECONDS) -> dict[str, Any]:
cutoff = time.time() - seconds
while self._access_events and self._access_events[0][0] < cutoff:
self._access_events.popleft()
while self._invalidation_events and self._invalidation_events[0] < cutoff:
self._invalidation_events.popleft()
def summarize(cache_name: str | None = None) -> dict[str, Any]:
events = [
event for event in self._access_events
if cache_name is None or event[1] == cache_name
]
hits = sum(event[2] for event in events)
total = len(events)
misses = total - hits
return {
"total_accesses": total,
"cache_hits": hits,
"cache_misses": misses,
"cache_invalidations": len(self._invalidation_events),
"hit_rate": round(hits / total * 100, 2) if total else 0.0,
"miss_rate": round(misses / total * 100, 2) if total else 0.0,
}
stats = summarize()
stats.update({
"window_seconds": seconds,
"scope": "sliding_window",
"by_cache": {
"project_permissions": summarize("project_permissions"),
"member_roles": summarize("member_roles"),
},
})
return stats
@dataclass
class PermissionSystemMetrics:
@@ -70,16 +124,14 @@ class PermissionMonitor:
self.metrics = PermissionSystemMetrics()
self._alerts: list[dict[str, Any]] = []
def record_cache_hit(self) -> None:
self.metrics.cache_metrics.total_accesses += 1
self.metrics.cache_metrics.cache_hits += 1
def record_cache_hit(self, cache_name: str = "project_permissions") -> None:
self.metrics.cache_metrics.record_access(cache_name, hit=True)
def record_cache_miss(self) -> None:
self.metrics.cache_metrics.total_accesses += 1
self.metrics.cache_metrics.cache_misses += 1
def record_cache_miss(self, cache_name: str = "project_permissions") -> None:
self.metrics.cache_metrics.record_access(cache_name, hit=False)
def record_cache_invalidation(self) -> None:
self.metrics.cache_metrics.cache_invalidations += 1
self.metrics.cache_metrics.record_invalidation()
def record_error_alert(self, error: Exception) -> None:
self._add_alert(
@@ -119,12 +171,14 @@ class PermissionMonitor:
cache = get_permission_cache()
return {
"cache_items": cache.get_cache_stats(),
"cache_metrics": self.metrics.cache_metrics.to_dict(),
"cache_metrics": self.metrics.cache_metrics.window_stats(),
"cache_metrics_lifetime": self.metrics.cache_metrics.to_dict(),
}
def get_metrics(self) -> dict[str, Any]:
return {
"cache_metrics": self.metrics.cache_metrics.to_dict(),
"cache_metrics": self.metrics.cache_metrics.window_stats(),
"cache_metrics_lifetime": self.metrics.cache_metrics.to_dict(),
"uptime_seconds": self.metrics.uptime_seconds,
}
backend/app/services/permission_metric_aggregator.py
+3 -3
View File
@@ -40,7 +40,7 @@ async def _aggregate_hour(bucket_start: datetime, bucket_end: datetime) -> None:
from app.core.permission_monitor import get_permission_monitor
monitor = get_permission_monitor()
cache_metrics = monitor.metrics.cache_metrics
cache_metrics = monitor.metrics.cache_metrics.window_stats(3600)
snapshot = PermissionMetricSnapshot(
id=uuid.uuid4(),
@@ -50,8 +50,8 @@ async def _aggregate_hour(bucket_start: datetime, bucket_end: datetime) -> None:
denied_checks=row.denied,
avg_elapsed_ms=float(row.avg_ms),
max_elapsed_ms=float(row.max_ms),
cache_hits=cache_metrics.cache_hits,
cache_misses=cache_metrics.cache_misses,
cache_hits=cache_metrics["cache_hits"],
cache_misses=cache_metrics["cache_misses"],
error_count=0,
)
session.add(snapshot)