Cheng Zhou
1929 lines
82 KiB
Python
1929 lines
82 KiB
Python
"""权限系统监控API
|
||
|
||
提供权限系统的监控数据、访问日志、趋势分析和告警信息。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import uuid
|
||
import json
|
||
import time
|
||
from collections import defaultdict
|
||
from datetime import datetime, timedelta, timezone
|
||
from typing import Optional
|
||
|
||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||
from sqlalchemy import String, case, cast, desc, func, literal, or_, select, union_all
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.core.config import settings
|
||
from app.core.deps import get_current_user, get_db_session, is_system_admin
|
||
from app.core.permission_monitor import (
|
||
CACHE_HIT_RATE_HEALTH_MIN_ACCESSES,
|
||
CACHE_METRICS_WINDOW_SECONDS,
|
||
get_permission_monitor,
|
||
)
|
||
from app.models.permission_access_log import PermissionAccessLog
|
||
from app.models.permission_metric_snapshot import PermissionMetricSnapshot
|
||
from app.models.security_access_log import SecurityAccessLog
|
||
from app.models.user import User
|
||
from app.services.geo_location_metadata import GeoLocationMetadata, resolve_geo_location_metadata
|
||
from app.services.ip_location import IpLocation, resolve_ip_location
|
||
from app.services.monitoring_retention import get_monitoring_retention_status
|
||
from app.services.permission_log_writer import get_log_writer
|
||
from app.services.security_access_log_writer import get_security_log_writer
|
||
from app.services.security_events import classify_security_event
|
||
from app.services.monitoring_server_location import resolve_monitoring_server_location
|
||
from app.services.source_location_aggregator import get_source_location_timeline
|
||
|
||
router = APIRouter(prefix="/permission-monitoring", tags=["permission-monitoring"])
|
||
|
||
|
||
class MonitoringScope:
|
||
def __init__(self, *, is_admin: bool, study_ids: set[uuid.UUID]) -> None:
|
||
self.is_admin = is_admin
|
||
self.study_ids = study_ids
|
||
|
||
def can_access_study(self, study_id: uuid.UUID | None) -> bool:
|
||
if self.is_admin:
|
||
return True
|
||
return study_id is not None and study_id in self.study_ids
|
||
|
||
|
||
async def resolve_monitoring_scope(db: AsyncSession, current_user) -> MonitoringScope:
|
||
if is_system_admin(current_user):
|
||
return MonitoringScope(is_admin=True, study_ids=set())
|
||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足")
|
||
|
||
|
||
def _apply_monitoring_scope_to_log_query(query, scope: MonitoringScope):
|
||
if scope.is_admin:
|
||
return query
|
||
return query.where(PermissionAccessLog.study_id.in_(scope.study_ids))
|
||
|
||
|
||
KNOWN_PERMISSION_CLIENT_TYPES = ("web", "desktop")
|
||
|
||
|
||
def _permission_client_type_condition(client_type: str):
|
||
normalized = client_type.strip().lower()
|
||
if normalized == "unknown":
|
||
return or_(
|
||
PermissionAccessLog.client_type.is_(None),
|
||
PermissionAccessLog.client_type == "",
|
||
~PermissionAccessLog.client_type.in_(KNOWN_PERMISSION_CLIENT_TYPES),
|
||
)
|
||
return PermissionAccessLog.client_type == normalized
|
||
|
||
|
||
def _security_client_type_condition(client_type: str):
|
||
normalized = client_type.strip().lower()
|
||
if normalized == "unknown":
|
||
return or_(
|
||
SecurityAccessLog.client_type.is_(None),
|
||
SecurityAccessLog.client_type == "",
|
||
~SecurityAccessLog.client_type.in_(KNOWN_PERMISSION_CLIENT_TYPES),
|
||
)
|
||
return SecurityAccessLog.client_type == normalized
|
||
|
||
|
||
def _coerce_request_headers(value) -> dict[str, str] | None:
|
||
if not value:
|
||
return None
|
||
if isinstance(value, str):
|
||
try:
|
||
value = json.loads(value)
|
||
except json.JSONDecodeError:
|
||
return None
|
||
if not isinstance(value, dict):
|
||
return None
|
||
headers = {str(key): str(header_value) for key, header_value in value.items() if header_value is not None}
|
||
return headers or None
|
||
|
||
|
||
def _coerce_request_snapshot(value) -> dict | None:
|
||
if not value:
|
||
return None
|
||
if isinstance(value, str):
|
||
try:
|
||
value = json.loads(value)
|
||
except json.JSONDecodeError:
|
||
return None
|
||
return value if isinstance(value, dict) else None
|
||
|
||
|
||
async def _resolve_permission_location_keyword_ips(
|
||
db: AsyncSession,
|
||
conditions: list,
|
||
keyword: str,
|
||
) -> list[str]:
|
||
token = keyword.strip().lower()
|
||
if not token:
|
||
return []
|
||
ip_query = (
|
||
select(PermissionAccessLog.ip_address)
|
||
.where(PermissionAccessLog.ip_address.is_not(None))
|
||
.distinct()
|
||
)
|
||
for condition in conditions:
|
||
ip_query = ip_query.where(condition)
|
||
result = await db.execute(ip_query)
|
||
matched_ips: list[str] = []
|
||
for ip_address in result.scalars().all():
|
||
ip_location = resolve_ip_location(ip_address)
|
||
haystack = " ".join(
|
||
[
|
||
ip_location.location or "",
|
||
ip_location.country or "",
|
||
ip_location.province or "",
|
||
ip_location.city or "",
|
||
ip_location.isp or "",
|
||
]
|
||
).lower()
|
||
if token in haystack:
|
||
matched_ips.append(ip_address)
|
||
return matched_ips
|
||
|
||
|
||
async def _resolve_security_location_keyword_ips(
|
||
db: AsyncSession,
|
||
conditions: list,
|
||
keyword: str,
|
||
) -> list[str]:
|
||
token = keyword.strip().lower()
|
||
if not token:
|
||
return []
|
||
ip_query = (
|
||
select(SecurityAccessLog.client_ip)
|
||
.where(SecurityAccessLog.client_ip.is_not(None))
|
||
.distinct()
|
||
)
|
||
for condition in conditions:
|
||
ip_query = ip_query.where(condition)
|
||
result = await db.execute(ip_query)
|
||
matched_ips: list[str] = []
|
||
for ip_address in result.scalars().all():
|
||
ip_location = resolve_ip_location(ip_address)
|
||
haystack = " ".join(
|
||
[
|
||
ip_location.location or "",
|
||
ip_location.country or "",
|
||
ip_location.province or "",
|
||
ip_location.city or "",
|
||
ip_location.isp or "",
|
||
]
|
||
).lower()
|
||
if token in haystack:
|
||
matched_ips.append(ip_address)
|
||
return matched_ips
|
||
|
||
|
||
def _permission_access_item(log: PermissionAccessLog, full_name: str | None, email: str | None) -> dict:
|
||
ip_location = resolve_ip_location(log.ip_address)
|
||
return {
|
||
"id": str(log.id),
|
||
"event_type": "permission",
|
||
"study_id": str(log.study_id) if log.study_id else "",
|
||
"user_id": str(log.user_id) if log.user_id else "",
|
||
"user_name": full_name or "未知用户",
|
||
"user_email": email,
|
||
"endpoint_key": log.endpoint_key,
|
||
"role": log.role,
|
||
"allowed": log.allowed,
|
||
"elapsed_ms": round(log.elapsed_ms, 2),
|
||
"ip_address": log.ip_address,
|
||
"ip_location": ip_location.location,
|
||
"ip_country": ip_location.country,
|
||
"ip_province": ip_location.province,
|
||
"ip_city": ip_location.city,
|
||
"ip_isp": ip_location.isp,
|
||
"user_agent": log.user_agent,
|
||
"client_type": log.client_type,
|
||
"client_version": log.client_version,
|
||
"client_platform": log.client_platform,
|
||
"build_channel": log.build_channel,
|
||
"build_commit": log.build_commit,
|
||
"request_headers": _coerce_request_headers(log.request_headers),
|
||
"request_snapshot": _coerce_request_snapshot(getattr(log, "request_snapshot", None)),
|
||
"request_id": getattr(log, "request_id", None),
|
||
"method": None,
|
||
"path": None,
|
||
"status_code": None,
|
||
"auth_status": "AUTHENTICATED",
|
||
"account_label": full_name or email,
|
||
"category": None,
|
||
"severity": None,
|
||
"created_at": log.created_at.isoformat(),
|
||
}
|
||
|
||
|
||
def _security_access_item(log: SecurityAccessLog, user_names: dict[str, str] | None = None) -> dict:
|
||
ip_location = resolve_ip_location(log.client_ip)
|
||
classification = _classify_security_access_log(log, ip_location)
|
||
account_label = _security_account_label(log.auth_status, log.user_identifier, user_names)
|
||
return {
|
||
"id": f"security:{log.id}",
|
||
"event_type": "security",
|
||
"study_id": "",
|
||
"user_id": log.user_identifier or "",
|
||
"user_name": account_label,
|
||
"user_email": None,
|
||
"endpoint_key": f"{log.method} {log.path}",
|
||
"role": classification["category"],
|
||
"allowed": log.status_code < 400,
|
||
"elapsed_ms": round(log.elapsed_ms, 2),
|
||
"ip_address": log.client_ip,
|
||
"ip_location": ip_location.location,
|
||
"ip_country": ip_location.country,
|
||
"ip_province": ip_location.province,
|
||
"ip_city": ip_location.city,
|
||
"ip_isp": ip_location.isp,
|
||
"user_agent": log.user_agent,
|
||
"client_type": log.client_type,
|
||
"client_version": log.client_version,
|
||
"client_platform": log.client_platform,
|
||
"build_channel": log.build_channel,
|
||
"build_commit": log.build_commit,
|
||
"request_headers": _coerce_request_headers(getattr(log, "request_headers", None)),
|
||
"request_snapshot": _coerce_request_snapshot(getattr(log, "request_snapshot", None)),
|
||
"request_id": getattr(log, "request_id", None),
|
||
"method": log.method,
|
||
"path": log.path,
|
||
"status_code": log.status_code,
|
||
"auth_status": log.auth_status,
|
||
"account_label": account_label,
|
||
"category": classification["category"],
|
||
"severity": classification["severity"],
|
||
"created_at": log.created_at.isoformat(),
|
||
}
|
||
|
||
|
||
async def _security_user_names(db: AsyncSession, logs: list[SecurityAccessLog]) -> dict[str, str]:
|
||
user_ids: list[uuid.UUID] = []
|
||
for log in logs:
|
||
if log.auth_status != "AUTHENTICATED" or not log.user_identifier:
|
||
continue
|
||
try:
|
||
user_ids.append(uuid.UUID(log.user_identifier))
|
||
except ValueError:
|
||
continue
|
||
if not user_ids:
|
||
return {}
|
||
users_result = await db.execute(select(User.id, User.full_name).where(User.id.in_(user_ids)))
|
||
return {str(row.id): row.full_name for row in users_result.all()}
|
||
|
||
|
||
# ═══════════════════════════════════════════
|
||
# 原有端点（保持兼容）
|
||
# ═══════════════════════════════════════════
|
||
|
||
@router.get("/metrics", status_code=status.HTTP_200_OK)
|
||
async def get_permission_metrics(
|
||
db: AsyncSession = Depends(get_db_session),
|
||
_=Depends(get_current_user),
|
||
hours: int = Query(24, ge=1, le=720),
|
||
) -> dict:
|
||
"""从 permission_access_logs 实时聚合权限检查指标"""
|
||
scope = await resolve_monitoring_scope(db, _)
|
||
if not scope.is_admin and not scope.study_ids:
|
||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足")
|
||
start_time = datetime.now(timezone.utc) - timedelta(hours=hours)
|
||
metrics_query = select(
|
||
func.count().label("total_checks"),
|
||
func.count().filter(PermissionAccessLog.allowed.is_(True)).label("allowed_checks"),
|
||
func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied_checks"),
|
||
func.coalesce(func.sum(PermissionAccessLog.elapsed_ms), 0).label("total_time_ms"),
|
||
func.coalesce(func.min(PermissionAccessLog.elapsed_ms), 0).label("min_time_ms"),
|
||
func.coalesce(func.max(PermissionAccessLog.elapsed_ms), 0).label("max_time_ms"),
|
||
func.coalesce(func.avg(PermissionAccessLog.elapsed_ms), 0).label("avg_time_ms"),
|
||
).where(PermissionAccessLog.created_at >= start_time)
|
||
result = await db.execute(
|
||
_apply_monitoring_scope_to_log_query(metrics_query, scope)
|
||
)
|
||
row = result.one()
|
||
total = row.total_checks or 0
|
||
monitor = get_permission_monitor()
|
||
cache_metrics = monitor.metrics.cache_metrics.window_stats()
|
||
return {
|
||
"window_hours": hours,
|
||
"check_metrics": {
|
||
"total_checks": total,
|
||
"allowed_checks": row.allowed_checks or 0,
|
||
"denied_checks": row.denied_checks or 0,
|
||
"total_time": round(float(row.total_time_ms) / 1000, 3),
|
||
"min_time": round(float(row.min_time_ms) / 1000, 3),
|
||
"max_time": round(float(row.max_time_ms) / 1000, 3),
|
||
"avg_time": round(float(row.avg_time_ms) / 1000, 3),
|
||
"allow_rate": round(row.allowed_checks / total * 100, 2) if total else 0,
|
||
"deny_rate": round(row.denied_checks / total * 100, 2) if total else 0,
|
||
"error_rate": 0,
|
||
"errors": 0,
|
||
},
|
||
"cache_metrics": cache_metrics,
|
||
"uptime_seconds": monitor.metrics.uptime_seconds,
|
||
}
|
||
|
||
|
||
@router.get("/cache-stats", status_code=status.HTTP_200_OK)
|
||
async def get_cache_statistics(
|
||
_=Depends(get_current_user),
|
||
db: AsyncSession = Depends(get_db_session),
|
||
) -> dict:
|
||
scope = await resolve_monitoring_scope(db, _)
|
||
if not scope.is_admin and not scope.study_ids:
|
||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足")
|
||
monitor = get_permission_monitor()
|
||
return monitor.get_cache_stats()
|
||
|
||
|
||
@router.get("/alerts", status_code=status.HTTP_200_OK)
|
||
async def get_alerts(
|
||
level: str | None = None,
|
||
limit: int = 100,
|
||
_=Depends(get_current_user),
|
||
db: AsyncSession = Depends(get_db_session),
|
||
) -> dict:
|
||
scope = await resolve_monitoring_scope(db, _)
|
||
if not scope.is_admin and not scope.study_ids:
|
||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足")
|
||
monitor = get_permission_monitor()
|
||
alerts = monitor.get_alerts(level=level, limit=limit)
|
||
return {
|
||
"total": len(alerts),
|
||
"alerts": alerts,
|
||
}
|
||
|
||
@router.post("/reset-metrics", status_code=status.HTTP_200_OK)
|
||
async def reset_metrics(
|
||
_=Depends(get_current_user),
|
||
db: AsyncSession = Depends(get_db_session),
|
||
) -> dict:
|
||
scope = await resolve_monitoring_scope(db, _)
|
||
if not scope.is_admin and not scope.study_ids:
|
||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足")
|
||
monitor = get_permission_monitor()
|
||
monitor.reset_metrics()
|
||
return {"message": "指标已重置"}
|
||
|
||
|
||
@router.post("/clear-alerts", status_code=status.HTTP_200_OK)
|
||
async def clear_alerts(
|
||
_=Depends(get_current_user),
|
||
db: AsyncSession = Depends(get_db_session),
|
||
) -> dict:
|
||
scope = await resolve_monitoring_scope(db, _)
|
||
if not scope.is_admin and not scope.study_ids:
|
||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足")
|
||
monitor = get_permission_monitor()
|
||
monitor.clear_alerts()
|
||
return {"message": "告警已清除"}
|
||
|
||
|
||
@router.get("/health", status_code=status.HTTP_200_OK)
|
||
async def permission_system_health(
|
||
db: AsyncSession = Depends(get_db_session),
|
||
_=Depends(get_current_user),
|
||
) -> dict:
|
||
"""从 DB 聚合最近 1 小时数据评估权限系统健康状态"""
|
||
scope = await resolve_monitoring_scope(db, _)
|
||
if not scope.is_admin and not scope.study_ids:
|
||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足")
|
||
start_time = datetime.now(timezone.utc) - timedelta(hours=1)
|
||
database_started_at = time.perf_counter()
|
||
await db.execute(select(literal(1)))
|
||
database_latency_ms = round((time.perf_counter() - database_started_at) * 1000, 2)
|
||
health_query = select(
|
||
func.count().label("total"),
|
||
func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied"),
|
||
func.coalesce(func.avg(PermissionAccessLog.elapsed_ms), 0).label("avg_ms"),
|
||
).where(PermissionAccessLog.created_at >= start_time)
|
||
result = await db.execute(
|
||
_apply_monitoring_scope_to_log_query(health_query, scope)
|
||
)
|
||
row = result.one()
|
||
total = row.total or 0
|
||
avg_ms = float(row.avg_ms)
|
||
deny_rate = round(row.denied / total * 100, 2) if total else 0
|
||
|
||
monitor = get_permission_monitor()
|
||
cache_metrics = monitor.metrics.cache_metrics.window_stats()
|
||
now_timestamp = datetime.now(timezone.utc).timestamp()
|
||
recent_alerts = [
|
||
alert
|
||
for alert in monitor.get_alerts(limit=1000)
|
||
if alert["timestamp"] >= now_timestamp - 3600
|
||
]
|
||
|
||
permission_writer = get_log_writer()
|
||
security_writer = get_security_log_writer()
|
||
permission_writer_stats = permission_writer.stats() if permission_writer else None
|
||
security_writer_stats = security_writer.stats() if security_writer else None
|
||
retention_stats = get_monitoring_retention_status()
|
||
|
||
def writer_component(stats: dict | None) -> dict:
|
||
if stats is None:
|
||
return {"status": "unknown", "reason": "writer_not_initialized"}
|
||
queue_capacity = max(1, int(stats.get("queue_capacity") or 1))
|
||
queue_usage = round(int(stats.get("queue_size") or 0) / queue_capacity * 100, 2)
|
||
if not stats.get("running"):
|
||
component_status = "unhealthy"
|
||
elif queue_usage >= 80 or stats.get("failed_batches") or stats.get("dropped_entries"):
|
||
component_status = "degraded"
|
||
else:
|
||
component_status = "healthy"
|
||
return {**stats, "status": component_status, "queue_usage_percent": queue_usage}
|
||
|
||
permission_writer_component = writer_component(permission_writer_stats)
|
||
security_writer_component = writer_component(security_writer_stats)
|
||
if retention_stats["running"]:
|
||
retention_has_unrecovered_error = bool(
|
||
retention_stats["last_error_at"]
|
||
and (
|
||
not retention_stats["last_success_at"]
|
||
or retention_stats["last_error_at"] > retention_stats["last_success_at"]
|
||
)
|
||
)
|
||
retention_status = "degraded" if retention_has_unrecovered_error else "healthy"
|
||
elif retention_stats["last_run_started_at"]:
|
||
retention_status = "unhealthy"
|
||
else:
|
||
retention_status = "unknown"
|
||
retention_component = {**retention_stats, "status": retention_status}
|
||
|
||
storage_result = await db.execute(
|
||
select(
|
||
select(func.count()).select_from(PermissionAccessLog).scalar_subquery().label("permission_count"),
|
||
select(func.min(PermissionAccessLog.created_at)).scalar_subquery().label("permission_oldest"),
|
||
select(func.count()).select_from(SecurityAccessLog).scalar_subquery().label("security_count"),
|
||
select(func.min(SecurityAccessLog.created_at)).scalar_subquery().label("security_oldest"),
|
||
select(func.count()).select_from(PermissionMetricSnapshot).scalar_subquery().label("metric_count"),
|
||
)
|
||
)
|
||
storage_row = storage_result.one()
|
||
|
||
deductions: dict[str, int] = {
|
||
"performance": 0,
|
||
"deny_rate": 0,
|
||
"cache": 0,
|
||
"alerts": 0,
|
||
"runtime": 0,
|
||
}
|
||
issues: list[str] = []
|
||
|
||
if avg_ms > 200:
|
||
deductions["performance"] = 25
|
||
elif avg_ms > 50:
|
||
deductions["performance"] = 20
|
||
elif avg_ms > 10:
|
||
deductions["performance"] = 10
|
||
if deductions["performance"]:
|
||
issues.append(f"权限检查平均响应时间过长（{avg_ms:.2f}ms）")
|
||
|
||
if deny_rate > 50:
|
||
deductions["deny_rate"] = 20
|
||
elif deny_rate > 20:
|
||
deductions["deny_rate"] = 10
|
||
if deductions["deny_rate"]:
|
||
issues.append(f"权限拒绝率偏高（{deny_rate:.2f}%）")
|
||
|
||
cache_sample_sufficient = (
|
||
cache_metrics["total_accesses"] >= CACHE_HIT_RATE_HEALTH_MIN_ACCESSES
|
||
)
|
||
if cache_sample_sufficient:
|
||
if cache_metrics["hit_rate"] < 20:
|
||
deductions["cache"] = 25
|
||
elif cache_metrics["hit_rate"] < 50:
|
||
deductions["cache"] = 20
|
||
elif cache_metrics["hit_rate"] < 80:
|
||
deductions["cache"] = 10
|
||
if deductions["cache"]:
|
||
issues.append(f"近 1 小时缓存命中率偏低（{cache_metrics['hit_rate']:.2f}%）")
|
||
|
||
error_alert_count = sum(alert["level"] == "error" for alert in recent_alerts)
|
||
warning_alert_count = sum(alert["level"] == "warning" for alert in recent_alerts)
|
||
if error_alert_count:
|
||
deductions["alerts"] = 10
|
||
issues.append(f"近 1 小时发生 {error_alert_count} 条错误告警")
|
||
elif warning_alert_count:
|
||
deductions["alerts"] = 5
|
||
issues.append(f"近 1 小时发生 {warning_alert_count} 条警告")
|
||
|
||
runtime_components = {
|
||
"database": {"status": "healthy", "latency_ms": database_latency_ms},
|
||
"permission_log_writer": permission_writer_component,
|
||
"security_log_writer": security_writer_component,
|
||
"retention": retention_component,
|
||
}
|
||
component_labels = {
|
||
"database": "数据库",
|
||
"permission_log_writer": "权限日志写入器",
|
||
"security_log_writer": "安全日志写入器",
|
||
"retention": "留存清理任务",
|
||
}
|
||
unhealthy_components = [
|
||
component_labels[name]
|
||
for name, component in runtime_components.items()
|
||
if component["status"] == "unhealthy"
|
||
]
|
||
degraded_components = [
|
||
component_labels[name]
|
||
for name, component in runtime_components.items()
|
||
if component["status"] == "degraded"
|
||
]
|
||
if unhealthy_components:
|
||
deductions["runtime"] = 20
|
||
issues.append(f"运行组件异常：{', '.join(unhealthy_components)}")
|
||
elif degraded_components:
|
||
deductions["runtime"] = 10
|
||
issues.append(f"运行组件降级：{', '.join(degraded_components)}")
|
||
elif database_latency_ms > 500:
|
||
deductions["runtime"] = 5
|
||
issues.append(f"数据库健康检查延迟偏高（{database_latency_ms:.2f}ms）")
|
||
|
||
health_score = max(0, 100 - sum(deductions.values()))
|
||
sample_sufficient = total >= 10
|
||
|
||
return {
|
||
"status": "healthy" if health_score >= 80 else "degraded" if health_score >= 50 else "unhealthy",
|
||
"health_score": health_score,
|
||
"issues": issues,
|
||
"sample_sufficient": sample_sufficient,
|
||
"score_breakdown": {
|
||
"performance": {"weight": 25, "deduction": deductions["performance"]},
|
||
"deny_rate": {"weight": 20, "deduction": deductions["deny_rate"]},
|
||
"cache": {
|
||
"weight": 25,
|
||
"deduction": deductions["cache"],
|
||
"sample_sufficient": cache_sample_sufficient,
|
||
"scope": "sliding_window",
|
||
"window_seconds": CACHE_METRICS_WINDOW_SECONDS,
|
||
},
|
||
"alerts": {"weight": 10, "deduction": deductions["alerts"]},
|
||
"runtime": {"weight": 20, "deduction": deductions["runtime"]},
|
||
},
|
||
"last_hour": {
|
||
"total_checks": total,
|
||
"denied_checks": row.denied or 0,
|
||
"deny_rate": deny_rate,
|
||
"avg_elapsed_ms": round(avg_ms, 2),
|
||
"error_alerts": error_alert_count,
|
||
"warning_alerts": warning_alert_count,
|
||
},
|
||
"cache_stats": monitor.get_cache_stats(),
|
||
"components": runtime_components,
|
||
"storage": {
|
||
"permission_access_logs": storage_row.permission_count or 0,
|
||
"security_access_logs": storage_row.security_count or 0,
|
||
"permission_metric_snapshots": storage_row.metric_count or 0,
|
||
"oldest_permission_access_log_at": (
|
||
storage_row.permission_oldest.isoformat() if storage_row.permission_oldest else None
|
||
),
|
||
"oldest_security_access_log_at": (
|
||
storage_row.security_oldest.isoformat() if storage_row.security_oldest else None
|
||
),
|
||
},
|
||
}
|
||
|
||
|
||
# ═══════════════════════════════════════════
|
||
# 新增端点：访问日志
|
||
# ═══════════════════════════════════════════
|
||
|
||
@router.get("/access-logs", status_code=status.HTTP_200_OK)
|
||
async def get_access_logs(
|
||
db: AsyncSession = Depends(get_db_session),
|
||
_=Depends(get_current_user),
|
||
study_id: Optional[uuid.UUID] = Query(None),
|
||
user_id: Optional[uuid.UUID] = Query(None),
|
||
endpoint_key: Optional[str] = Query(None),
|
||
role: Optional[str] = Query(None),
|
||
allowed: Optional[bool] = Query(None),
|
||
start_time: Optional[datetime] = Query(None),
|
||
end_time: Optional[datetime] = Query(None),
|
||
client_ip: Optional[str] = Query(None),
|
||
client_type: Optional[str] = Query(None),
|
||
event_type: str = Query("all", pattern="^(all|permission)$"),
|
||
keyword: Optional[str] = Query(None),
|
||
page: int = Query(1, ge=1),
|
||
page_size: int = Query(50, ge=1, le=200),
|
||
) -> dict:
|
||
"""分页查询权限访问日志"""
|
||
scope = await resolve_monitoring_scope(db, _)
|
||
if not scope.is_admin and not scope.study_ids:
|
||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足")
|
||
if study_id and not scope.can_access_study(study_id):
|
||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足")
|
||
client_ip = client_ip if isinstance(client_ip, str) else None
|
||
client_type = client_type if isinstance(client_type, str) else None
|
||
keyword = keyword if isinstance(keyword, str) else None
|
||
start_time = start_time if isinstance(start_time, datetime) else None
|
||
end_time = end_time if isinstance(end_time, datetime) else None
|
||
conditions = []
|
||
if study_id:
|
||
conditions.append(PermissionAccessLog.study_id == study_id)
|
||
elif not scope.is_admin:
|
||
conditions.append(PermissionAccessLog.study_id.in_(scope.study_ids))
|
||
if user_id:
|
||
conditions.append(PermissionAccessLog.user_id == user_id)
|
||
if endpoint_key:
|
||
conditions.append(PermissionAccessLog.endpoint_key == endpoint_key)
|
||
if role:
|
||
conditions.append(PermissionAccessLog.role == role)
|
||
if allowed is not None:
|
||
conditions.append(PermissionAccessLog.allowed == allowed)
|
||
if start_time:
|
||
conditions.append(PermissionAccessLog.created_at >= start_time)
|
||
if end_time:
|
||
conditions.append(PermissionAccessLog.created_at <= end_time)
|
||
if client_ip and client_ip.strip():
|
||
conditions.append(PermissionAccessLog.ip_address.ilike(f"%{client_ip.strip()}%"))
|
||
if client_type and client_type.strip():
|
||
conditions.append(_permission_client_type_condition(client_type))
|
||
if keyword and keyword.strip():
|
||
token = keyword.strip()
|
||
like_token = f"%{token}%"
|
||
location_ips = await _resolve_permission_location_keyword_ips(db, conditions, token)
|
||
keyword_conditions = [
|
||
PermissionAccessLog.endpoint_key.ilike(like_token),
|
||
PermissionAccessLog.role.ilike(like_token),
|
||
PermissionAccessLog.ip_address.ilike(like_token),
|
||
PermissionAccessLog.client_type.ilike(like_token),
|
||
PermissionAccessLog.client_version.ilike(like_token),
|
||
PermissionAccessLog.client_platform.ilike(like_token),
|
||
PermissionAccessLog.user_agent.ilike(like_token),
|
||
cast(PermissionAccessLog.request_headers, String).ilike(like_token),
|
||
cast(PermissionAccessLog.request_snapshot, String).ilike(like_token),
|
||
User.full_name.ilike(like_token),
|
||
User.email.ilike(like_token),
|
||
cast(PermissionAccessLog.user_id, String).ilike(like_token),
|
||
]
|
||
if location_ips:
|
||
keyword_conditions.append(PermissionAccessLog.ip_address.in_(location_ips))
|
||
conditions.append(or_(*keyword_conditions))
|
||
|
||
include_security_logs = (
|
||
event_type != "permission"
|
||
and scope.is_admin
|
||
and study_id is None
|
||
and role is None
|
||
and endpoint_key is None
|
||
and allowed is not True
|
||
)
|
||
event_conditions = list(conditions)
|
||
if include_security_logs:
|
||
matching_security_event = select(SecurityAccessLog.id).where(
|
||
PermissionAccessLog.request_id.is_not(None),
|
||
SecurityAccessLog.request_id == PermissionAccessLog.request_id,
|
||
SecurityAccessLog.status_code >= 400,
|
||
).exists()
|
||
event_conditions.append(~matching_security_event)
|
||
|
||
permission_event_query = (
|
||
select(
|
||
literal("permission").label("event_type"),
|
||
cast(PermissionAccessLog.id, String).label("event_id"),
|
||
PermissionAccessLog.created_at.label("created_at"),
|
||
cast(PermissionAccessLog.user_id, String).label("user_id"),
|
||
PermissionAccessLog.ip_address.label("ip_address"),
|
||
PermissionAccessLog.allowed.label("allowed"),
|
||
PermissionAccessLog.elapsed_ms.label("elapsed_ms"),
|
||
)
|
||
.outerjoin(User, PermissionAccessLog.user_id == User.id)
|
||
)
|
||
user_stats_query = (
|
||
select(
|
||
PermissionAccessLog.ip_address.label("sample_ip_address"),
|
||
func.count().label("total_count"),
|
||
func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied_count"),
|
||
func.count(func.distinct(PermissionAccessLog.ip_address)).label("unique_ip_count"),
|
||
func.max(PermissionAccessLog.created_at).label("last_seen_at"),
|
||
)
|
||
.outerjoin(User, PermissionAccessLog.user_id == User.id)
|
||
.group_by(PermissionAccessLog.ip_address)
|
||
.order_by(desc("total_count"), desc("last_seen_at"))
|
||
.limit(10)
|
||
)
|
||
for condition in event_conditions:
|
||
permission_event_query = permission_event_query.where(condition)
|
||
for condition in conditions:
|
||
user_stats_query = user_stats_query.where(condition)
|
||
|
||
user_stats_result = await db.execute(user_stats_query)
|
||
user_stats_rows = user_stats_result.all()
|
||
latest_user_by_ip = {}
|
||
if user_stats_rows:
|
||
ranked_ips = [stat.sample_ip_address for stat in user_stats_rows if stat.sample_ip_address]
|
||
latest_user_query = (
|
||
select(
|
||
PermissionAccessLog.ip_address,
|
||
PermissionAccessLog.user_id,
|
||
User.full_name,
|
||
User.email,
|
||
PermissionAccessLog.role,
|
||
PermissionAccessLog.created_at,
|
||
)
|
||
.outerjoin(User, PermissionAccessLog.user_id == User.id)
|
||
.where(PermissionAccessLog.ip_address.in_(ranked_ips))
|
||
.order_by(PermissionAccessLog.ip_address, desc(PermissionAccessLog.created_at))
|
||
)
|
||
for condition in conditions:
|
||
latest_user_query = latest_user_query.where(condition)
|
||
latest_user_result = await db.execute(latest_user_query)
|
||
for row in latest_user_result.all():
|
||
latest_user_by_ip.setdefault(row.ip_address, row)
|
||
|
||
event_queries = [permission_event_query]
|
||
if include_security_logs:
|
||
security_conditions = [SecurityAccessLog.status_code >= 400]
|
||
if user_id:
|
||
security_conditions.append(SecurityAccessLog.user_identifier == str(user_id))
|
||
if start_time:
|
||
security_conditions.append(SecurityAccessLog.created_at >= start_time)
|
||
if end_time:
|
||
security_conditions.append(SecurityAccessLog.created_at <= end_time)
|
||
if client_ip and client_ip.strip():
|
||
security_conditions.append(SecurityAccessLog.client_ip.ilike(f"%{client_ip.strip()}%"))
|
||
if client_type and client_type.strip():
|
||
security_conditions.append(_security_client_type_condition(client_type))
|
||
if keyword and keyword.strip():
|
||
token = keyword.strip()
|
||
like_token = f"%{token}%"
|
||
location_ips = await _resolve_security_location_keyword_ips(db, security_conditions, token)
|
||
matching_users_result = await db.execute(
|
||
select(cast(User.id, String)).where(
|
||
or_(User.full_name.ilike(like_token), User.email.ilike(like_token))
|
||
)
|
||
)
|
||
matching_user_ids = list(matching_users_result.scalars().all())
|
||
keyword_conditions = [
|
||
SecurityAccessLog.method.ilike(like_token),
|
||
SecurityAccessLog.path.ilike(like_token),
|
||
SecurityAccessLog.client_ip.ilike(like_token),
|
||
SecurityAccessLog.client_type.ilike(like_token),
|
||
SecurityAccessLog.client_version.ilike(like_token),
|
||
SecurityAccessLog.client_platform.ilike(like_token),
|
||
SecurityAccessLog.user_agent.ilike(like_token),
|
||
SecurityAccessLog.auth_status.ilike(like_token),
|
||
SecurityAccessLog.user_identifier.ilike(like_token),
|
||
cast(SecurityAccessLog.status_code, String).ilike(like_token),
|
||
cast(SecurityAccessLog.request_headers, String).ilike(like_token),
|
||
cast(SecurityAccessLog.request_snapshot, String).ilike(like_token),
|
||
]
|
||
if location_ips:
|
||
keyword_conditions.append(SecurityAccessLog.client_ip.in_(location_ips))
|
||
if matching_user_ids:
|
||
keyword_conditions.append(SecurityAccessLog.user_identifier.in_(matching_user_ids))
|
||
security_conditions.append(or_(*keyword_conditions))
|
||
|
||
security_event_query = select(
|
||
literal("security").label("event_type"),
|
||
cast(SecurityAccessLog.id, String).label("event_id"),
|
||
SecurityAccessLog.created_at.label("created_at"),
|
||
SecurityAccessLog.user_identifier.label("user_id"),
|
||
SecurityAccessLog.client_ip.label("ip_address"),
|
||
(SecurityAccessLog.status_code < 400).label("allowed"),
|
||
SecurityAccessLog.elapsed_ms.label("elapsed_ms"),
|
||
)
|
||
for condition in security_conditions:
|
||
security_event_query = security_event_query.where(condition)
|
||
event_queries.append(security_event_query)
|
||
|
||
events = (
|
||
union_all(*event_queries).subquery("monitoring_events")
|
||
if len(event_queries) > 1
|
||
else event_queries[0].subquery("monitoring_events")
|
||
)
|
||
summary_result = await db.execute(
|
||
select(
|
||
func.count().label("total_count"),
|
||
func.count(func.distinct(events.c.user_id)).label("unique_user_count"),
|
||
func.count(func.distinct(events.c.ip_address)).label("unique_ip_count"),
|
||
func.count().filter(events.c.allowed.is_(False)).label("denied_count"),
|
||
func.coalesce(func.avg(events.c.elapsed_ms), 0).label("avg_elapsed_ms"),
|
||
).select_from(events)
|
||
)
|
||
summary_row = summary_result.one()
|
||
total = summary_row.total_count or 0
|
||
|
||
page_result = await db.execute(
|
||
select(events.c.event_type, events.c.event_id, events.c.created_at)
|
||
.order_by(desc(events.c.created_at), desc(events.c.event_id))
|
||
.offset((page - 1) * page_size)
|
||
.limit(page_size)
|
||
)
|
||
page_events = page_result.all()
|
||
permission_ids = [uuid.UUID(row.event_id) for row in page_events if row.event_type == "permission"]
|
||
security_ids = [uuid.UUID(row.event_id) for row in page_events if row.event_type == "security"]
|
||
|
||
items_by_key: dict[tuple[str, str], dict] = {}
|
||
if permission_ids:
|
||
permission_result = await db.execute(
|
||
select(PermissionAccessLog, User.full_name, User.email)
|
||
.outerjoin(User, PermissionAccessLog.user_id == User.id)
|
||
.where(PermissionAccessLog.id.in_(permission_ids))
|
||
)
|
||
for log, full_name, email in permission_result.all():
|
||
items_by_key[("permission", str(log.id))] = _permission_access_item(log, full_name, email)
|
||
if security_ids:
|
||
security_result = await db.execute(
|
||
select(SecurityAccessLog).where(SecurityAccessLog.id.in_(security_ids))
|
||
)
|
||
security_logs = list(security_result.scalars().all())
|
||
security_user_names = await _security_user_names(db, security_logs)
|
||
for log in security_logs:
|
||
items_by_key[("security", str(log.id))] = _security_access_item(log, security_user_names)
|
||
|
||
page_items = [
|
||
items_by_key[(row.event_type, row.event_id)]
|
||
for row in page_events
|
||
if (row.event_type, row.event_id) in items_by_key
|
||
]
|
||
summary = {
|
||
"total_count": total,
|
||
"unique_user_count": summary_row.unique_user_count or 0,
|
||
"unique_ip_count": summary_row.unique_ip_count or 0,
|
||
"denied_count": summary_row.denied_count or 0,
|
||
"avg_elapsed_ms": round(float(summary_row.avg_elapsed_ms), 2),
|
||
}
|
||
|
||
user_stats = []
|
||
for stat in user_stats_rows:
|
||
sample_location = resolve_ip_location(stat.sample_ip_address)
|
||
latest_user = latest_user_by_ip.get(stat.sample_ip_address)
|
||
user_stats.append(
|
||
{
|
||
"user_id": str(latest_user.user_id) if latest_user else "",
|
||
"user_name": latest_user.full_name if latest_user and latest_user.full_name else "未知用户",
|
||
"user_email": latest_user.email if latest_user else None,
|
||
"role": latest_user.role if latest_user else "",
|
||
"total_count": stat.total_count,
|
||
"denied_count": stat.denied_count,
|
||
"unique_ip_count": stat.unique_ip_count,
|
||
"sample_ip_address": stat.sample_ip_address,
|
||
"last_seen_at": stat.last_seen_at.isoformat() if stat.last_seen_at else None,
|
||
"primary_location": sample_location.location,
|
||
}
|
||
)
|
||
|
||
return {
|
||
"total": total,
|
||
"page": page,
|
||
"page_size": page_size,
|
||
"summary": summary,
|
||
"user_stats": user_stats,
|
||
"items": page_items,
|
||
}
|
||
|
||
|
||
def _security_account_label(auth_status: str, user_identifier: str | None, user_names: dict[str, str] | None = None) -> str:
|
||
if auth_status == "AUTHENTICATED" and user_identifier:
|
||
return (user_names or {}).get(user_identifier) or user_identifier
|
||
if auth_status == "INVALID_TOKEN":
|
||
return "无效令牌"
|
||
return "未知账号"
|
||
|
||
|
||
def _classify_security_access_log(log: SecurityAccessLog, ip_location: IpLocation | None = None) -> dict[str, str]:
|
||
if getattr(log, "category", None) and getattr(log, "severity", None):
|
||
return {"category": log.category, "severity": log.severity}
|
||
return classify_security_event(
|
||
path=log.path,
|
||
status_code=log.status_code,
|
||
auth_status=log.auth_status,
|
||
ip_location=ip_location,
|
||
)
|
||
|
||
|
||
def _security_endpoint_type(path: str) -> str:
|
||
normalized_path = (path or "").lower()
|
||
if "/auth" in normalized_path or "/login" in normalized_path:
|
||
return "认证接口"
|
||
if normalized_path.startswith("/api/"):
|
||
return "业务 API"
|
||
if normalized_path.endswith((".js", ".css", ".png", ".jpg", ".jpeg", ".svg", ".ico", ".map")):
|
||
return "静态资源"
|
||
if any(marker in normalized_path for marker in (".env", "wp-", "php", "admin", "shell", "config")):
|
||
return "探测路径"
|
||
return "其他请求"
|
||
|
||
|
||
@router.get("/security-logs", status_code=status.HTTP_200_OK)
|
||
async def get_security_access_logs(
|
||
db: AsyncSession = Depends(get_db_session),
|
||
_=Depends(get_current_user),
|
||
status_min: Optional[int] = Query(None, ge=100, le=599),
|
||
auth_status: Optional[str] = Query(None),
|
||
client_type: Optional[str] = Query(None),
|
||
client_version: Optional[str] = Query(None),
|
||
client_ip: Optional[str] = Query(None),
|
||
category: Optional[str] = Query(None),
|
||
severity: Optional[str] = Query(None),
|
||
keyword: Optional[str] = Query(None),
|
||
events_only: bool = Query(False),
|
||
start_time: Optional[datetime] = Query(None),
|
||
end_time: Optional[datetime] = Query(None),
|
||
page: int = Query(1, ge=1),
|
||
page_size: int = Query(50, ge=1, le=200),
|
||
) -> dict:
|
||
"""查询底层安全访问日志，覆盖匿名、无效令牌和异常状态请求。"""
|
||
scope = await resolve_monitoring_scope(db, _)
|
||
if not scope.is_admin:
|
||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足")
|
||
client_type = client_type if isinstance(client_type, str) else None
|
||
client_version = client_version if isinstance(client_version, str) else None
|
||
client_ip = client_ip if isinstance(client_ip, str) else None
|
||
category = category if isinstance(category, str) else None
|
||
severity = severity if isinstance(severity, str) else None
|
||
keyword = keyword if isinstance(keyword, str) else None
|
||
events_only = events_only if isinstance(events_only, bool) else False
|
||
start_time = start_time if isinstance(start_time, datetime) else None
|
||
end_time = end_time if isinstance(end_time, datetime) else None
|
||
conditions = []
|
||
if status_min is not None:
|
||
conditions.append(SecurityAccessLog.status_code >= status_min)
|
||
if auth_status:
|
||
conditions.append(SecurityAccessLog.auth_status == auth_status)
|
||
if client_type:
|
||
conditions.append(SecurityAccessLog.client_type == client_type)
|
||
if client_version:
|
||
conditions.append(SecurityAccessLog.client_version == client_version)
|
||
if client_ip:
|
||
conditions.append(SecurityAccessLog.client_ip.ilike(f"%{client_ip.strip()}%"))
|
||
if category:
|
||
conditions.append(SecurityAccessLog.category == category.strip().upper())
|
||
if severity:
|
||
conditions.append(SecurityAccessLog.severity == severity.strip().upper())
|
||
if events_only:
|
||
conditions.append(SecurityAccessLog.category != "OTHER")
|
||
if start_time:
|
||
conditions.append(SecurityAccessLog.created_at >= start_time)
|
||
if end_time:
|
||
conditions.append(SecurityAccessLog.created_at <= end_time)
|
||
if keyword and keyword.strip():
|
||
token = keyword.strip()
|
||
like_token = f"%{token}%"
|
||
location_ips = await _resolve_security_location_keyword_ips(db, conditions, token)
|
||
matching_users_result = await db.execute(
|
||
select(cast(User.id, String)).where(
|
||
or_(User.full_name.ilike(like_token), User.email.ilike(like_token))
|
||
)
|
||
)
|
||
matching_user_ids = list(matching_users_result.scalars().all())
|
||
keyword_conditions = [
|
||
SecurityAccessLog.method.ilike(like_token),
|
||
SecurityAccessLog.path.ilike(like_token),
|
||
SecurityAccessLog.client_ip.ilike(like_token),
|
||
SecurityAccessLog.client_type.ilike(like_token),
|
||
SecurityAccessLog.client_version.ilike(like_token),
|
||
SecurityAccessLog.client_platform.ilike(like_token),
|
||
SecurityAccessLog.user_agent.ilike(like_token),
|
||
SecurityAccessLog.auth_status.ilike(like_token),
|
||
SecurityAccessLog.user_identifier.ilike(like_token),
|
||
SecurityAccessLog.category.ilike(like_token),
|
||
SecurityAccessLog.severity.ilike(like_token),
|
||
cast(SecurityAccessLog.status_code, String).ilike(like_token),
|
||
]
|
||
if location_ips:
|
||
keyword_conditions.append(SecurityAccessLog.client_ip.in_(location_ips))
|
||
if matching_user_ids:
|
||
keyword_conditions.append(SecurityAccessLog.user_identifier.in_(matching_user_ids))
|
||
conditions.append(or_(*keyword_conditions))
|
||
|
||
query = select(SecurityAccessLog)
|
||
count_query = select(func.count()).select_from(SecurityAccessLog)
|
||
summary_query = select(
|
||
func.count().label("total_count"),
|
||
func.count().filter(SecurityAccessLog.auth_status == "ANONYMOUS").label("anonymous_count"),
|
||
func.count().filter(SecurityAccessLog.auth_status == "INVALID_TOKEN").label("invalid_token_count"),
|
||
func.count().filter(SecurityAccessLog.status_code >= 400).label("error_count"),
|
||
func.count(func.distinct(SecurityAccessLog.client_ip)).label("unique_ip_count"),
|
||
func.count().filter(SecurityAccessLog.severity.in_(("CRITICAL", "HIGH"))).label("high_risk_count"),
|
||
func.count().filter(SecurityAccessLog.category == "SERVER_ERROR").label("server_error_count"),
|
||
).select_from(SecurityAccessLog)
|
||
for condition in conditions:
|
||
query = query.where(condition)
|
||
count_query = count_query.where(condition)
|
||
summary_query = summary_query.where(condition)
|
||
|
||
total_result = await db.execute(count_query)
|
||
summary_result = await db.execute(summary_query)
|
||
result = await db.execute(
|
||
query.order_by(desc(SecurityAccessLog.created_at))
|
||
.offset((page - 1) * page_size)
|
||
.limit(page_size)
|
||
)
|
||
summary_row = summary_result.one()
|
||
category_query = select(SecurityAccessLog.category, func.count().label("count")).group_by(SecurityAccessLog.category)
|
||
severity_query = select(SecurityAccessLog.severity, func.count().label("count")).group_by(SecurityAccessLog.severity)
|
||
top_ip_query = (
|
||
select(
|
||
SecurityAccessLog.client_ip,
|
||
func.count().label("total_count"),
|
||
func.count().filter(SecurityAccessLog.severity.in_(("CRITICAL", "HIGH"))).label("high_risk_count"),
|
||
func.max(SecurityAccessLog.created_at).label("last_seen_at"),
|
||
func.max(SecurityAccessLog.category).label("category"),
|
||
)
|
||
.where(SecurityAccessLog.client_ip.is_not(None))
|
||
.group_by(SecurityAccessLog.client_ip)
|
||
.order_by(desc("high_risk_count"), desc("total_count"), desc("last_seen_at"))
|
||
.limit(10)
|
||
)
|
||
endpoint_query = (
|
||
select(SecurityAccessLog.path, func.count().label("count"))
|
||
.group_by(SecurityAccessLog.path)
|
||
.order_by(desc("count"))
|
||
.limit(100)
|
||
)
|
||
for condition in conditions:
|
||
category_query = category_query.where(condition)
|
||
severity_query = severity_query.where(condition)
|
||
top_ip_query = top_ip_query.where(condition)
|
||
endpoint_query = endpoint_query.where(condition)
|
||
category_result = await db.execute(category_query)
|
||
severity_result = await db.execute(severity_query)
|
||
top_ip_result = await db.execute(top_ip_query)
|
||
endpoint_result = await db.execute(endpoint_query)
|
||
endpoint_type_counts: dict[str, dict[str, str | int]] = {}
|
||
for row in endpoint_result.all():
|
||
endpoint_type = _security_endpoint_type(row.path)
|
||
current = endpoint_type_counts.setdefault(endpoint_type, {"count": 0, "sample_path": row.path})
|
||
current["count"] = int(current["count"]) + row.count
|
||
|
||
logs = result.scalars().all()
|
||
user_names = await _security_user_names(db, list(logs))
|
||
|
||
items = []
|
||
for log in logs:
|
||
ip_location = resolve_ip_location(log.client_ip)
|
||
items.append(
|
||
{
|
||
"id": str(log.id),
|
||
"method": log.method,
|
||
"path": log.path,
|
||
"status_code": log.status_code,
|
||
"elapsed_ms": round(log.elapsed_ms, 2),
|
||
"client_ip": log.client_ip,
|
||
"ip_location": ip_location.location,
|
||
"ip_country": ip_location.country,
|
||
"ip_province": ip_location.province,
|
||
"ip_city": ip_location.city,
|
||
"ip_isp": ip_location.isp,
|
||
"user_agent": log.user_agent,
|
||
"client_type": log.client_type,
|
||
"client_version": log.client_version,
|
||
"client_platform": log.client_platform,
|
||
"build_channel": log.build_channel,
|
||
"build_commit": log.build_commit,
|
||
"request_headers": _coerce_request_headers(getattr(log, "request_headers", None)),
|
||
"request_snapshot": _coerce_request_snapshot(getattr(log, "request_snapshot", None)),
|
||
"request_id": getattr(log, "request_id", None),
|
||
"auth_status": log.auth_status,
|
||
"user_identifier": log.user_identifier,
|
||
"account_label": _security_account_label(log.auth_status, log.user_identifier, user_names),
|
||
**_classify_security_access_log(log, ip_location),
|
||
"created_at": log.created_at.isoformat(),
|
||
}
|
||
)
|
||
|
||
return {
|
||
"total": total_result.scalar() or 0,
|
||
"page": page,
|
||
"page_size": page_size,
|
||
"summary": {
|
||
"total_count": summary_row.total_count,
|
||
"anonymous_count": summary_row.anonymous_count,
|
||
"invalid_token_count": summary_row.invalid_token_count,
|
||
"error_count": summary_row.error_count,
|
||
"unique_ip_count": summary_row.unique_ip_count,
|
||
"high_risk_count": summary_row.high_risk_count,
|
||
"server_error_count": summary_row.server_error_count,
|
||
"category_counts": {str(row.category or "OTHER"): row.count for row in category_result.all()},
|
||
"severity_counts": {str(row.severity or "LOW"): row.count for row in severity_result.all()},
|
||
"endpoint_type_counts": endpoint_type_counts,
|
||
"top_ips": [
|
||
{
|
||
"ip": row.client_ip,
|
||
"location": resolve_ip_location(row.client_ip).location,
|
||
"category": row.category or "OTHER",
|
||
"total_count": row.total_count,
|
||
"high_risk_count": row.high_risk_count,
|
||
"last_seen_at": row.last_seen_at.isoformat() if row.last_seen_at else None,
|
||
}
|
||
for row in top_ip_result.all()
|
||
],
|
||
},
|
||
"items": items,
|
||
}
|
||
|
||
|
||
# ═══════════════════════════════════════════
|
||
# 新增端点：趋势数据
|
||
# ═══════════════════════════════════════════
|
||
|
||
@router.get("/trends", status_code=status.HTTP_200_OK)
|
||
async def get_trends(
|
||
db: AsyncSession = Depends(get_db_session),
|
||
_=Depends(get_current_user),
|
||
period: str = Query("24h", pattern="^(24h|7d|30d)$"),
|
||
) -> dict:
|
||
"""获取权限检查趋势、周期摘要和归因数据。"""
|
||
scope = await resolve_monitoring_scope(db, _)
|
||
if not scope.is_admin and not scope.study_ids:
|
||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足")
|
||
now = datetime.now(timezone.utc)
|
||
bucket_delta, bucket_count = _trend_bucket_config(period)
|
||
current_bucket = _trend_bucket_time(now, period)
|
||
start_time = current_bucket - bucket_delta * (bucket_count - 1)
|
||
current_duration = now - start_time
|
||
previous_end = start_time
|
||
previous_start = previous_end - current_duration
|
||
|
||
log_buckets = await _load_trend_log_buckets(db, scope, start_time, now, period)
|
||
cache_buckets, current_cache, previous_cache = await _load_trend_cache_buckets(
|
||
db,
|
||
scope,
|
||
previous_start,
|
||
previous_end,
|
||
start_time,
|
||
now,
|
||
period,
|
||
)
|
||
summary = await _load_trend_summary(db, scope, start_time, now)
|
||
previous_summary = await _load_trend_summary(db, scope, previous_start, previous_end)
|
||
summary.update(current_cache)
|
||
previous_summary.update(previous_cache)
|
||
previous_summary.pop("observed_end_at", None)
|
||
|
||
data_points = []
|
||
for index in range(bucket_count):
|
||
bucket_time = start_time + bucket_delta * index
|
||
values = log_buckets.get(bucket_time, _empty_trend_bucket())
|
||
cache_values = cache_buckets.get(bucket_time, {"cache_hits": 0, "cache_misses": 0})
|
||
cache_total = int(cache_values["cache_hits"]) + int(cache_values["cache_misses"])
|
||
total = int(values["total"])
|
||
data_points.append(
|
||
{
|
||
"bucket_time": bucket_time.isoformat(),
|
||
"sample_state": "partial" if bucket_time == current_bucket else "complete",
|
||
"total_checks": total,
|
||
"allowed_checks": int(values["allowed"]),
|
||
"denied_checks": int(values["denied"]),
|
||
"avg_elapsed_ms": round(float(values["elapsed_sum"]) / total, 2) if total else 0,
|
||
"max_elapsed_ms": round(float(values["max_ms"]), 2),
|
||
"cache_hits": int(cache_values["cache_hits"]),
|
||
"cache_misses": int(cache_values["cache_misses"]),
|
||
"cache_hit_rate": round(int(cache_values["cache_hits"]) / cache_total * 100, 1) if cache_total else 0,
|
||
"cache_sample_available": cache_total > 0,
|
||
"error_count": 0,
|
||
}
|
||
)
|
||
|
||
return {
|
||
"period": period,
|
||
"generated_at": now.isoformat(),
|
||
"range": {
|
||
"start_at": start_time.isoformat(),
|
||
"end_at": now.isoformat(),
|
||
"previous_start_at": previous_start.isoformat(),
|
||
"previous_end_at": previous_end.isoformat(),
|
||
"bucket_seconds": int(bucket_delta.total_seconds()),
|
||
"bucket_count": bucket_count,
|
||
"timezone": "UTC",
|
||
"observed_end_at": summary.pop("observed_end_at"),
|
||
},
|
||
"summary": summary,
|
||
"previous_summary": previous_summary,
|
||
"attribution": await _load_trend_attribution(db, scope, start_time, now),
|
||
"data_points": data_points,
|
||
}
|
||
|
||
|
||
def _trend_bucket_config(period: str) -> tuple[timedelta, int]:
|
||
return {
|
||
"24h": (timedelta(hours=1), 24),
|
||
"7d": (timedelta(hours=6), 28),
|
||
"30d": (timedelta(days=1), 30),
|
||
}[period]
|
||
|
||
|
||
def _empty_trend_bucket() -> dict[str, float | int]:
|
||
return {"total": 0, "allowed": 0, "denied": 0, "elapsed_sum": 0.0, "max_ms": 0.0}
|
||
|
||
|
||
def _merge_trend_bucket(target: dict[str, float | int], *, total: int, allowed: int, denied: int, elapsed_sum: float, max_ms: float) -> None:
|
||
target["total"] = int(target["total"]) + total
|
||
target["allowed"] = int(target["allowed"]) + allowed
|
||
target["denied"] = int(target["denied"]) + denied
|
||
target["elapsed_sum"] = float(target["elapsed_sum"]) + elapsed_sum
|
||
target["max_ms"] = max(float(target["max_ms"]), max_ms)
|
||
|
||
|
||
async def _load_trend_log_buckets(
|
||
db: AsyncSession,
|
||
scope: MonitoringScope,
|
||
start_time: datetime,
|
||
end_time: datetime,
|
||
period: str,
|
||
) -> dict[datetime, dict[str, float | int]]:
|
||
"""PostgreSQL 先按小时聚合，SQLite 测试库使用 Python 分桶。"""
|
||
buckets: dict[datetime, dict[str, float | int]] = defaultdict(_empty_trend_bucket)
|
||
dialect_name = db.get_bind().dialect.name
|
||
if dialect_name == "postgresql":
|
||
hour_bucket = func.date_trunc("hour", PermissionAccessLog.created_at).label("hour_bucket")
|
||
query = select(
|
||
hour_bucket,
|
||
func.count().label("total"),
|
||
func.count().filter(PermissionAccessLog.allowed.is_(True)).label("allowed"),
|
||
func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied"),
|
||
func.coalesce(func.sum(PermissionAccessLog.elapsed_ms), 0).label("elapsed_sum"),
|
||
func.coalesce(func.max(PermissionAccessLog.elapsed_ms), 0).label("max_ms"),
|
||
).where(
|
||
PermissionAccessLog.created_at >= start_time,
|
||
PermissionAccessLog.created_at <= end_time,
|
||
).group_by(hour_bucket)
|
||
result = await db.execute(_apply_monitoring_scope_to_log_query(query, scope))
|
||
for row in result.all():
|
||
bucket_time = _trend_bucket_time(row.hour_bucket, period)
|
||
_merge_trend_bucket(
|
||
buckets[bucket_time],
|
||
total=int(row.total or 0),
|
||
allowed=int(row.allowed or 0),
|
||
denied=int(row.denied or 0),
|
||
elapsed_sum=float(row.elapsed_sum or 0),
|
||
max_ms=float(row.max_ms or 0),
|
||
)
|
||
return buckets
|
||
|
||
query = select(
|
||
PermissionAccessLog.created_at,
|
||
PermissionAccessLog.allowed,
|
||
PermissionAccessLog.elapsed_ms,
|
||
).where(
|
||
PermissionAccessLog.created_at >= start_time,
|
||
PermissionAccessLog.created_at <= end_time,
|
||
)
|
||
result = await db.execute(_apply_monitoring_scope_to_log_query(query, scope))
|
||
for created_at, allowed, elapsed_ms in result.all():
|
||
bucket_time = _trend_bucket_time(created_at, period)
|
||
_merge_trend_bucket(
|
||
buckets[bucket_time],
|
||
total=1,
|
||
allowed=1 if allowed else 0,
|
||
denied=0 if allowed else 1,
|
||
elapsed_sum=float(elapsed_ms or 0),
|
||
max_ms=float(elapsed_ms or 0),
|
||
)
|
||
return buckets
|
||
|
||
|
||
def _percentile(values: list[float], percentile: float) -> float:
|
||
if not values:
|
||
return 0.0
|
||
ordered = sorted(values)
|
||
position = (len(ordered) - 1) * percentile
|
||
lower = int(position)
|
||
upper = min(lower + 1, len(ordered) - 1)
|
||
fraction = position - lower
|
||
return ordered[lower] + (ordered[upper] - ordered[lower]) * fraction
|
||
|
||
|
||
async def _load_trend_summary(
|
||
db: AsyncSession,
|
||
scope: MonitoringScope,
|
||
start_time: datetime,
|
||
end_time: datetime,
|
||
) -> dict:
|
||
query = select(
|
||
func.count().label("total"),
|
||
func.count().filter(PermissionAccessLog.allowed.is_(True)).label("allowed"),
|
||
func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied"),
|
||
func.count().filter(PermissionAccessLog.elapsed_ms > 50).label("slow"),
|
||
func.count(func.distinct(PermissionAccessLog.study_id)).label("studies"),
|
||
func.count(func.distinct(PermissionAccessLog.user_id)).label("users"),
|
||
func.count(func.distinct(PermissionAccessLog.endpoint_key)).label("endpoints"),
|
||
func.coalesce(func.avg(PermissionAccessLog.elapsed_ms), 0).label("avg_ms"),
|
||
func.coalesce(func.max(PermissionAccessLog.elapsed_ms), 0).label("max_ms"),
|
||
func.max(PermissionAccessLog.created_at).label("observed_end_at"),
|
||
).where(
|
||
PermissionAccessLog.created_at >= start_time,
|
||
PermissionAccessLog.created_at <= end_time,
|
||
)
|
||
row = (await db.execute(_apply_monitoring_scope_to_log_query(query, scope))).one()
|
||
if db.get_bind().dialect.name == "postgresql":
|
||
percentile_query = select(
|
||
func.percentile_cont(0.95).within_group(PermissionAccessLog.elapsed_ms)
|
||
).where(
|
||
PermissionAccessLog.created_at >= start_time,
|
||
PermissionAccessLog.created_at <= end_time,
|
||
)
|
||
percentile_result = await db.execute(
|
||
_apply_monitoring_scope_to_log_query(percentile_query, scope)
|
||
)
|
||
p95_ms = float(percentile_result.scalar() or 0)
|
||
else:
|
||
elapsed_query = select(PermissionAccessLog.elapsed_ms).where(
|
||
PermissionAccessLog.created_at >= start_time,
|
||
PermissionAccessLog.created_at <= end_time,
|
||
)
|
||
elapsed_result = await db.execute(
|
||
_apply_monitoring_scope_to_log_query(elapsed_query, scope)
|
||
)
|
||
p95_ms = _percentile(
|
||
[float(value) for value in elapsed_result.scalars().all()],
|
||
0.95,
|
||
)
|
||
total = int(row.total or 0)
|
||
denied = int(row.denied or 0)
|
||
return {
|
||
"total_checks": total,
|
||
"allowed_checks": int(row.allowed or 0),
|
||
"denied_checks": denied,
|
||
"deny_rate": round(denied / total * 100, 2) if total else 0,
|
||
"avg_elapsed_ms": round(float(row.avg_ms or 0), 2),
|
||
"p95_elapsed_ms": round(p95_ms, 2),
|
||
"max_elapsed_ms": round(float(row.max_ms or 0), 2),
|
||
"slow_check_count": int(row.slow or 0),
|
||
"active_study_count": int(row.studies or 0),
|
||
"active_user_count": int(row.users or 0),
|
||
"active_endpoint_count": int(row.endpoints or 0),
|
||
"cache_hits": 0,
|
||
"cache_misses": 0,
|
||
"cache_hit_rate": 0,
|
||
"cache_sample_available": False,
|
||
"observed_end_at": row.observed_end_at.isoformat() if row.observed_end_at else None,
|
||
}
|
||
|
||
|
||
async def _load_trend_cache_buckets(
|
||
db: AsyncSession,
|
||
scope: MonitoringScope,
|
||
previous_start: datetime,
|
||
previous_end: datetime,
|
||
current_start: datetime,
|
||
current_end: datetime,
|
||
period: str,
|
||
) -> tuple[dict[datetime, dict[str, int]], dict, dict]:
|
||
empty_summary = {"cache_hits": 0, "cache_misses": 0, "cache_hit_rate": 0, "cache_sample_available": False}
|
||
if not scope.is_admin:
|
||
return {}, dict(empty_summary), dict(empty_summary)
|
||
result = await db.execute(
|
||
select(PermissionMetricSnapshot).where(
|
||
PermissionMetricSnapshot.bucket_time >= previous_start,
|
||
PermissionMetricSnapshot.bucket_time <= current_end,
|
||
)
|
||
)
|
||
buckets: dict[datetime, dict[str, int]] = defaultdict(lambda: {"cache_hits": 0, "cache_misses": 0})
|
||
current_counts = {"cache_hits": 0, "cache_misses": 0}
|
||
previous_counts = {"cache_hits": 0, "cache_misses": 0}
|
||
for snapshot in result.scalars().all():
|
||
snapshot_time = snapshot.bucket_time
|
||
if snapshot_time.tzinfo is None:
|
||
snapshot_time = snapshot_time.replace(tzinfo=timezone.utc)
|
||
if snapshot_time >= current_start:
|
||
current_counts["cache_hits"] += int(snapshot.cache_hits or 0)
|
||
current_counts["cache_misses"] += int(snapshot.cache_misses or 0)
|
||
bucket = buckets[_trend_bucket_time(snapshot_time, period)]
|
||
bucket["cache_hits"] += int(snapshot.cache_hits or 0)
|
||
bucket["cache_misses"] += int(snapshot.cache_misses or 0)
|
||
elif snapshot_time < previous_end:
|
||
previous_counts["cache_hits"] += int(snapshot.cache_hits or 0)
|
||
previous_counts["cache_misses"] += int(snapshot.cache_misses or 0)
|
||
|
||
def serialize(counts: dict[str, int]) -> dict:
|
||
total = counts["cache_hits"] + counts["cache_misses"]
|
||
return {
|
||
**counts,
|
||
"cache_hit_rate": round(counts["cache_hits"] / total * 100, 2) if total else 0,
|
||
"cache_sample_available": total > 0,
|
||
}
|
||
|
||
return dict(buckets), serialize(current_counts), serialize(previous_counts)
|
||
|
||
|
||
async def _load_trend_attribution(
|
||
db: AsyncSession,
|
||
scope: MonitoringScope,
|
||
start_time: datetime,
|
||
end_time: datetime,
|
||
) -> dict:
|
||
base_conditions = (
|
||
PermissionAccessLog.created_at >= start_time,
|
||
PermissionAccessLog.created_at <= end_time,
|
||
)
|
||
denied_query = select(
|
||
PermissionAccessLog.endpoint_key,
|
||
PermissionAccessLog.role,
|
||
func.count().label("denied_count"),
|
||
).where(*base_conditions, PermissionAccessLog.allowed.is_(False)).group_by(
|
||
PermissionAccessLog.endpoint_key,
|
||
PermissionAccessLog.role,
|
||
).order_by(desc("denied_count")).limit(5)
|
||
denied_rows = (await db.execute(_apply_monitoring_scope_to_log_query(denied_query, scope))).all()
|
||
|
||
slow_query = select(
|
||
PermissionAccessLog.endpoint_key,
|
||
func.count().label("sample_count"),
|
||
func.count().filter(PermissionAccessLog.elapsed_ms > 50).label("slow_count"),
|
||
func.coalesce(func.avg(PermissionAccessLog.elapsed_ms), 0).label("avg_ms"),
|
||
func.coalesce(func.max(PermissionAccessLog.elapsed_ms), 0).label("max_ms"),
|
||
).where(*base_conditions).group_by(PermissionAccessLog.endpoint_key).order_by(
|
||
desc("slow_count"), desc("avg_ms"), desc("sample_count")
|
||
).limit(5)
|
||
slow_rows = (await db.execute(_apply_monitoring_scope_to_log_query(slow_query, scope))).all()
|
||
|
||
client_type = func.coalesce(PermissionAccessLog.client_type, "unknown")
|
||
client_version = func.coalesce(PermissionAccessLog.client_version, "-")
|
||
client_platform = func.coalesce(PermissionAccessLog.client_platform, "-")
|
||
client_query = select(
|
||
client_type.label("client_type"),
|
||
client_version.label("client_version"),
|
||
client_platform.label("client_platform"),
|
||
func.count().label("total_count"),
|
||
func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied_count"),
|
||
func.coalesce(func.avg(PermissionAccessLog.elapsed_ms), 0).label("avg_ms"),
|
||
).where(*base_conditions).group_by(client_type, client_version, client_platform).order_by(
|
||
desc("total_count")
|
||
).limit(5)
|
||
client_rows = (await db.execute(_apply_monitoring_scope_to_log_query(client_query, scope))).all()
|
||
|
||
return {
|
||
"top_denied": [
|
||
{"endpoint_key": row.endpoint_key, "role": row.role, "denied_count": int(row.denied_count)}
|
||
for row in denied_rows
|
||
],
|
||
"top_slow": [
|
||
{
|
||
"endpoint_key": row.endpoint_key,
|
||
"sample_count": int(row.sample_count),
|
||
"slow_count": int(row.slow_count),
|
||
"avg_elapsed_ms": round(float(row.avg_ms), 2),
|
||
"max_elapsed_ms": round(float(row.max_ms), 2),
|
||
}
|
||
for row in slow_rows
|
||
],
|
||
"client_breakdown": [
|
||
{
|
||
"client_type": row.client_type,
|
||
"client_version": row.client_version,
|
||
"client_platform": row.client_platform,
|
||
"total_count": int(row.total_count),
|
||
"denied_count": int(row.denied_count),
|
||
"deny_rate": round(int(row.denied_count) / int(row.total_count) * 100, 2) if row.total_count else 0,
|
||
"avg_elapsed_ms": round(float(row.avg_ms), 2),
|
||
}
|
||
for row in client_rows
|
||
],
|
||
}
|
||
|
||
|
||
def _trend_bucket_time(value: datetime, period: str) -> datetime:
|
||
if value.tzinfo is None:
|
||
value = value.replace(tzinfo=timezone.utc)
|
||
if period == "24h":
|
||
return value.replace(minute=0, second=0, microsecond=0)
|
||
if period == "7d":
|
||
return value.replace(hour=(value.hour // 6) * 6, minute=0, second=0, microsecond=0)
|
||
return value.replace(hour=0, minute=0, second=0, microsecond=0)
|
||
|
||
|
||
# ═══════════════════════════════════════════
|
||
# 新增端点：被拒绝最多的权限
|
||
# ═══════════════════════════════════════════
|
||
|
||
@router.get("/top-denied", status_code=status.HTTP_200_OK)
|
||
async def get_top_denied(
|
||
db: AsyncSession = Depends(get_db_session),
|
||
_=Depends(get_current_user),
|
||
days: int = Query(7, ge=1, le=90),
|
||
limit: int = Query(20, ge=1, le=100),
|
||
) -> dict:
|
||
"""获取被拒绝最多的权限"""
|
||
scope = await resolve_monitoring_scope(db, _)
|
||
if not scope.is_admin and not scope.study_ids:
|
||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足")
|
||
start_time = datetime.now(timezone.utc) - timedelta(days=days)
|
||
|
||
query = (
|
||
select(
|
||
PermissionAccessLog.endpoint_key,
|
||
PermissionAccessLog.role,
|
||
func.count().label("denied_count"),
|
||
)
|
||
.where(
|
||
PermissionAccessLog.created_at >= start_time,
|
||
PermissionAccessLog.allowed.is_(False),
|
||
)
|
||
.group_by(PermissionAccessLog.endpoint_key, PermissionAccessLog.role)
|
||
.order_by(desc("denied_count"))
|
||
.limit(limit)
|
||
)
|
||
|
||
result = await db.execute(_apply_monitoring_scope_to_log_query(query, scope))
|
||
rows = result.all()
|
||
|
||
return {
|
||
"days": days,
|
||
"items": [
|
||
{
|
||
"endpoint_key": row.endpoint_key,
|
||
"role": row.role,
|
||
"denied_count": row.denied_count,
|
||
}
|
||
for row in rows
|
||
],
|
||
}
|
||
|
||
|
||
# ═══════════════════════════════════════════
|
||
# 新增端点：IP 属地统计
|
||
# ═══════════════════════════════════════════
|
||
|
||
@router.get("/ip-locations", status_code=status.HTTP_200_OK)
|
||
async def get_ip_locations(
|
||
db: AsyncSession = Depends(get_db_session),
|
||
_=Depends(get_current_user),
|
||
days: int = Query(7, ge=1, le=90),
|
||
limit: int = Query(20, ge=1, le=100),
|
||
all_time: bool = False,
|
||
) -> dict:
|
||
"""获取 IP 省市属地统计。"""
|
||
scope = await resolve_monitoring_scope(db, _)
|
||
if not scope.is_admin and not scope.study_ids:
|
||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足")
|
||
generated_at = datetime.now(timezone.utc)
|
||
server_location = await resolve_monitoring_server_location()
|
||
start_time = None if all_time else generated_at - timedelta(days=days)
|
||
matching_security_request = select(SecurityAccessLog.id).where(
|
||
PermissionAccessLog.request_id.is_not(None),
|
||
SecurityAccessLog.request_id == PermissionAccessLog.request_id,
|
||
).exists()
|
||
permission_conditions = [
|
||
PermissionAccessLog.ip_address.is_not(None),
|
||
~matching_security_request,
|
||
]
|
||
security_conditions = [SecurityAccessLog.client_ip.is_not(None)]
|
||
if start_time is not None:
|
||
permission_conditions.insert(0, PermissionAccessLog.created_at >= start_time)
|
||
security_conditions.insert(0, SecurityAccessLog.created_at >= start_time)
|
||
security_allowed = case((SecurityAccessLog.status_code < 400, True), else_=False)
|
||
permission_result = await db.execute(
|
||
select(
|
||
PermissionAccessLog.ip_address,
|
||
PermissionAccessLog.user_id,
|
||
PermissionAccessLog.allowed,
|
||
func.count().label("event_count"),
|
||
func.min(PermissionAccessLog.created_at).label("first_seen_at"),
|
||
func.max(PermissionAccessLog.created_at).label("last_seen_at"),
|
||
)
|
||
.where(*permission_conditions)
|
||
.group_by(
|
||
PermissionAccessLog.ip_address,
|
||
PermissionAccessLog.user_id,
|
||
PermissionAccessLog.allowed,
|
||
)
|
||
)
|
||
security_result = await db.execute(
|
||
select(
|
||
SecurityAccessLog.client_ip,
|
||
SecurityAccessLog.user_identifier,
|
||
SecurityAccessLog.auth_status,
|
||
security_allowed.label("allowed"),
|
||
SecurityAccessLog.severity,
|
||
SecurityAccessLog.category,
|
||
func.count().label("event_count"),
|
||
func.min(SecurityAccessLog.created_at).label("first_seen_at"),
|
||
func.max(SecurityAccessLog.created_at).label("last_seen_at"),
|
||
)
|
||
.where(*security_conditions)
|
||
.group_by(
|
||
SecurityAccessLog.client_ip,
|
||
SecurityAccessLog.user_identifier,
|
||
SecurityAccessLog.auth_status,
|
||
security_allowed,
|
||
SecurityAccessLog.severity,
|
||
SecurityAccessLog.category,
|
||
)
|
||
)
|
||
buckets: dict[tuple[str, str, str], dict] = {}
|
||
all_ip_addresses: set[str] = set()
|
||
all_user_ids: set[str] = set()
|
||
located_ip_addresses: set[str] = set()
|
||
private_ip_addresses: set[str] = set()
|
||
unknown_ip_addresses: set[str] = set()
|
||
total_count = 0
|
||
allowed_count = 0
|
||
denied_count = 0
|
||
observed_start_at: datetime | None = None
|
||
observed_end_at: datetime | None = None
|
||
|
||
resolved_locations: dict[str, IpLocation] = {}
|
||
resolved_metadata: dict[str, GeoLocationMetadata] = {}
|
||
|
||
def normalize_observed_at(value: datetime | None) -> datetime | None:
|
||
if value is None:
|
||
return None
|
||
return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc)
|
||
|
||
def add_ip_location_row(
|
||
ip_address: str,
|
||
user_id: str | uuid.UUID | None,
|
||
allowed: bool,
|
||
event_count: int,
|
||
first_seen_at: datetime | None,
|
||
last_seen_at: datetime | None,
|
||
*,
|
||
source_kind: str,
|
||
severity: str | None = None,
|
||
category: str | None = None,
|
||
auth_status: str | None = None,
|
||
) -> None:
|
||
nonlocal total_count, allowed_count, denied_count, observed_start_at, observed_end_at
|
||
ip_info = resolved_locations.get(ip_address)
|
||
if ip_info is None:
|
||
ip_info = resolve_ip_location(ip_address)
|
||
resolved_locations[ip_address] = ip_info
|
||
metadata = resolved_metadata.get(ip_address)
|
||
if metadata is None:
|
||
metadata = resolve_geo_location_metadata(ip_info)
|
||
resolved_metadata[ip_address] = metadata
|
||
|
||
longitude = metadata.longitude
|
||
latitude = metadata.latitude
|
||
if metadata.accuracy_level == "private":
|
||
longitude = server_location.longitude if server_location else None
|
||
latitude = server_location.latitude if server_location else None
|
||
private_ip_addresses.add(ip_address)
|
||
elif longitude is not None and latitude is not None:
|
||
located_ip_addresses.add(ip_address)
|
||
else:
|
||
unknown_ip_addresses.add(ip_address)
|
||
|
||
country = metadata.country or ip_info.country
|
||
key = (metadata.country_code or country, metadata.region_code or ip_info.province, ip_info.city)
|
||
location = " / ".join(part for part in [country, ip_info.province, ip_info.city] if part) or ip_info.location or "未知"
|
||
bucket = buckets.setdefault(
|
||
key,
|
||
{
|
||
"country": country,
|
||
"country_code": metadata.country_code,
|
||
"province": ip_info.province,
|
||
"region_code": metadata.region_code,
|
||
"city": ip_info.city,
|
||
"isp": ip_info.isp,
|
||
"location": location,
|
||
"longitude": longitude,
|
||
"latitude": latitude,
|
||
"accuracy_level": metadata.accuracy_level,
|
||
"total_count": 0,
|
||
"allowed_count": 0,
|
||
"denied_count": 0,
|
||
"security_event_count": 0,
|
||
"high_risk_count": 0,
|
||
"auth_failure_count": 0,
|
||
"categories": set(),
|
||
"ip_addresses": set(),
|
||
"user_ids": set(),
|
||
"first_seen_at": None,
|
||
"last_seen_at": None,
|
||
},
|
||
)
|
||
bucket["total_count"] += event_count
|
||
bucket["allowed_count" if allowed else "denied_count"] += event_count
|
||
if ip_info.isp and bucket["isp"] and bucket["isp"] != ip_info.isp:
|
||
bucket["isp"] = "多运营商"
|
||
elif ip_info.isp:
|
||
bucket["isp"] = ip_info.isp
|
||
bucket["ip_addresses"].add(ip_address)
|
||
if user_id:
|
||
bucket["user_ids"].add(str(user_id))
|
||
normalized_first = normalize_observed_at(first_seen_at)
|
||
normalized_last = normalize_observed_at(last_seen_at)
|
||
if normalized_first and (bucket["first_seen_at"] is None or normalized_first < bucket["first_seen_at"]):
|
||
bucket["first_seen_at"] = normalized_first
|
||
if normalized_last and (bucket["last_seen_at"] is None or normalized_last > bucket["last_seen_at"]):
|
||
bucket["last_seen_at"] = normalized_last
|
||
if normalized_first and (observed_start_at is None or normalized_first < observed_start_at):
|
||
observed_start_at = normalized_first
|
||
if normalized_last and (observed_end_at is None or normalized_last > observed_end_at):
|
||
observed_end_at = normalized_last
|
||
|
||
normalized_category = (category or "").upper()
|
||
normalized_severity = (severity or "").upper()
|
||
if source_kind == "security" and normalized_category not in {"", "OTHER", "NOT_FOUND_NOISE"}:
|
||
bucket["security_event_count"] += event_count
|
||
bucket["categories"].add(normalized_category)
|
||
if source_kind == "security" and normalized_severity in {"HIGH", "CRITICAL"}:
|
||
bucket["high_risk_count"] += event_count
|
||
if source_kind == "security" and not allowed and auth_status in {"INVALID_TOKEN", "ANONYMOUS"}:
|
||
bucket["auth_failure_count"] += event_count
|
||
total_count += event_count
|
||
if allowed:
|
||
allowed_count += event_count
|
||
else:
|
||
denied_count += event_count
|
||
all_ip_addresses.add(ip_address)
|
||
if user_id:
|
||
all_user_ids.add(str(user_id))
|
||
|
||
for ip_address, user_id, allowed, event_count, first_seen_at, last_seen_at in permission_result.all():
|
||
add_ip_location_row(
|
||
ip_address,
|
||
user_id,
|
||
allowed,
|
||
int(event_count or 0),
|
||
first_seen_at,
|
||
last_seen_at,
|
||
source_kind="permission",
|
||
)
|
||
|
||
for client_ip, user_identifier, auth_status, allowed, severity, category, event_count, first_seen_at, last_seen_at in security_result.all():
|
||
user_id = None
|
||
if auth_status == "AUTHENTICATED" and user_identifier:
|
||
try:
|
||
user_id = uuid.UUID(user_identifier)
|
||
except ValueError:
|
||
user_id = user_identifier
|
||
add_ip_location_row(
|
||
client_ip,
|
||
user_id,
|
||
bool(allowed),
|
||
int(event_count or 0),
|
||
first_seen_at,
|
||
last_seen_at,
|
||
source_kind="security",
|
||
severity=severity,
|
||
category=category,
|
||
auth_status=auth_status,
|
||
)
|
||
|
||
items = sorted(buckets.values(), key=lambda item: item["total_count"], reverse=True)[:limit]
|
||
|
||
def serialize_item(item: dict) -> dict:
|
||
total = max(1, int(item["total_count"]))
|
||
denied_rate = item["denied_count"] / total
|
||
risk_score = min(
|
||
100,
|
||
round(
|
||
denied_rate * 60
|
||
+ min(item["high_risk_count"] * 25, 40)
|
||
+ min(item["auth_failure_count"] * 5, 20)
|
||
),
|
||
)
|
||
if item["high_risk_count"] > 0 or risk_score >= 60:
|
||
risk_level = "high"
|
||
elif item["denied_count"] > 0 or item["security_event_count"] > 0 or item["auth_failure_count"] > 0:
|
||
risk_level = "attention"
|
||
else:
|
||
risk_level = "normal"
|
||
risk_reasons = []
|
||
if item["high_risk_count"]:
|
||
risk_reasons.append(f"{item['high_risk_count']} 次高风险安全事件")
|
||
if item["auth_failure_count"]:
|
||
risk_reasons.append(f"{item['auth_failure_count']} 次认证失败")
|
||
if item["denied_count"]:
|
||
risk_reasons.append(f"拒绝率 {round(denied_rate * 100, 1)}%")
|
||
|
||
return {
|
||
"country": item["country"],
|
||
"country_code": item["country_code"],
|
||
"province": item["province"],
|
||
"region_code": item["region_code"],
|
||
"city": item["city"],
|
||
"isp": item["isp"],
|
||
"location": item["location"],
|
||
"longitude": item["longitude"],
|
||
"latitude": item["latitude"],
|
||
"accuracy_level": item["accuracy_level"],
|
||
"total_count": item["total_count"],
|
||
"allowed_count": item["allowed_count"],
|
||
"denied_count": item["denied_count"],
|
||
"denied_rate": round(denied_rate * 100, 1),
|
||
"unique_ip_count": len(item["ip_addresses"]),
|
||
"unique_user_count": len(item["user_ids"]),
|
||
"security_event_count": item["security_event_count"],
|
||
"high_risk_count": item["high_risk_count"],
|
||
"risk_score": risk_score,
|
||
"risk_level": risk_level,
|
||
"risk_reasons": risk_reasons,
|
||
"categories": sorted(item["categories"]),
|
||
"first_seen_at": item["first_seen_at"].isoformat() if item["first_seen_at"] else None,
|
||
"last_seen_at": item["last_seen_at"].isoformat() if item["last_seen_at"] else None,
|
||
}
|
||
|
||
effective_start_at = start_time or observed_start_at or generated_at - timedelta(
|
||
days=settings.MONITORING_ACCESS_LOG_RETENTION_DAYS
|
||
)
|
||
located_count = len(located_ip_addresses)
|
||
private_count = len(private_ip_addresses)
|
||
unknown_count = len(unknown_ip_addresses)
|
||
public_ip_count = located_count + unknown_count
|
||
timeline_granularity = "hour" if not all_time and days <= 1 else "day"
|
||
timeline = await get_source_location_timeline(
|
||
db,
|
||
start_at=effective_start_at,
|
||
end_at=generated_at,
|
||
granularity=timeline_granularity,
|
||
)
|
||
return {
|
||
"days": None if all_time else days,
|
||
"generated_at": generated_at.isoformat(),
|
||
"period": {
|
||
"mode": "retained_all" if all_time else "rolling",
|
||
"requested_days": None if all_time else days,
|
||
"start_at": effective_start_at.isoformat(),
|
||
"end_at": generated_at.isoformat(),
|
||
"observed_end_at": observed_end_at.isoformat() if observed_end_at else None,
|
||
"retention_days": settings.MONITORING_ACCESS_LOG_RETENTION_DAYS,
|
||
},
|
||
"server_location": server_location.to_public_dict() if server_location else None,
|
||
"data_quality": {
|
||
"resolver": "ip2region",
|
||
"located_ip_count": located_count,
|
||
"private_ip_count": private_count,
|
||
"unknown_ip_count": unknown_count,
|
||
"location_coverage_rate": round((located_count / public_ip_count) * 100, 1) if public_ip_count else 100.0,
|
||
"returned_region_count": len(items),
|
||
},
|
||
"timeline_granularity": timeline_granularity,
|
||
"timeline_complete_through": generated_at.replace(minute=0, second=0, microsecond=0).isoformat(),
|
||
"timeline": timeline,
|
||
"summary": {
|
||
"total_count": total_count,
|
||
"allowed_count": allowed_count,
|
||
"denied_count": denied_count,
|
||
"unique_ip_count": len(all_ip_addresses),
|
||
"unique_user_count": len(all_user_ids),
|
||
"security_event_count": sum(item["security_event_count"] for item in buckets.values()),
|
||
"high_risk_count": sum(item["high_risk_count"] for item in buckets.values()),
|
||
},
|
||
"items": [serialize_item(item) for item in items],
|
||
}
|
||
|
||
|
||
# ═══════════════════════════════════════════
|
||
# 新增端点：实时统计摘要（从DB获取）
|
||
# ═══════════════════════════════════════════
|
||
|
||
@router.get("/stats-summary", status_code=status.HTTP_200_OK)
|
||
async def get_stats_summary(
|
||
db: AsyncSession = Depends(get_db_session),
|
||
_=Depends(get_current_user),
|
||
) -> dict:
|
||
"""获取基于数据库的统计摘要"""
|
||
scope = await resolve_monitoring_scope(db, _)
|
||
if not scope.is_admin and not scope.study_ids:
|
||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足")
|
||
now = datetime.now(timezone.utc)
|
||
today_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
|
||
hour_ago = now - timedelta(hours=1)
|
||
|
||
# 今日统计
|
||
today_query = select(
|
||
func.count().label("total"),
|
||
func.count().filter(PermissionAccessLog.allowed.is_(True)).label("allowed"),
|
||
func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied"),
|
||
func.coalesce(func.avg(PermissionAccessLog.elapsed_ms), 0).label("avg_ms"),
|
||
func.coalesce(func.max(PermissionAccessLog.elapsed_ms), 0).label("max_ms"),
|
||
).where(PermissionAccessLog.created_at >= today_start)
|
||
today_result = await db.execute(
|
||
_apply_monitoring_scope_to_log_query(today_query, scope)
|
||
)
|
||
today = today_result.one()
|
||
|
||
# 最近一小时
|
||
hour_query = select(
|
||
func.count().label("total"),
|
||
func.count().filter(PermissionAccessLog.allowed.is_(True)).label("allowed"),
|
||
func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied"),
|
||
).where(PermissionAccessLog.created_at >= hour_ago)
|
||
hour_result = await db.execute(
|
||
_apply_monitoring_scope_to_log_query(hour_query, scope)
|
||
)
|
||
hour = hour_result.one()
|
||
|
||
# 总记录数
|
||
total_query = select(func.count()).select_from(PermissionAccessLog)
|
||
total_result = await db.execute(
|
||
_apply_monitoring_scope_to_log_query(total_query, scope)
|
||
)
|
||
total_logs = total_result.scalar() or 0
|
||
|
||
return {
|
||
"total_logs": total_logs,
|
||
"today": {
|
||
"total_checks": today.total,
|
||
"allowed_checks": today.allowed,
|
||
"denied_checks": today.denied,
|
||
"avg_elapsed_ms": round(float(today.avg_ms), 2),
|
||
"max_elapsed_ms": round(float(today.max_ms), 2),
|
||
"allow_rate": round(today.allowed / today.total * 100, 1) if today.total > 0 else 0,
|
||
"deny_rate": round(today.denied / today.total * 100, 1) if today.total > 0 else 0,
|
||
},
|
||
"last_hour": {
|
||
"total_checks": hour.total,
|
||
"allowed_checks": hour.allowed,
|
||
"denied_checks": hour.denied,
|
||
"requests_per_minute": round(hour.total / 60, 1),
|
||
},
|
||
}
|