"""权限系统监控API 提供权限系统的监控数据、访问日志、趋势分析和告警信息。 """ from __future__ import annotations import uuid import json import time from collections import defaultdict from datetime import datetime, timedelta, timezone from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query, status from sqlalchemy import String, case, cast, desc, func, literal, or_, select, union_all from sqlalchemy.ext.asyncio import AsyncSession from app.core.config import settings from app.core.deps import get_current_user, get_db_session, is_system_admin from app.core.permission_monitor import ( CACHE_HIT_RATE_HEALTH_MIN_ACCESSES, CACHE_METRICS_WINDOW_SECONDS, get_permission_monitor, ) from app.models.permission_access_log import PermissionAccessLog from app.models.permission_metric_snapshot import PermissionMetricSnapshot from app.models.security_access_log import SecurityAccessLog from app.models.user import User from app.services.geo_location_metadata import GeoLocationMetadata, resolve_geo_location_metadata from app.services.ip_location import IpLocation, resolve_ip_location from app.services.monitoring_retention import get_monitoring_retention_status from app.services.permission_log_writer import get_log_writer from app.services.security_access_log_writer import get_security_log_writer from app.services.security_events import classify_security_event from app.services.monitoring_server_location import resolve_monitoring_server_location from app.services.source_location_aggregator import get_source_location_timeline router = APIRouter(prefix="/permission-monitoring", tags=["permission-monitoring"]) class MonitoringScope: def __init__(self, *, is_admin: bool, study_ids: set[uuid.UUID]) -> None: self.is_admin = is_admin self.study_ids = study_ids def can_access_study(self, study_id: uuid.UUID | None) -> bool: if self.is_admin: return True return study_id is not None and study_id in self.study_ids async def resolve_monitoring_scope(db: AsyncSession, current_user) -> MonitoringScope: if is_system_admin(current_user): return MonitoringScope(is_admin=True, study_ids=set()) raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") def _apply_monitoring_scope_to_log_query(query, scope: MonitoringScope): if scope.is_admin: return query return query.where(PermissionAccessLog.study_id.in_(scope.study_ids)) KNOWN_PERMISSION_CLIENT_TYPES = ("web", "desktop") def _permission_client_type_condition(client_type: str): normalized = client_type.strip().lower() if normalized == "unknown": return or_( PermissionAccessLog.client_type.is_(None), PermissionAccessLog.client_type == "", ~PermissionAccessLog.client_type.in_(KNOWN_PERMISSION_CLIENT_TYPES), ) return PermissionAccessLog.client_type == normalized def _security_client_type_condition(client_type: str): normalized = client_type.strip().lower() if normalized == "unknown": return or_( SecurityAccessLog.client_type.is_(None), SecurityAccessLog.client_type == "", ~SecurityAccessLog.client_type.in_(KNOWN_PERMISSION_CLIENT_TYPES), ) return SecurityAccessLog.client_type == normalized def _coerce_request_headers(value) -> dict[str, str] | None: if not value: return None if isinstance(value, str): try: value = json.loads(value) except json.JSONDecodeError: return None if not isinstance(value, dict): return None headers = {str(key): str(header_value) for key, header_value in value.items() if header_value is not None} return headers or None def _coerce_request_snapshot(value) -> dict | None: if not value: return None if isinstance(value, str): try: value = json.loads(value) except json.JSONDecodeError: return None return value if isinstance(value, dict) else None async def _resolve_permission_location_keyword_ips( db: AsyncSession, conditions: list, keyword: str, ) -> list[str]: token = keyword.strip().lower() if not token: return [] ip_query = ( select(PermissionAccessLog.ip_address) .where(PermissionAccessLog.ip_address.is_not(None)) .distinct() ) for condition in conditions: ip_query = ip_query.where(condition) result = await db.execute(ip_query) matched_ips: list[str] = [] for ip_address in result.scalars().all(): ip_location = resolve_ip_location(ip_address) haystack = " ".join( [ ip_location.location or "", ip_location.country or "", ip_location.province or "", ip_location.city or "", ip_location.isp or "", ] ).lower() if token in haystack: matched_ips.append(ip_address) return matched_ips async def _resolve_security_location_keyword_ips( db: AsyncSession, conditions: list, keyword: str, ) -> list[str]: token = keyword.strip().lower() if not token: return [] ip_query = ( select(SecurityAccessLog.client_ip) .where(SecurityAccessLog.client_ip.is_not(None)) .distinct() ) for condition in conditions: ip_query = ip_query.where(condition) result = await db.execute(ip_query) matched_ips: list[str] = [] for ip_address in result.scalars().all(): ip_location = resolve_ip_location(ip_address) haystack = " ".join( [ ip_location.location or "", ip_location.country or "", ip_location.province or "", ip_location.city or "", ip_location.isp or "", ] ).lower() if token in haystack: matched_ips.append(ip_address) return matched_ips def _permission_access_item(log: PermissionAccessLog, full_name: str | None, email: str | None) -> dict: ip_location = resolve_ip_location(log.ip_address) return { "id": str(log.id), "event_type": "permission", "study_id": str(log.study_id) if log.study_id else "", "user_id": str(log.user_id) if log.user_id else "", "user_name": full_name or "未知用户", "user_email": email, "endpoint_key": log.endpoint_key, "role": log.role, "allowed": log.allowed, "elapsed_ms": round(log.elapsed_ms, 2), "ip_address": log.ip_address, "ip_location": ip_location.location, "ip_country": ip_location.country, "ip_province": ip_location.province, "ip_city": ip_location.city, "ip_isp": ip_location.isp, "user_agent": log.user_agent, "client_type": log.client_type, "client_version": log.client_version, "client_platform": log.client_platform, "build_channel": log.build_channel, "build_commit": log.build_commit, "request_headers": _coerce_request_headers(log.request_headers), "request_snapshot": _coerce_request_snapshot(getattr(log, "request_snapshot", None)), "request_id": getattr(log, "request_id", None), "method": None, "path": None, "status_code": None, "auth_status": "AUTHENTICATED", "account_label": full_name or email, "category": None, "severity": None, "created_at": log.created_at.isoformat(), } def _security_access_item(log: SecurityAccessLog, user_names: dict[str, str] | None = None) -> dict: ip_location = resolve_ip_location(log.client_ip) classification = _classify_security_access_log(log, ip_location) account_label = _security_account_label(log.auth_status, log.user_identifier, user_names) return { "id": f"security:{log.id}", "event_type": "security", "study_id": "", "user_id": log.user_identifier or "", "user_name": account_label, "user_email": None, "endpoint_key": f"{log.method} {log.path}", "role": classification["category"], "allowed": log.status_code < 400, "elapsed_ms": round(log.elapsed_ms, 2), "ip_address": log.client_ip, "ip_location": ip_location.location, "ip_country": ip_location.country, "ip_province": ip_location.province, "ip_city": ip_location.city, "ip_isp": ip_location.isp, "user_agent": log.user_agent, "client_type": log.client_type, "client_version": log.client_version, "client_platform": log.client_platform, "build_channel": log.build_channel, "build_commit": log.build_commit, "request_headers": _coerce_request_headers(getattr(log, "request_headers", None)), "request_snapshot": _coerce_request_snapshot(getattr(log, "request_snapshot", None)), "request_id": getattr(log, "request_id", None), "method": log.method, "path": log.path, "status_code": log.status_code, "auth_status": log.auth_status, "account_label": account_label, "category": classification["category"], "severity": classification["severity"], "created_at": log.created_at.isoformat(), } async def _security_user_names(db: AsyncSession, logs: list[SecurityAccessLog]) -> dict[str, str]: user_ids: list[uuid.UUID] = [] for log in logs: if log.auth_status != "AUTHENTICATED" or not log.user_identifier: continue try: user_ids.append(uuid.UUID(log.user_identifier)) except ValueError: continue if not user_ids: return {} users_result = await db.execute(select(User.id, User.full_name).where(User.id.in_(user_ids))) return {str(row.id): row.full_name for row in users_result.all()} # ═══════════════════════════════════════════ # 原有端点（保持兼容） # ═══════════════════════════════════════════ @router.get("/metrics", status_code=status.HTTP_200_OK) async def get_permission_metrics( db: AsyncSession = Depends(get_db_session), _=Depends(get_current_user), hours: int = Query(24, ge=1, le=720), ) -> dict: """从 permission_access_logs 实时聚合权限检查指标""" scope = await resolve_monitoring_scope(db, _) if not scope.is_admin and not scope.study_ids: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") start_time = datetime.now(timezone.utc) - timedelta(hours=hours) metrics_query = select( func.count().label("total_checks"), func.count().filter(PermissionAccessLog.allowed.is_(True)).label("allowed_checks"), func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied_checks"), func.coalesce(func.sum(PermissionAccessLog.elapsed_ms), 0).label("total_time_ms"), func.coalesce(func.min(PermissionAccessLog.elapsed_ms), 0).label("min_time_ms"), func.coalesce(func.max(PermissionAccessLog.elapsed_ms), 0).label("max_time_ms"), func.coalesce(func.avg(PermissionAccessLog.elapsed_ms), 0).label("avg_time_ms"), ).where(PermissionAccessLog.created_at >= start_time) result = await db.execute( _apply_monitoring_scope_to_log_query(metrics_query, scope) ) row = result.one() total = row.total_checks or 0 monitor = get_permission_monitor() cache_metrics = monitor.metrics.cache_metrics.window_stats() return { "window_hours": hours, "check_metrics": { "total_checks": total, "allowed_checks": row.allowed_checks or 0, "denied_checks": row.denied_checks or 0, "total_time": round(float(row.total_time_ms) / 1000, 3), "min_time": round(float(row.min_time_ms) / 1000, 3), "max_time": round(float(row.max_time_ms) / 1000, 3), "avg_time": round(float(row.avg_time_ms) / 1000, 3), "allow_rate": round(row.allowed_checks / total * 100, 2) if total else 0, "deny_rate": round(row.denied_checks / total * 100, 2) if total else 0, "error_rate": 0, "errors": 0, }, "cache_metrics": cache_metrics, "uptime_seconds": monitor.metrics.uptime_seconds, } @router.get("/cache-stats", status_code=status.HTTP_200_OK) async def get_cache_statistics( _=Depends(get_current_user), db: AsyncSession = Depends(get_db_session), ) -> dict: scope = await resolve_monitoring_scope(db, _) if not scope.is_admin and not scope.study_ids: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") monitor = get_permission_monitor() return monitor.get_cache_stats() @router.get("/alerts", status_code=status.HTTP_200_OK) async def get_alerts( level: str | None = None, limit: int = 100, _=Depends(get_current_user), db: AsyncSession = Depends(get_db_session), ) -> dict: scope = await resolve_monitoring_scope(db, _) if not scope.is_admin and not scope.study_ids: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") monitor = get_permission_monitor() alerts = monitor.get_alerts(level=level, limit=limit) return { "total": len(alerts), "alerts": alerts, } @router.post("/reset-metrics", status_code=status.HTTP_200_OK) async def reset_metrics( _=Depends(get_current_user), db: AsyncSession = Depends(get_db_session), ) -> dict: scope = await resolve_monitoring_scope(db, _) if not scope.is_admin and not scope.study_ids: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") monitor = get_permission_monitor() monitor.reset_metrics() return {"message": "指标已重置"} @router.post("/clear-alerts", status_code=status.HTTP_200_OK) async def clear_alerts( _=Depends(get_current_user), db: AsyncSession = Depends(get_db_session), ) -> dict: scope = await resolve_monitoring_scope(db, _) if not scope.is_admin and not scope.study_ids: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") monitor = get_permission_monitor() monitor.clear_alerts() return {"message": "告警已清除"} @router.get("/health", status_code=status.HTTP_200_OK) async def permission_system_health( db: AsyncSession = Depends(get_db_session), _=Depends(get_current_user), ) -> dict: """从 DB 聚合最近 1 小时数据评估权限系统健康状态""" scope = await resolve_monitoring_scope(db, _) if not scope.is_admin and not scope.study_ids: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") start_time = datetime.now(timezone.utc) - timedelta(hours=1) database_started_at = time.perf_counter() await db.execute(select(literal(1))) database_latency_ms = round((time.perf_counter() - database_started_at) * 1000, 2) health_query = select( func.count().label("total"), func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied"), func.coalesce(func.avg(PermissionAccessLog.elapsed_ms), 0).label("avg_ms"), ).where(PermissionAccessLog.created_at >= start_time) result = await db.execute( _apply_monitoring_scope_to_log_query(health_query, scope) ) row = result.one() total = row.total or 0 avg_ms = float(row.avg_ms) deny_rate = round(row.denied / total * 100, 2) if total else 0 monitor = get_permission_monitor() cache_metrics = monitor.metrics.cache_metrics.window_stats() now_timestamp = datetime.now(timezone.utc).timestamp() recent_alerts = [ alert for alert in monitor.get_alerts(limit=1000) if alert["timestamp"] >= now_timestamp - 3600 ] permission_writer = get_log_writer() security_writer = get_security_log_writer() permission_writer_stats = permission_writer.stats() if permission_writer else None security_writer_stats = security_writer.stats() if security_writer else None retention_stats = get_monitoring_retention_status() def writer_component(stats: dict | None) -> dict: if stats is None: return {"status": "unknown", "reason": "writer_not_initialized"} queue_capacity = max(1, int(stats.get("queue_capacity") or 1)) queue_usage = round(int(stats.get("queue_size") or 0) / queue_capacity * 100, 2) if not stats.get("running"): component_status = "unhealthy" elif queue_usage >= 80 or stats.get("failed_batches") or stats.get("dropped_entries"): component_status = "degraded" else: component_status = "healthy" return {**stats, "status": component_status, "queue_usage_percent": queue_usage} permission_writer_component = writer_component(permission_writer_stats) security_writer_component = writer_component(security_writer_stats) if retention_stats["running"]: retention_has_unrecovered_error = bool( retention_stats["last_error_at"] and ( not retention_stats["last_success_at"] or retention_stats["last_error_at"] > retention_stats["last_success_at"] ) ) retention_status = "degraded" if retention_has_unrecovered_error else "healthy" elif retention_stats["last_run_started_at"]: retention_status = "unhealthy" else: retention_status = "unknown" retention_component = {**retention_stats, "status": retention_status} storage_result = await db.execute( select( select(func.count()).select_from(PermissionAccessLog).scalar_subquery().label("permission_count"), select(func.min(PermissionAccessLog.created_at)).scalar_subquery().label("permission_oldest"), select(func.count()).select_from(SecurityAccessLog).scalar_subquery().label("security_count"), select(func.min(SecurityAccessLog.created_at)).scalar_subquery().label("security_oldest"), select(func.count()).select_from(PermissionMetricSnapshot).scalar_subquery().label("metric_count"), ) ) storage_row = storage_result.one() deductions: dict[str, int] = { "performance": 0, "deny_rate": 0, "cache": 0, "alerts": 0, "runtime": 0, } issues: list[str] = [] if avg_ms > 200: deductions["performance"] = 25 elif avg_ms > 50: deductions["performance"] = 20 elif avg_ms > 10: deductions["performance"] = 10 if deductions["performance"]: issues.append(f"权限检查平均响应时间过长（{avg_ms:.2f}ms）") if deny_rate > 50: deductions["deny_rate"] = 20 elif deny_rate > 20: deductions["deny_rate"] = 10 if deductions["deny_rate"]: issues.append(f"权限拒绝率偏高（{deny_rate:.2f}%）") cache_sample_sufficient = ( cache_metrics["total_accesses"] >= CACHE_HIT_RATE_HEALTH_MIN_ACCESSES ) if cache_sample_sufficient: if cache_metrics["hit_rate"] < 20: deductions["cache"] = 25 elif cache_metrics["hit_rate"] < 50: deductions["cache"] = 20 elif cache_metrics["hit_rate"] < 80: deductions["cache"] = 10 if deductions["cache"]: issues.append(f"近 1 小时缓存命中率偏低（{cache_metrics['hit_rate']:.2f}%）") error_alert_count = sum(alert["level"] == "error" for alert in recent_alerts) warning_alert_count = sum(alert["level"] == "warning" for alert in recent_alerts) if error_alert_count: deductions["alerts"] = 10 issues.append(f"近 1 小时发生 {error_alert_count} 条错误告警") elif warning_alert_count: deductions["alerts"] = 5 issues.append(f"近 1 小时发生 {warning_alert_count} 条警告") runtime_components = { "database": {"status": "healthy", "latency_ms": database_latency_ms}, "permission_log_writer": permission_writer_component, "security_log_writer": security_writer_component, "retention": retention_component, } component_labels = { "database": "数据库", "permission_log_writer": "权限日志写入器", "security_log_writer": "安全日志写入器", "retention": "留存清理任务", } unhealthy_components = [ component_labels[name] for name, component in runtime_components.items() if component["status"] == "unhealthy" ] degraded_components = [ component_labels[name] for name, component in runtime_components.items() if component["status"] == "degraded" ] if unhealthy_components: deductions["runtime"] = 20 issues.append(f"运行组件异常：{', '.join(unhealthy_components)}") elif degraded_components: deductions["runtime"] = 10 issues.append(f"运行组件降级：{', '.join(degraded_components)}") elif database_latency_ms > 500: deductions["runtime"] = 5 issues.append(f"数据库健康检查延迟偏高（{database_latency_ms:.2f}ms）") health_score = max(0, 100 - sum(deductions.values())) sample_sufficient = total >= 10 return { "status": "healthy" if health_score >= 80 else "degraded" if health_score >= 50 else "unhealthy", "health_score": health_score, "issues": issues, "sample_sufficient": sample_sufficient, "score_breakdown": { "performance": {"weight": 25, "deduction": deductions["performance"]}, "deny_rate": {"weight": 20, "deduction": deductions["deny_rate"]}, "cache": { "weight": 25, "deduction": deductions["cache"], "sample_sufficient": cache_sample_sufficient, "scope": "sliding_window", "window_seconds": CACHE_METRICS_WINDOW_SECONDS, }, "alerts": {"weight": 10, "deduction": deductions["alerts"]}, "runtime": {"weight": 20, "deduction": deductions["runtime"]}, }, "last_hour": { "total_checks": total, "denied_checks": row.denied or 0, "deny_rate": deny_rate, "avg_elapsed_ms": round(avg_ms, 2), "error_alerts": error_alert_count, "warning_alerts": warning_alert_count, }, "cache_stats": monitor.get_cache_stats(), "components": runtime_components, "storage": { "permission_access_logs": storage_row.permission_count or 0, "security_access_logs": storage_row.security_count or 0, "permission_metric_snapshots": storage_row.metric_count or 0, "oldest_permission_access_log_at": ( storage_row.permission_oldest.isoformat() if storage_row.permission_oldest else None ), "oldest_security_access_log_at": ( storage_row.security_oldest.isoformat() if storage_row.security_oldest else None ), }, } # ═══════════════════════════════════════════ # 新增端点：访问日志 # ═══════════════════════════════════════════ @router.get("/access-logs", status_code=status.HTTP_200_OK) async def get_access_logs( db: AsyncSession = Depends(get_db_session), _=Depends(get_current_user), study_id: Optional[uuid.UUID] = Query(None), user_id: Optional[uuid.UUID] = Query(None), endpoint_key: Optional[str] = Query(None), role: Optional[str] = Query(None), allowed: Optional[bool] = Query(None), start_time: Optional[datetime] = Query(None), end_time: Optional[datetime] = Query(None), client_ip: Optional[str] = Query(None), client_type: Optional[str] = Query(None), event_type: str = Query("all", pattern="^(all|permission)$"), keyword: Optional[str] = Query(None), page: int = Query(1, ge=1), page_size: int = Query(50, ge=1, le=200), ) -> dict: """分页查询权限访问日志""" scope = await resolve_monitoring_scope(db, _) if not scope.is_admin and not scope.study_ids: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") if study_id and not scope.can_access_study(study_id): raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") client_ip = client_ip if isinstance(client_ip, str) else None client_type = client_type if isinstance(client_type, str) else None keyword = keyword if isinstance(keyword, str) else None start_time = start_time if isinstance(start_time, datetime) else None end_time = end_time if isinstance(end_time, datetime) else None conditions = [] if study_id: conditions.append(PermissionAccessLog.study_id == study_id) elif not scope.is_admin: conditions.append(PermissionAccessLog.study_id.in_(scope.study_ids)) if user_id: conditions.append(PermissionAccessLog.user_id == user_id) if endpoint_key: conditions.append(PermissionAccessLog.endpoint_key == endpoint_key) if role: conditions.append(PermissionAccessLog.role == role) if allowed is not None: conditions.append(PermissionAccessLog.allowed == allowed) if start_time: conditions.append(PermissionAccessLog.created_at >= start_time) if end_time: conditions.append(PermissionAccessLog.created_at <= end_time) if client_ip and client_ip.strip(): conditions.append(PermissionAccessLog.ip_address.ilike(f"%{client_ip.strip()}%")) if client_type and client_type.strip(): conditions.append(_permission_client_type_condition(client_type)) if keyword and keyword.strip(): token = keyword.strip() like_token = f"%{token}%" location_ips = await _resolve_permission_location_keyword_ips(db, conditions, token) keyword_conditions = [ PermissionAccessLog.endpoint_key.ilike(like_token), PermissionAccessLog.role.ilike(like_token), PermissionAccessLog.ip_address.ilike(like_token), PermissionAccessLog.client_type.ilike(like_token), PermissionAccessLog.client_version.ilike(like_token), PermissionAccessLog.client_platform.ilike(like_token), PermissionAccessLog.user_agent.ilike(like_token), cast(PermissionAccessLog.request_headers, String).ilike(like_token), cast(PermissionAccessLog.request_snapshot, String).ilike(like_token), User.full_name.ilike(like_token), User.email.ilike(like_token), cast(PermissionAccessLog.user_id, String).ilike(like_token), ] if location_ips: keyword_conditions.append(PermissionAccessLog.ip_address.in_(location_ips)) conditions.append(or_(*keyword_conditions)) include_security_logs = ( event_type != "permission" and scope.is_admin and study_id is None and role is None and endpoint_key is None and allowed is not True ) event_conditions = list(conditions) if include_security_logs: matching_security_event = select(SecurityAccessLog.id).where( PermissionAccessLog.request_id.is_not(None), SecurityAccessLog.request_id == PermissionAccessLog.request_id, SecurityAccessLog.status_code >= 400, ).exists() event_conditions.append(~matching_security_event) permission_event_query = ( select( literal("permission").label("event_type"), cast(PermissionAccessLog.id, String).label("event_id"), PermissionAccessLog.created_at.label("created_at"), cast(PermissionAccessLog.user_id, String).label("user_id"), PermissionAccessLog.ip_address.label("ip_address"), PermissionAccessLog.allowed.label("allowed"), PermissionAccessLog.elapsed_ms.label("elapsed_ms"), ) .outerjoin(User, PermissionAccessLog.user_id == User.id) ) user_stats_query = ( select( PermissionAccessLog.ip_address.label("sample_ip_address"), func.count().label("total_count"), func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied_count"), func.count(func.distinct(PermissionAccessLog.ip_address)).label("unique_ip_count"), func.max(PermissionAccessLog.created_at).label("last_seen_at"), ) .outerjoin(User, PermissionAccessLog.user_id == User.id) .group_by(PermissionAccessLog.ip_address) .order_by(desc("total_count"), desc("last_seen_at")) .limit(10) ) for condition in event_conditions: permission_event_query = permission_event_query.where(condition) for condition in conditions: user_stats_query = user_stats_query.where(condition) user_stats_result = await db.execute(user_stats_query) user_stats_rows = user_stats_result.all() latest_user_by_ip = {} if user_stats_rows: ranked_ips = [stat.sample_ip_address for stat in user_stats_rows if stat.sample_ip_address] latest_user_query = ( select( PermissionAccessLog.ip_address, PermissionAccessLog.user_id, User.full_name, User.email, PermissionAccessLog.role, PermissionAccessLog.created_at, ) .outerjoin(User, PermissionAccessLog.user_id == User.id) .where(PermissionAccessLog.ip_address.in_(ranked_ips)) .order_by(PermissionAccessLog.ip_address, desc(PermissionAccessLog.created_at)) ) for condition in conditions: latest_user_query = latest_user_query.where(condition) latest_user_result = await db.execute(latest_user_query) for row in latest_user_result.all(): latest_user_by_ip.setdefault(row.ip_address, row) event_queries = [permission_event_query] if include_security_logs: security_conditions = [SecurityAccessLog.status_code >= 400] if user_id: security_conditions.append(SecurityAccessLog.user_identifier == str(user_id)) if start_time: security_conditions.append(SecurityAccessLog.created_at >= start_time) if end_time: security_conditions.append(SecurityAccessLog.created_at <= end_time) if client_ip and client_ip.strip(): security_conditions.append(SecurityAccessLog.client_ip.ilike(f"%{client_ip.strip()}%")) if client_type and client_type.strip(): security_conditions.append(_security_client_type_condition(client_type)) if keyword and keyword.strip(): token = keyword.strip() like_token = f"%{token}%" location_ips = await _resolve_security_location_keyword_ips(db, security_conditions, token) matching_users_result = await db.execute( select(cast(User.id, String)).where( or_(User.full_name.ilike(like_token), User.email.ilike(like_token)) ) ) matching_user_ids = list(matching_users_result.scalars().all()) keyword_conditions = [ SecurityAccessLog.method.ilike(like_token), SecurityAccessLog.path.ilike(like_token), SecurityAccessLog.client_ip.ilike(like_token), SecurityAccessLog.client_type.ilike(like_token), SecurityAccessLog.client_version.ilike(like_token), SecurityAccessLog.client_platform.ilike(like_token), SecurityAccessLog.user_agent.ilike(like_token), SecurityAccessLog.auth_status.ilike(like_token), SecurityAccessLog.user_identifier.ilike(like_token), cast(SecurityAccessLog.status_code, String).ilike(like_token), cast(SecurityAccessLog.request_headers, String).ilike(like_token), cast(SecurityAccessLog.request_snapshot, String).ilike(like_token), ] if location_ips: keyword_conditions.append(SecurityAccessLog.client_ip.in_(location_ips)) if matching_user_ids: keyword_conditions.append(SecurityAccessLog.user_identifier.in_(matching_user_ids)) security_conditions.append(or_(*keyword_conditions)) security_event_query = select( literal("security").label("event_type"), cast(SecurityAccessLog.id, String).label("event_id"), SecurityAccessLog.created_at.label("created_at"), SecurityAccessLog.user_identifier.label("user_id"), SecurityAccessLog.client_ip.label("ip_address"), (SecurityAccessLog.status_code < 400).label("allowed"), SecurityAccessLog.elapsed_ms.label("elapsed_ms"), ) for condition in security_conditions: security_event_query = security_event_query.where(condition) event_queries.append(security_event_query) events = ( union_all(*event_queries).subquery("monitoring_events") if len(event_queries) > 1 else event_queries[0].subquery("monitoring_events") ) summary_result = await db.execute( select( func.count().label("total_count"), func.count(func.distinct(events.c.user_id)).label("unique_user_count"), func.count(func.distinct(events.c.ip_address)).label("unique_ip_count"), func.count().filter(events.c.allowed.is_(False)).label("denied_count"), func.coalesce(func.avg(events.c.elapsed_ms), 0).label("avg_elapsed_ms"), ).select_from(events) ) summary_row = summary_result.one() total = summary_row.total_count or 0 page_result = await db.execute( select(events.c.event_type, events.c.event_id, events.c.created_at) .order_by(desc(events.c.created_at), desc(events.c.event_id)) .offset((page - 1) * page_size) .limit(page_size) ) page_events = page_result.all() permission_ids = [uuid.UUID(row.event_id) for row in page_events if row.event_type == "permission"] security_ids = [uuid.UUID(row.event_id) for row in page_events if row.event_type == "security"] items_by_key: dict[tuple[str, str], dict] = {} if permission_ids: permission_result = await db.execute( select(PermissionAccessLog, User.full_name, User.email) .outerjoin(User, PermissionAccessLog.user_id == User.id) .where(PermissionAccessLog.id.in_(permission_ids)) ) for log, full_name, email in permission_result.all(): items_by_key[("permission", str(log.id))] = _permission_access_item(log, full_name, email) if security_ids: security_result = await db.execute( select(SecurityAccessLog).where(SecurityAccessLog.id.in_(security_ids)) ) security_logs = list(security_result.scalars().all()) security_user_names = await _security_user_names(db, security_logs) for log in security_logs: items_by_key[("security", str(log.id))] = _security_access_item(log, security_user_names) page_items = [ items_by_key[(row.event_type, row.event_id)] for row in page_events if (row.event_type, row.event_id) in items_by_key ] summary = { "total_count": total, "unique_user_count": summary_row.unique_user_count or 0, "unique_ip_count": summary_row.unique_ip_count or 0, "denied_count": summary_row.denied_count or 0, "avg_elapsed_ms": round(float(summary_row.avg_elapsed_ms), 2), } user_stats = [] for stat in user_stats_rows: sample_location = resolve_ip_location(stat.sample_ip_address) latest_user = latest_user_by_ip.get(stat.sample_ip_address) user_stats.append( { "user_id": str(latest_user.user_id) if latest_user else "", "user_name": latest_user.full_name if latest_user and latest_user.full_name else "未知用户", "user_email": latest_user.email if latest_user else None, "role": latest_user.role if latest_user else "", "total_count": stat.total_count, "denied_count": stat.denied_count, "unique_ip_count": stat.unique_ip_count, "sample_ip_address": stat.sample_ip_address, "last_seen_at": stat.last_seen_at.isoformat() if stat.last_seen_at else None, "primary_location": sample_location.location, } ) return { "total": total, "page": page, "page_size": page_size, "summary": summary, "user_stats": user_stats, "items": page_items, } def _security_account_label(auth_status: str, user_identifier: str | None, user_names: dict[str, str] | None = None) -> str: if auth_status == "AUTHENTICATED" and user_identifier: return (user_names or {}).get(user_identifier) or user_identifier if auth_status == "INVALID_TOKEN": return "无效令牌" return "未知账号" def _classify_security_access_log(log: SecurityAccessLog, ip_location: IpLocation | None = None) -> dict[str, str]: if getattr(log, "category", None) and getattr(log, "severity", None): return {"category": log.category, "severity": log.severity} return classify_security_event( path=log.path, status_code=log.status_code, auth_status=log.auth_status, ip_location=ip_location, ) def _security_endpoint_type(path: str) -> str: normalized_path = (path or "").lower() if "/auth" in normalized_path or "/login" in normalized_path: return "认证接口" if normalized_path.startswith("/api/"): return "业务 API" if normalized_path.endswith((".js", ".css", ".png", ".jpg", ".jpeg", ".svg", ".ico", ".map")): return "静态资源" if any(marker in normalized_path for marker in (".env", "wp-", "php", "admin", "shell", "config")): return "探测路径" return "其他请求" @router.get("/security-logs", status_code=status.HTTP_200_OK) async def get_security_access_logs( db: AsyncSession = Depends(get_db_session), _=Depends(get_current_user), status_min: Optional[int] = Query(None, ge=100, le=599), auth_status: Optional[str] = Query(None), client_type: Optional[str] = Query(None), client_version: Optional[str] = Query(None), client_ip: Optional[str] = Query(None), category: Optional[str] = Query(None), severity: Optional[str] = Query(None), keyword: Optional[str] = Query(None), events_only: bool = Query(False), start_time: Optional[datetime] = Query(None), end_time: Optional[datetime] = Query(None), page: int = Query(1, ge=1), page_size: int = Query(50, ge=1, le=200), ) -> dict: """查询底层安全访问日志，覆盖匿名、无效令牌和异常状态请求。""" scope = await resolve_monitoring_scope(db, _) if not scope.is_admin: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") client_type = client_type if isinstance(client_type, str) else None client_version = client_version if isinstance(client_version, str) else None client_ip = client_ip if isinstance(client_ip, str) else None category = category if isinstance(category, str) else None severity = severity if isinstance(severity, str) else None keyword = keyword if isinstance(keyword, str) else None events_only = events_only if isinstance(events_only, bool) else False start_time = start_time if isinstance(start_time, datetime) else None end_time = end_time if isinstance(end_time, datetime) else None conditions = [] if status_min is not None: conditions.append(SecurityAccessLog.status_code >= status_min) if auth_status: conditions.append(SecurityAccessLog.auth_status == auth_status) if client_type: conditions.append(SecurityAccessLog.client_type == client_type) if client_version: conditions.append(SecurityAccessLog.client_version == client_version) if client_ip: conditions.append(SecurityAccessLog.client_ip.ilike(f"%{client_ip.strip()}%")) if category: conditions.append(SecurityAccessLog.category == category.strip().upper()) if severity: conditions.append(SecurityAccessLog.severity == severity.strip().upper()) if events_only: conditions.append(SecurityAccessLog.category != "OTHER") if start_time: conditions.append(SecurityAccessLog.created_at >= start_time) if end_time: conditions.append(SecurityAccessLog.created_at <= end_time) if keyword and keyword.strip(): token = keyword.strip() like_token = f"%{token}%" location_ips = await _resolve_security_location_keyword_ips(db, conditions, token) matching_users_result = await db.execute( select(cast(User.id, String)).where( or_(User.full_name.ilike(like_token), User.email.ilike(like_token)) ) ) matching_user_ids = list(matching_users_result.scalars().all()) keyword_conditions = [ SecurityAccessLog.method.ilike(like_token), SecurityAccessLog.path.ilike(like_token), SecurityAccessLog.client_ip.ilike(like_token), SecurityAccessLog.client_type.ilike(like_token), SecurityAccessLog.client_version.ilike(like_token), SecurityAccessLog.client_platform.ilike(like_token), SecurityAccessLog.user_agent.ilike(like_token), SecurityAccessLog.auth_status.ilike(like_token), SecurityAccessLog.user_identifier.ilike(like_token), SecurityAccessLog.category.ilike(like_token), SecurityAccessLog.severity.ilike(like_token), cast(SecurityAccessLog.status_code, String).ilike(like_token), ] if location_ips: keyword_conditions.append(SecurityAccessLog.client_ip.in_(location_ips)) if matching_user_ids: keyword_conditions.append(SecurityAccessLog.user_identifier.in_(matching_user_ids)) conditions.append(or_(*keyword_conditions)) query = select(SecurityAccessLog) count_query = select(func.count()).select_from(SecurityAccessLog) summary_query = select( func.count().label("total_count"), func.count().filter(SecurityAccessLog.auth_status == "ANONYMOUS").label("anonymous_count"), func.count().filter(SecurityAccessLog.auth_status == "INVALID_TOKEN").label("invalid_token_count"), func.count().filter(SecurityAccessLog.status_code >= 400).label("error_count"), func.count(func.distinct(SecurityAccessLog.client_ip)).label("unique_ip_count"), func.count().filter(SecurityAccessLog.severity.in_(("CRITICAL", "HIGH"))).label("high_risk_count"), func.count().filter(SecurityAccessLog.category == "SERVER_ERROR").label("server_error_count"), ).select_from(SecurityAccessLog) for condition in conditions: query = query.where(condition) count_query = count_query.where(condition) summary_query = summary_query.where(condition) total_result = await db.execute(count_query) summary_result = await db.execute(summary_query) result = await db.execute( query.order_by(desc(SecurityAccessLog.created_at)) .offset((page - 1) * page_size) .limit(page_size) ) summary_row = summary_result.one() category_query = select(SecurityAccessLog.category, func.count().label("count")).group_by(SecurityAccessLog.category) severity_query = select(SecurityAccessLog.severity, func.count().label("count")).group_by(SecurityAccessLog.severity) top_ip_query = ( select( SecurityAccessLog.client_ip, func.count().label("total_count"), func.count().filter(SecurityAccessLog.severity.in_(("CRITICAL", "HIGH"))).label("high_risk_count"), func.max(SecurityAccessLog.created_at).label("last_seen_at"), func.max(SecurityAccessLog.category).label("category"), ) .where(SecurityAccessLog.client_ip.is_not(None)) .group_by(SecurityAccessLog.client_ip) .order_by(desc("high_risk_count"), desc("total_count"), desc("last_seen_at")) .limit(10) ) endpoint_query = ( select(SecurityAccessLog.path, func.count().label("count")) .group_by(SecurityAccessLog.path) .order_by(desc("count")) .limit(100) ) for condition in conditions: category_query = category_query.where(condition) severity_query = severity_query.where(condition) top_ip_query = top_ip_query.where(condition) endpoint_query = endpoint_query.where(condition) category_result = await db.execute(category_query) severity_result = await db.execute(severity_query) top_ip_result = await db.execute(top_ip_query) endpoint_result = await db.execute(endpoint_query) endpoint_type_counts: dict[str, dict[str, str | int]] = {} for row in endpoint_result.all(): endpoint_type = _security_endpoint_type(row.path) current = endpoint_type_counts.setdefault(endpoint_type, {"count": 0, "sample_path": row.path}) current["count"] = int(current["count"]) + row.count logs = result.scalars().all() user_names = await _security_user_names(db, list(logs)) items = [] for log in logs: ip_location = resolve_ip_location(log.client_ip) items.append( { "id": str(log.id), "method": log.method, "path": log.path, "status_code": log.status_code, "elapsed_ms": round(log.elapsed_ms, 2), "client_ip": log.client_ip, "ip_location": ip_location.location, "ip_country": ip_location.country, "ip_province": ip_location.province, "ip_city": ip_location.city, "ip_isp": ip_location.isp, "user_agent": log.user_agent, "client_type": log.client_type, "client_version": log.client_version, "client_platform": log.client_platform, "build_channel": log.build_channel, "build_commit": log.build_commit, "request_headers": _coerce_request_headers(getattr(log, "request_headers", None)), "request_snapshot": _coerce_request_snapshot(getattr(log, "request_snapshot", None)), "request_id": getattr(log, "request_id", None), "auth_status": log.auth_status, "user_identifier": log.user_identifier, "account_label": _security_account_label(log.auth_status, log.user_identifier, user_names), **_classify_security_access_log(log, ip_location), "created_at": log.created_at.isoformat(), } ) return { "total": total_result.scalar() or 0, "page": page, "page_size": page_size, "summary": { "total_count": summary_row.total_count, "anonymous_count": summary_row.anonymous_count, "invalid_token_count": summary_row.invalid_token_count, "error_count": summary_row.error_count, "unique_ip_count": summary_row.unique_ip_count, "high_risk_count": summary_row.high_risk_count, "server_error_count": summary_row.server_error_count, "category_counts": {str(row.category or "OTHER"): row.count for row in category_result.all()}, "severity_counts": {str(row.severity or "LOW"): row.count for row in severity_result.all()}, "endpoint_type_counts": endpoint_type_counts, "top_ips": [ { "ip": row.client_ip, "location": resolve_ip_location(row.client_ip).location, "category": row.category or "OTHER", "total_count": row.total_count, "high_risk_count": row.high_risk_count, "last_seen_at": row.last_seen_at.isoformat() if row.last_seen_at else None, } for row in top_ip_result.all() ], }, "items": items, } # ═══════════════════════════════════════════ # 新增端点：趋势数据 # ═══════════════════════════════════════════ @router.get("/trends", status_code=status.HTTP_200_OK) async def get_trends( db: AsyncSession = Depends(get_db_session), _=Depends(get_current_user), period: str = Query("24h", pattern="^(24h|7d|30d)$"), ) -> dict: """获取权限检查趋势、周期摘要和归因数据。""" scope = await resolve_monitoring_scope(db, _) if not scope.is_admin and not scope.study_ids: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") now = datetime.now(timezone.utc) bucket_delta, bucket_count = _trend_bucket_config(period) current_bucket = _trend_bucket_time(now, period) start_time = current_bucket - bucket_delta * (bucket_count - 1) current_duration = now - start_time previous_end = start_time previous_start = previous_end - current_duration log_buckets = await _load_trend_log_buckets(db, scope, start_time, now, period) cache_buckets, current_cache, previous_cache = await _load_trend_cache_buckets( db, scope, previous_start, previous_end, start_time, now, period, ) summary = await _load_trend_summary(db, scope, start_time, now) previous_summary = await _load_trend_summary(db, scope, previous_start, previous_end) summary.update(current_cache) previous_summary.update(previous_cache) previous_summary.pop("observed_end_at", None) data_points = [] for index in range(bucket_count): bucket_time = start_time + bucket_delta * index values = log_buckets.get(bucket_time, _empty_trend_bucket()) cache_values = cache_buckets.get(bucket_time, {"cache_hits": 0, "cache_misses": 0}) cache_total = int(cache_values["cache_hits"]) + int(cache_values["cache_misses"]) total = int(values["total"]) data_points.append( { "bucket_time": bucket_time.isoformat(), "sample_state": "partial" if bucket_time == current_bucket else "complete", "total_checks": total, "allowed_checks": int(values["allowed"]), "denied_checks": int(values["denied"]), "avg_elapsed_ms": round(float(values["elapsed_sum"]) / total, 2) if total else 0, "max_elapsed_ms": round(float(values["max_ms"]), 2), "cache_hits": int(cache_values["cache_hits"]), "cache_misses": int(cache_values["cache_misses"]), "cache_hit_rate": round(int(cache_values["cache_hits"]) / cache_total * 100, 1) if cache_total else 0, "cache_sample_available": cache_total > 0, "error_count": 0, } ) return { "period": period, "generated_at": now.isoformat(), "range": { "start_at": start_time.isoformat(), "end_at": now.isoformat(), "previous_start_at": previous_start.isoformat(), "previous_end_at": previous_end.isoformat(), "bucket_seconds": int(bucket_delta.total_seconds()), "bucket_count": bucket_count, "timezone": "UTC", "observed_end_at": summary.pop("observed_end_at"), }, "summary": summary, "previous_summary": previous_summary, "attribution": await _load_trend_attribution(db, scope, start_time, now), "data_points": data_points, } def _trend_bucket_config(period: str) -> tuple[timedelta, int]: return { "24h": (timedelta(hours=1), 24), "7d": (timedelta(hours=6), 28), "30d": (timedelta(days=1), 30), }[period] def _empty_trend_bucket() -> dict[str, float | int]: return {"total": 0, "allowed": 0, "denied": 0, "elapsed_sum": 0.0, "max_ms": 0.0} def _merge_trend_bucket(target: dict[str, float | int], *, total: int, allowed: int, denied: int, elapsed_sum: float, max_ms: float) -> None: target["total"] = int(target["total"]) + total target["allowed"] = int(target["allowed"]) + allowed target["denied"] = int(target["denied"]) + denied target["elapsed_sum"] = float(target["elapsed_sum"]) + elapsed_sum target["max_ms"] = max(float(target["max_ms"]), max_ms) async def _load_trend_log_buckets( db: AsyncSession, scope: MonitoringScope, start_time: datetime, end_time: datetime, period: str, ) -> dict[datetime, dict[str, float | int]]: """PostgreSQL 先按小时聚合，SQLite 测试库使用 Python 分桶。""" buckets: dict[datetime, dict[str, float | int]] = defaultdict(_empty_trend_bucket) dialect_name = db.get_bind().dialect.name if dialect_name == "postgresql": hour_bucket = func.date_trunc("hour", PermissionAccessLog.created_at).label("hour_bucket") query = select( hour_bucket, func.count().label("total"), func.count().filter(PermissionAccessLog.allowed.is_(True)).label("allowed"), func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied"), func.coalesce(func.sum(PermissionAccessLog.elapsed_ms), 0).label("elapsed_sum"), func.coalesce(func.max(PermissionAccessLog.elapsed_ms), 0).label("max_ms"), ).where( PermissionAccessLog.created_at >= start_time, PermissionAccessLog.created_at <= end_time, ).group_by(hour_bucket) result = await db.execute(_apply_monitoring_scope_to_log_query(query, scope)) for row in result.all(): bucket_time = _trend_bucket_time(row.hour_bucket, period) _merge_trend_bucket( buckets[bucket_time], total=int(row.total or 0), allowed=int(row.allowed or 0), denied=int(row.denied or 0), elapsed_sum=float(row.elapsed_sum or 0), max_ms=float(row.max_ms or 0), ) return buckets query = select( PermissionAccessLog.created_at, PermissionAccessLog.allowed, PermissionAccessLog.elapsed_ms, ).where( PermissionAccessLog.created_at >= start_time, PermissionAccessLog.created_at <= end_time, ) result = await db.execute(_apply_monitoring_scope_to_log_query(query, scope)) for created_at, allowed, elapsed_ms in result.all(): bucket_time = _trend_bucket_time(created_at, period) _merge_trend_bucket( buckets[bucket_time], total=1, allowed=1 if allowed else 0, denied=0 if allowed else 1, elapsed_sum=float(elapsed_ms or 0), max_ms=float(elapsed_ms or 0), ) return buckets def _percentile(values: list[float], percentile: float) -> float: if not values: return 0.0 ordered = sorted(values) position = (len(ordered) - 1) * percentile lower = int(position) upper = min(lower + 1, len(ordered) - 1) fraction = position - lower return ordered[lower] + (ordered[upper] - ordered[lower]) * fraction async def _load_trend_summary( db: AsyncSession, scope: MonitoringScope, start_time: datetime, end_time: datetime, ) -> dict: query = select( func.count().label("total"), func.count().filter(PermissionAccessLog.allowed.is_(True)).label("allowed"), func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied"), func.count().filter(PermissionAccessLog.elapsed_ms > 50).label("slow"), func.count(func.distinct(PermissionAccessLog.study_id)).label("studies"), func.count(func.distinct(PermissionAccessLog.user_id)).label("users"), func.count(func.distinct(PermissionAccessLog.endpoint_key)).label("endpoints"), func.coalesce(func.avg(PermissionAccessLog.elapsed_ms), 0).label("avg_ms"), func.coalesce(func.max(PermissionAccessLog.elapsed_ms), 0).label("max_ms"), func.max(PermissionAccessLog.created_at).label("observed_end_at"), ).where( PermissionAccessLog.created_at >= start_time, PermissionAccessLog.created_at <= end_time, ) row = (await db.execute(_apply_monitoring_scope_to_log_query(query, scope))).one() if db.get_bind().dialect.name == "postgresql": percentile_query = select( func.percentile_cont(0.95).within_group(PermissionAccessLog.elapsed_ms) ).where( PermissionAccessLog.created_at >= start_time, PermissionAccessLog.created_at <= end_time, ) percentile_result = await db.execute( _apply_monitoring_scope_to_log_query(percentile_query, scope) ) p95_ms = float(percentile_result.scalar() or 0) else: elapsed_query = select(PermissionAccessLog.elapsed_ms).where( PermissionAccessLog.created_at >= start_time, PermissionAccessLog.created_at <= end_time, ) elapsed_result = await db.execute( _apply_monitoring_scope_to_log_query(elapsed_query, scope) ) p95_ms = _percentile( [float(value) for value in elapsed_result.scalars().all()], 0.95, ) total = int(row.total or 0) denied = int(row.denied or 0) return { "total_checks": total, "allowed_checks": int(row.allowed or 0), "denied_checks": denied, "deny_rate": round(denied / total * 100, 2) if total else 0, "avg_elapsed_ms": round(float(row.avg_ms or 0), 2), "p95_elapsed_ms": round(p95_ms, 2), "max_elapsed_ms": round(float(row.max_ms or 0), 2), "slow_check_count": int(row.slow or 0), "active_study_count": int(row.studies or 0), "active_user_count": int(row.users or 0), "active_endpoint_count": int(row.endpoints or 0), "cache_hits": 0, "cache_misses": 0, "cache_hit_rate": 0, "cache_sample_available": False, "observed_end_at": row.observed_end_at.isoformat() if row.observed_end_at else None, } async def _load_trend_cache_buckets( db: AsyncSession, scope: MonitoringScope, previous_start: datetime, previous_end: datetime, current_start: datetime, current_end: datetime, period: str, ) -> tuple[dict[datetime, dict[str, int]], dict, dict]: empty_summary = {"cache_hits": 0, "cache_misses": 0, "cache_hit_rate": 0, "cache_sample_available": False} if not scope.is_admin: return {}, dict(empty_summary), dict(empty_summary) result = await db.execute( select(PermissionMetricSnapshot).where( PermissionMetricSnapshot.bucket_time >= previous_start, PermissionMetricSnapshot.bucket_time <= current_end, ) ) buckets: dict[datetime, dict[str, int]] = defaultdict(lambda: {"cache_hits": 0, "cache_misses": 0}) current_counts = {"cache_hits": 0, "cache_misses": 0} previous_counts = {"cache_hits": 0, "cache_misses": 0} for snapshot in result.scalars().all(): snapshot_time = snapshot.bucket_time if snapshot_time.tzinfo is None: snapshot_time = snapshot_time.replace(tzinfo=timezone.utc) if snapshot_time >= current_start: current_counts["cache_hits"] += int(snapshot.cache_hits or 0) current_counts["cache_misses"] += int(snapshot.cache_misses or 0) bucket = buckets[_trend_bucket_time(snapshot_time, period)] bucket["cache_hits"] += int(snapshot.cache_hits or 0) bucket["cache_misses"] += int(snapshot.cache_misses or 0) elif snapshot_time < previous_end: previous_counts["cache_hits"] += int(snapshot.cache_hits or 0) previous_counts["cache_misses"] += int(snapshot.cache_misses or 0) def serialize(counts: dict[str, int]) -> dict: total = counts["cache_hits"] + counts["cache_misses"] return { **counts, "cache_hit_rate": round(counts["cache_hits"] / total * 100, 2) if total else 0, "cache_sample_available": total > 0, } return dict(buckets), serialize(current_counts), serialize(previous_counts) async def _load_trend_attribution( db: AsyncSession, scope: MonitoringScope, start_time: datetime, end_time: datetime, ) -> dict: base_conditions = ( PermissionAccessLog.created_at >= start_time, PermissionAccessLog.created_at <= end_time, ) denied_query = select( PermissionAccessLog.endpoint_key, PermissionAccessLog.role, func.count().label("denied_count"), ).where(*base_conditions, PermissionAccessLog.allowed.is_(False)).group_by( PermissionAccessLog.endpoint_key, PermissionAccessLog.role, ).order_by(desc("denied_count")).limit(5) denied_rows = (await db.execute(_apply_monitoring_scope_to_log_query(denied_query, scope))).all() slow_query = select( PermissionAccessLog.endpoint_key, func.count().label("sample_count"), func.count().filter(PermissionAccessLog.elapsed_ms > 50).label("slow_count"), func.coalesce(func.avg(PermissionAccessLog.elapsed_ms), 0).label("avg_ms"), func.coalesce(func.max(PermissionAccessLog.elapsed_ms), 0).label("max_ms"), ).where(*base_conditions).group_by(PermissionAccessLog.endpoint_key).order_by( desc("slow_count"), desc("avg_ms"), desc("sample_count") ).limit(5) slow_rows = (await db.execute(_apply_monitoring_scope_to_log_query(slow_query, scope))).all() client_type = func.coalesce(PermissionAccessLog.client_type, "unknown") client_version = func.coalesce(PermissionAccessLog.client_version, "-") client_platform = func.coalesce(PermissionAccessLog.client_platform, "-") client_query = select( client_type.label("client_type"), client_version.label("client_version"), client_platform.label("client_platform"), func.count().label("total_count"), func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied_count"), func.coalesce(func.avg(PermissionAccessLog.elapsed_ms), 0).label("avg_ms"), ).where(*base_conditions).group_by(client_type, client_version, client_platform).order_by( desc("total_count") ).limit(5) client_rows = (await db.execute(_apply_monitoring_scope_to_log_query(client_query, scope))).all() return { "top_denied": [ {"endpoint_key": row.endpoint_key, "role": row.role, "denied_count": int(row.denied_count)} for row in denied_rows ], "top_slow": [ { "endpoint_key": row.endpoint_key, "sample_count": int(row.sample_count), "slow_count": int(row.slow_count), "avg_elapsed_ms": round(float(row.avg_ms), 2), "max_elapsed_ms": round(float(row.max_ms), 2), } for row in slow_rows ], "client_breakdown": [ { "client_type": row.client_type, "client_version": row.client_version, "client_platform": row.client_platform, "total_count": int(row.total_count), "denied_count": int(row.denied_count), "deny_rate": round(int(row.denied_count) / int(row.total_count) * 100, 2) if row.total_count else 0, "avg_elapsed_ms": round(float(row.avg_ms), 2), } for row in client_rows ], } def _trend_bucket_time(value: datetime, period: str) -> datetime: if value.tzinfo is None: value = value.replace(tzinfo=timezone.utc) if period == "24h": return value.replace(minute=0, second=0, microsecond=0) if period == "7d": return value.replace(hour=(value.hour // 6) * 6, minute=0, second=0, microsecond=0) return value.replace(hour=0, minute=0, second=0, microsecond=0) # ═══════════════════════════════════════════ # 新增端点：被拒绝最多的权限 # ═══════════════════════════════════════════ @router.get("/top-denied", status_code=status.HTTP_200_OK) async def get_top_denied( db: AsyncSession = Depends(get_db_session), _=Depends(get_current_user), days: int = Query(7, ge=1, le=90), limit: int = Query(20, ge=1, le=100), ) -> dict: """获取被拒绝最多的权限""" scope = await resolve_monitoring_scope(db, _) if not scope.is_admin and not scope.study_ids: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") start_time = datetime.now(timezone.utc) - timedelta(days=days) query = ( select( PermissionAccessLog.endpoint_key, PermissionAccessLog.role, func.count().label("denied_count"), ) .where( PermissionAccessLog.created_at >= start_time, PermissionAccessLog.allowed.is_(False), ) .group_by(PermissionAccessLog.endpoint_key, PermissionAccessLog.role) .order_by(desc("denied_count")) .limit(limit) ) result = await db.execute(_apply_monitoring_scope_to_log_query(query, scope)) rows = result.all() return { "days": days, "items": [ { "endpoint_key": row.endpoint_key, "role": row.role, "denied_count": row.denied_count, } for row in rows ], } # ═══════════════════════════════════════════ # 新增端点：IP 属地统计 # ═══════════════════════════════════════════ @router.get("/ip-locations", status_code=status.HTTP_200_OK) async def get_ip_locations( db: AsyncSession = Depends(get_db_session), _=Depends(get_current_user), days: int = Query(7, ge=1, le=90), limit: int = Query(20, ge=1, le=100), all_time: bool = False, ) -> dict: """获取 IP 省市属地统计。""" scope = await resolve_monitoring_scope(db, _) if not scope.is_admin and not scope.study_ids: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") generated_at = datetime.now(timezone.utc) server_location = await resolve_monitoring_server_location() start_time = None if all_time else generated_at - timedelta(days=days) matching_security_request = select(SecurityAccessLog.id).where( PermissionAccessLog.request_id.is_not(None), SecurityAccessLog.request_id == PermissionAccessLog.request_id, ).exists() permission_conditions = [ PermissionAccessLog.ip_address.is_not(None), ~matching_security_request, ] security_conditions = [SecurityAccessLog.client_ip.is_not(None)] if start_time is not None: permission_conditions.insert(0, PermissionAccessLog.created_at >= start_time) security_conditions.insert(0, SecurityAccessLog.created_at >= start_time) security_allowed = case((SecurityAccessLog.status_code < 400, True), else_=False) permission_result = await db.execute( select( PermissionAccessLog.ip_address, PermissionAccessLog.user_id, PermissionAccessLog.allowed, func.count().label("event_count"), func.min(PermissionAccessLog.created_at).label("first_seen_at"), func.max(PermissionAccessLog.created_at).label("last_seen_at"), ) .where(*permission_conditions) .group_by( PermissionAccessLog.ip_address, PermissionAccessLog.user_id, PermissionAccessLog.allowed, ) ) security_result = await db.execute( select( SecurityAccessLog.client_ip, SecurityAccessLog.user_identifier, SecurityAccessLog.auth_status, security_allowed.label("allowed"), SecurityAccessLog.severity, SecurityAccessLog.category, func.count().label("event_count"), func.min(SecurityAccessLog.created_at).label("first_seen_at"), func.max(SecurityAccessLog.created_at).label("last_seen_at"), ) .where(*security_conditions) .group_by( SecurityAccessLog.client_ip, SecurityAccessLog.user_identifier, SecurityAccessLog.auth_status, security_allowed, SecurityAccessLog.severity, SecurityAccessLog.category, ) ) buckets: dict[tuple[str, str, str], dict] = {} all_ip_addresses: set[str] = set() all_user_ids: set[str] = set() located_ip_addresses: set[str] = set() private_ip_addresses: set[str] = set() unknown_ip_addresses: set[str] = set() total_count = 0 allowed_count = 0 denied_count = 0 observed_start_at: datetime | None = None observed_end_at: datetime | None = None resolved_locations: dict[str, IpLocation] = {} resolved_metadata: dict[str, GeoLocationMetadata] = {} def normalize_observed_at(value: datetime | None) -> datetime | None: if value is None: return None return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc) def add_ip_location_row( ip_address: str, user_id: str | uuid.UUID | None, allowed: bool, event_count: int, first_seen_at: datetime | None, last_seen_at: datetime | None, *, source_kind: str, severity: str | None = None, category: str | None = None, auth_status: str | None = None, ) -> None: nonlocal total_count, allowed_count, denied_count, observed_start_at, observed_end_at ip_info = resolved_locations.get(ip_address) if ip_info is None: ip_info = resolve_ip_location(ip_address) resolved_locations[ip_address] = ip_info metadata = resolved_metadata.get(ip_address) if metadata is None: metadata = resolve_geo_location_metadata(ip_info) resolved_metadata[ip_address] = metadata longitude = metadata.longitude latitude = metadata.latitude if metadata.accuracy_level == "private": longitude = server_location.longitude if server_location else None latitude = server_location.latitude if server_location else None private_ip_addresses.add(ip_address) elif longitude is not None and latitude is not None: located_ip_addresses.add(ip_address) else: unknown_ip_addresses.add(ip_address) country = metadata.country or ip_info.country key = (metadata.country_code or country, metadata.region_code or ip_info.province, ip_info.city) location = " / ".join(part for part in [country, ip_info.province, ip_info.city] if part) or ip_info.location or "未知" bucket = buckets.setdefault( key, { "country": country, "country_code": metadata.country_code, "province": ip_info.province, "region_code": metadata.region_code, "city": ip_info.city, "isp": ip_info.isp, "location": location, "longitude": longitude, "latitude": latitude, "accuracy_level": metadata.accuracy_level, "total_count": 0, "allowed_count": 0, "denied_count": 0, "security_event_count": 0, "high_risk_count": 0, "auth_failure_count": 0, "categories": set(), "ip_addresses": set(), "user_ids": set(), "first_seen_at": None, "last_seen_at": None, }, ) bucket["total_count"] += event_count bucket["allowed_count" if allowed else "denied_count"] += event_count if ip_info.isp and bucket["isp"] and bucket["isp"] != ip_info.isp: bucket["isp"] = "多运营商" elif ip_info.isp: bucket["isp"] = ip_info.isp bucket["ip_addresses"].add(ip_address) if user_id: bucket["user_ids"].add(str(user_id)) normalized_first = normalize_observed_at(first_seen_at) normalized_last = normalize_observed_at(last_seen_at) if normalized_first and (bucket["first_seen_at"] is None or normalized_first < bucket["first_seen_at"]): bucket["first_seen_at"] = normalized_first if normalized_last and (bucket["last_seen_at"] is None or normalized_last > bucket["last_seen_at"]): bucket["last_seen_at"] = normalized_last if normalized_first and (observed_start_at is None or normalized_first < observed_start_at): observed_start_at = normalized_first if normalized_last and (observed_end_at is None or normalized_last > observed_end_at): observed_end_at = normalized_last normalized_category = (category or "").upper() normalized_severity = (severity or "").upper() if source_kind == "security" and normalized_category not in {"", "OTHER", "NOT_FOUND_NOISE"}: bucket["security_event_count"] += event_count bucket["categories"].add(normalized_category) if source_kind == "security" and normalized_severity in {"HIGH", "CRITICAL"}: bucket["high_risk_count"] += event_count if source_kind == "security" and not allowed and auth_status in {"INVALID_TOKEN", "ANONYMOUS"}: bucket["auth_failure_count"] += event_count total_count += event_count if allowed: allowed_count += event_count else: denied_count += event_count all_ip_addresses.add(ip_address) if user_id: all_user_ids.add(str(user_id)) for ip_address, user_id, allowed, event_count, first_seen_at, last_seen_at in permission_result.all(): add_ip_location_row( ip_address, user_id, allowed, int(event_count or 0), first_seen_at, last_seen_at, source_kind="permission", ) for client_ip, user_identifier, auth_status, allowed, severity, category, event_count, first_seen_at, last_seen_at in security_result.all(): user_id = None if auth_status == "AUTHENTICATED" and user_identifier: try: user_id = uuid.UUID(user_identifier) except ValueError: user_id = user_identifier add_ip_location_row( client_ip, user_id, bool(allowed), int(event_count or 0), first_seen_at, last_seen_at, source_kind="security", severity=severity, category=category, auth_status=auth_status, ) items = sorted(buckets.values(), key=lambda item: item["total_count"], reverse=True)[:limit] def serialize_item(item: dict) -> dict: total = max(1, int(item["total_count"])) denied_rate = item["denied_count"] / total risk_score = min( 100, round( denied_rate * 60 + min(item["high_risk_count"] * 25, 40) + min(item["auth_failure_count"] * 5, 20) ), ) if item["high_risk_count"] > 0 or risk_score >= 60: risk_level = "high" elif item["denied_count"] > 0 or item["security_event_count"] > 0 or item["auth_failure_count"] > 0: risk_level = "attention" else: risk_level = "normal" risk_reasons = [] if item["high_risk_count"]: risk_reasons.append(f"{item['high_risk_count']} 次高风险安全事件") if item["auth_failure_count"]: risk_reasons.append(f"{item['auth_failure_count']} 次认证失败") if item["denied_count"]: risk_reasons.append(f"拒绝率 {round(denied_rate * 100, 1)}%") return { "country": item["country"], "country_code": item["country_code"], "province": item["province"], "region_code": item["region_code"], "city": item["city"], "isp": item["isp"], "location": item["location"], "longitude": item["longitude"], "latitude": item["latitude"], "accuracy_level": item["accuracy_level"], "total_count": item["total_count"], "allowed_count": item["allowed_count"], "denied_count": item["denied_count"], "denied_rate": round(denied_rate * 100, 1), "unique_ip_count": len(item["ip_addresses"]), "unique_user_count": len(item["user_ids"]), "security_event_count": item["security_event_count"], "high_risk_count": item["high_risk_count"], "risk_score": risk_score, "risk_level": risk_level, "risk_reasons": risk_reasons, "categories": sorted(item["categories"]), "first_seen_at": item["first_seen_at"].isoformat() if item["first_seen_at"] else None, "last_seen_at": item["last_seen_at"].isoformat() if item["last_seen_at"] else None, } effective_start_at = start_time or observed_start_at or generated_at - timedelta( days=settings.MONITORING_ACCESS_LOG_RETENTION_DAYS ) located_count = len(located_ip_addresses) private_count = len(private_ip_addresses) unknown_count = len(unknown_ip_addresses) public_ip_count = located_count + unknown_count timeline_granularity = "hour" if not all_time and days <= 1 else "day" timeline = await get_source_location_timeline( db, start_at=effective_start_at, end_at=generated_at, granularity=timeline_granularity, ) return { "days": None if all_time else days, "generated_at": generated_at.isoformat(), "period": { "mode": "retained_all" if all_time else "rolling", "requested_days": None if all_time else days, "start_at": effective_start_at.isoformat(), "end_at": generated_at.isoformat(), "observed_end_at": observed_end_at.isoformat() if observed_end_at else None, "retention_days": settings.MONITORING_ACCESS_LOG_RETENTION_DAYS, }, "server_location": server_location.to_public_dict() if server_location else None, "data_quality": { "resolver": "ip2region", "located_ip_count": located_count, "private_ip_count": private_count, "unknown_ip_count": unknown_count, "location_coverage_rate": round((located_count / public_ip_count) * 100, 1) if public_ip_count else 100.0, "returned_region_count": len(items), }, "timeline_granularity": timeline_granularity, "timeline_complete_through": generated_at.replace(minute=0, second=0, microsecond=0).isoformat(), "timeline": timeline, "summary": { "total_count": total_count, "allowed_count": allowed_count, "denied_count": denied_count, "unique_ip_count": len(all_ip_addresses), "unique_user_count": len(all_user_ids), "security_event_count": sum(item["security_event_count"] for item in buckets.values()), "high_risk_count": sum(item["high_risk_count"] for item in buckets.values()), }, "items": [serialize_item(item) for item in items], } # ═══════════════════════════════════════════ # 新增端点：实时统计摘要（从DB获取） # ═══════════════════════════════════════════ @router.get("/stats-summary", status_code=status.HTTP_200_OK) async def get_stats_summary( db: AsyncSession = Depends(get_db_session), _=Depends(get_current_user), ) -> dict: """获取基于数据库的统计摘要""" scope = await resolve_monitoring_scope(db, _) if not scope.is_admin and not scope.study_ids: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") now = datetime.now(timezone.utc) today_start = now.replace(hour=0, minute=0, second=0, microsecond=0) hour_ago = now - timedelta(hours=1) # 今日统计 today_query = select( func.count().label("total"), func.count().filter(PermissionAccessLog.allowed.is_(True)).label("allowed"), func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied"), func.coalesce(func.avg(PermissionAccessLog.elapsed_ms), 0).label("avg_ms"), func.coalesce(func.max(PermissionAccessLog.elapsed_ms), 0).label("max_ms"), ).where(PermissionAccessLog.created_at >= today_start) today_result = await db.execute( _apply_monitoring_scope_to_log_query(today_query, scope) ) today = today_result.one() # 最近一小时 hour_query = select( func.count().label("total"), func.count().filter(PermissionAccessLog.allowed.is_(True)).label("allowed"), func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied"), ).where(PermissionAccessLog.created_at >= hour_ago) hour_result = await db.execute( _apply_monitoring_scope_to_log_query(hour_query, scope) ) hour = hour_result.one() # 总记录数 total_query = select(func.count()).select_from(PermissionAccessLog) total_result = await db.execute( _apply_monitoring_scope_to_log_query(total_query, scope) ) total_logs = total_result.scalar() or 0 return { "total_logs": total_logs, "today": { "total_checks": today.total, "allowed_checks": today.allowed, "denied_checks": today.denied, "avg_elapsed_ms": round(float(today.avg_ms), 2), "max_elapsed_ms": round(float(today.max_ms), 2), "allow_rate": round(today.allowed / today.total * 100, 1) if today.total > 0 else 0, "deny_rate": round(today.denied / today.total * 100, 1) if today.total > 0 else 0, }, "last_hour": { "total_checks": hour.total, "allowed_checks": hour.allowed, "denied_checks": hour.denied, "requests_per_minute": round(hour.total / 60, 1), }, }