feat(监控): 完善系统监控、登录状态与访问审计能力
Client Quality Gates / Shared client and Web (push) Has been cancelled
Client Quality Gates / macOS Desktop (push) Has been cancelled

This commit is contained in:
Cheng Zhou
2026-07-10 17:11:24 +08:00
parent 400c9be3a7
commit f11a5c84d9
82 changed files with 10283 additions and 1781 deletions
@@ -0,0 +1,42 @@
"""add access context to permission access logs
Revision ID: 20260709_02
Revises: 20260709_01
Create Date: 2026-07-09 15:45:00.000000
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "20260709_02"
down_revision: Union[str, None] = "20260709_01"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("permission_access_logs", sa.Column("user_agent", sa.String(500), nullable=True))
op.add_column("permission_access_logs", sa.Column("client_type", sa.String(16), nullable=True))
op.add_column("permission_access_logs", sa.Column("client_version", sa.String(32), nullable=True))
op.add_column("permission_access_logs", sa.Column("client_platform", sa.String(16), nullable=True))
op.add_column("permission_access_logs", sa.Column("build_channel", sa.String(16), nullable=True))
op.add_column("permission_access_logs", sa.Column("build_commit", sa.String(64), nullable=True))
op.create_index("ix_perm_log_ip_created", "permission_access_logs", ["ip_address", "created_at"])
op.create_index("ix_perm_log_client_source_created", "permission_access_logs", ["client_type", "created_at"])
def downgrade() -> None:
op.drop_index("ix_perm_log_client_source_created", table_name="permission_access_logs")
op.drop_index("ix_perm_log_ip_created", table_name="permission_access_logs")
for column in (
"build_commit",
"build_channel",
"client_platform",
"client_version",
"client_type",
"user_agent",
):
op.drop_column("permission_access_logs", column)
@@ -0,0 +1,35 @@
"""add request headers to permission access logs
Revision ID: 20260709_03
Revises: 20260709_02
Create Date: 2026-07-09 16:58:00.000000
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision: str = "20260709_03"
down_revision: Union[str, None] = "20260709_02"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
REQUEST_HEADERS_TYPE = sa.JSON().with_variant(
postgresql.JSONB(astext_type=sa.Text()),
"postgresql",
)
def upgrade() -> None:
op.add_column(
"permission_access_logs",
sa.Column("request_headers", REQUEST_HEADERS_TYPE, nullable=True),
)
def downgrade() -> None:
op.drop_column("permission_access_logs", "request_headers")
@@ -0,0 +1,35 @@
"""add request headers to security access logs
Revision ID: 20260709_04
Revises: 20260709_03
Create Date: 2026-07-09 17:30:00.000000
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision: str = "20260709_04"
down_revision: Union[str, None] = "20260709_03"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
REQUEST_HEADERS_TYPE = sa.JSON().with_variant(
postgresql.JSONB(astext_type=sa.Text()),
"postgresql",
)
def upgrade() -> None:
op.add_column(
"security_access_logs",
sa.Column("request_headers", REQUEST_HEADERS_TYPE, nullable=True),
)
def downgrade() -> None:
op.drop_column("security_access_logs", "request_headers")
@@ -0,0 +1,40 @@
"""add request snapshots to access logs
Revision ID: 20260709_05
Revises: 20260709_04
Create Date: 2026-07-09 18:00:00.000000
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision: str = "20260709_05"
down_revision: Union[str, None] = "20260709_04"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
REQUEST_SNAPSHOT_TYPE = sa.JSON().with_variant(
postgresql.JSONB(astext_type=sa.Text()),
"postgresql",
)
def upgrade() -> None:
op.add_column(
"permission_access_logs",
sa.Column("request_snapshot", REQUEST_SNAPSHOT_TYPE, nullable=True),
)
op.add_column(
"security_access_logs",
sa.Column("request_snapshot", REQUEST_SNAPSHOT_TYPE, nullable=True),
)
def downgrade() -> None:
op.drop_column("security_access_logs", "request_snapshot")
op.drop_column("permission_access_logs", "request_snapshot")
@@ -0,0 +1,67 @@
"""add monitoring event identity and persisted security classification
Revision ID: 20260710_01
Revises: 20260709_05
Create Date: 2026-07-10 09:30:00.000000
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "20260710_01"
down_revision: Union[str, None] = "20260709_05"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("permission_access_logs", sa.Column("request_id", sa.String(36), nullable=True))
op.add_column("security_access_logs", sa.Column("request_id", sa.String(36), nullable=True))
op.add_column("security_access_logs", sa.Column("category", sa.String(30), nullable=True))
op.add_column("security_access_logs", sa.Column("severity", sa.String(16), nullable=True))
op.create_index("ix_perm_log_request_id", "permission_access_logs", ["request_id"])
op.create_index("ix_security_log_request_id", "security_access_logs", ["request_id"])
op.create_index("ix_security_log_category_created", "security_access_logs", ["category", "created_at"])
op.create_index("ix_security_log_severity_created", "security_access_logs", ["severity", "created_at"])
op.execute(
"""
UPDATE security_access_logs
SET category = CASE
WHEN lower(path) LIKE '%/.env%' OR lower(path) LIKE '%.git%'
OR lower(path) LIKE '%backup%' OR lower(path) LIKE '%config.php%'
OR lower(path) LIKE '%wp-config%' OR lower(path) LIKE '%database.yml%'
THEN 'PROBE'
WHEN status_code >= 500 THEN 'SERVER_ERROR'
WHEN auth_status = 'INVALID_TOKEN' THEN 'INVALID_TOKEN'
WHEN auth_status = 'ANONYMOUS' AND status_code IN (401, 403) THEN 'ANONYMOUS_API'
WHEN status_code = 404 THEN 'NOT_FOUND_NOISE'
ELSE 'OTHER'
END,
severity = CASE
WHEN lower(path) LIKE '%/.env%' OR lower(path) LIKE '%.git%'
OR lower(path) LIKE '%backup%' OR lower(path) LIKE '%config.php%'
OR lower(path) LIKE '%wp-config%' OR lower(path) LIKE '%database.yml%'
THEN 'CRITICAL'
WHEN status_code >= 500 THEN 'HIGH'
WHEN auth_status = 'INVALID_TOKEN' THEN 'MEDIUM'
WHEN auth_status = 'ANONYMOUS' AND status_code IN (401, 403) THEN 'MEDIUM'
ELSE 'LOW'
END
"""
)
def downgrade() -> None:
op.drop_index("ix_security_log_severity_created", table_name="security_access_logs")
op.drop_index("ix_security_log_category_created", table_name="security_access_logs")
op.drop_index("ix_security_log_request_id", table_name="security_access_logs")
op.drop_index("ix_perm_log_request_id", table_name="permission_access_logs")
op.drop_column("security_access_logs", "severity")
op.drop_column("security_access_logs", "category")
op.drop_column("security_access_logs", "request_id")
op.drop_column("permission_access_logs", "request_id")
@@ -0,0 +1,93 @@
"""add source location hourly snapshots
Revision ID: 20260710_02
Revises: 20260710_01
Create Date: 2026-07-10 15:30:00.000000
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "20260710_02"
down_revision: Union[str, None] = "20260710_01"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"source_location_snapshots",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("bucket_time", sa.DateTime(timezone=True), nullable=False),
sa.Column("ip_hash", sa.String(length=64), nullable=False),
sa.Column("user_hash", sa.String(length=64), nullable=False, server_default=""),
sa.Column("country", sa.String(length=100), nullable=False, server_default=""),
sa.Column("country_code", sa.String(length=16), nullable=False, server_default=""),
sa.Column("province", sa.String(length=100), nullable=False, server_default=""),
sa.Column("region_code", sa.String(length=24), nullable=False, server_default=""),
sa.Column("city", sa.String(length=100), nullable=False, server_default=""),
sa.Column("isp", sa.String(length=160), nullable=False, server_default=""),
sa.Column("location", sa.String(length=320), nullable=False, server_default=""),
sa.Column("longitude", sa.Float(), nullable=True),
sa.Column("latitude", sa.Float(), nullable=True),
sa.Column("accuracy_level", sa.String(length=16), nullable=False, server_default="unknown"),
sa.Column("allowed_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("denied_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("security_event_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("high_risk_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("auth_failure_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("bucket_time", "ip_hash", "user_hash", name="uq_source_location_bucket_identity"),
)
op.create_index("ix_source_location_bucket", "source_location_snapshots", ["bucket_time"])
op.create_index(
"ix_source_location_country_bucket",
"source_location_snapshots",
["country_code", "bucket_time"],
)
op.create_index(
"ix_source_location_risk_bucket",
"source_location_snapshots",
["high_risk_count", "bucket_time"],
)
op.execute(
"""
UPDATE security_access_logs
SET category = CASE
WHEN lower(path) LIKE '%/.env%' OR lower(path) LIKE '%.git%'
OR lower(path) LIKE '%backup%' OR lower(path) LIKE '%config.php%'
OR lower(path) LIKE '%wp-config%' OR lower(path) LIKE '%database.yml%'
THEN 'PROBE'
WHEN status_code >= 500 THEN 'SERVER_ERROR'
WHEN auth_status = 'INVALID_TOKEN' THEN 'INVALID_TOKEN'
WHEN auth_status = 'ANONYMOUS' AND status_code IN (401, 403) THEN 'ANONYMOUS_API'
WHEN status_code = 404 THEN 'NOT_FOUND_NOISE'
ELSE 'OTHER'
END,
severity = CASE
WHEN lower(path) LIKE '%/.env%' OR lower(path) LIKE '%.git%'
OR lower(path) LIKE '%backup%' OR lower(path) LIKE '%config.php%'
OR lower(path) LIKE '%wp-config%' OR lower(path) LIKE '%database.yml%'
THEN 'CRITICAL'
WHEN status_code >= 500 THEN 'HIGH'
WHEN auth_status = 'INVALID_TOKEN' THEN 'MEDIUM'
WHEN auth_status = 'ANONYMOUS' AND status_code IN (401, 403) THEN 'MEDIUM'
ELSE 'LOW'
END
WHERE category = 'ABNORMAL_IP'
"""
)
def downgrade() -> None:
op.drop_index("ix_source_location_risk_bucket", table_name="source_location_snapshots")
op.drop_index("ix_source_location_country_bucket", table_name="source_location_snapshots")
op.drop_index("ix_source_location_bucket", table_name="source_location_snapshots")
op.drop_table("source_location_snapshots")
@@ -0,0 +1,30 @@
"""Remove deployment-specific coordinates from private source snapshots.
Revision ID: 20260710_03
Revises: 20260710_02
"""
from alembic import op
revision = "20260710_03"
down_revision = "20260710_02"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.execute(
"""
UPDATE source_location_snapshots
SET longitude = NULL,
latitude = NULL
WHERE accuracy_level = 'private'
"""
)
def downgrade() -> None:
# The previous coordinates were a hardcoded deployment assumption and
# cannot be restored without reintroducing incorrect data.
pass
@@ -0,0 +1,40 @@
"""Add user login session activity records.
Revision ID: 20260710_04
Revises: 20260710_03
"""
import sqlalchemy as sa
from alembic import op
revision = "20260710_04"
down_revision = "20260710_03"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"user_login_sessions",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("user_id", sa.Uuid(), nullable=False),
sa.Column("client_type", sa.String(length=16), nullable=False, server_default="web"),
sa.Column("client_platform", sa.String(length=32), nullable=True),
sa.Column("client_version", sa.String(length=64), nullable=True),
sa.Column("client_source", sa.String(length=32), nullable=True),
sa.Column("login_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("CURRENT_TIMESTAMP")),
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("CURRENT_TIMESTAMP")),
sa.Column("ended_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("end_reason", sa.String(length=32), nullable=True),
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_user_login_sessions_user_login", "user_login_sessions", ["user_id", "login_at"])
op.create_index("ix_user_login_sessions_last_seen", "user_login_sessions", ["last_seen_at"])
def downgrade() -> None:
op.drop_index("ix_user_login_sessions_last_seen", table_name="user_login_sessions")
op.drop_index("ix_user_login_sessions_user_login", table_name="user_login_sessions")
op.drop_table("user_login_sessions")
+57 -5
View File
@@ -9,7 +9,8 @@ import uuid
from app.core.config import settings
from app.core.login_crypto import create_login_challenge, decrypt_login_payload, get_public_key_pem
from app.core.security import create_access_token, decode_token_allow_expired, oauth2_scheme, verify_password
from app.core.request_context import resolve_ctms_client_type
from app.core.security import create_access_token, decode_token, decode_token_allow_expired, oauth2_scheme, verify_password
from app.core.deps import get_current_user, get_db_session
from app.crud import user as user_crud
from app.models.user import UserStatus
@@ -26,6 +27,12 @@ from app.schemas.email_settings import (
)
from app.schemas.user import Token, UserRead, UserRegisterRequest, UserSelfUpdate, UserUpdate
from app.services import email_service
from app.services.user_login_sessions import (
create_login_session,
end_login_session,
session_id_from_payload,
touch_login_session,
)
from fastapi.responses import FileResponse
@@ -95,7 +102,7 @@ def get_session_policy_for_client_type(client_type: str) -> SessionPolicy:
def get_request_session_client_type(request: Request) -> str:
return normalize_session_client_type(request.headers.get("x-ctms-client-type"))
return normalize_session_client_type(resolve_ctms_client_type(request.headers))
def policy_expires_at(issued_at: datetime, session_start: datetime, policy: SessionPolicy) -> datetime:
@@ -104,8 +111,9 @@ def policy_expires_at(issued_at: datetime, session_start: datetime, policy: Sess
return min(access_expires_at, session_expires_at)
def issue_user_token(db_user, request: Request) -> Token:
async def issue_user_token(db_user, request: Request, db: AsyncSession) -> Token:
session_start = datetime.now(timezone.utc)
session_id = uuid.uuid4()
client_type = get_request_session_client_type(request)
policy = get_session_policy_for_client_type(client_type)
access_token = create_access_token(
@@ -115,6 +123,14 @@ def issue_user_token(db_user, request: Request) -> Token:
max_age_seconds=policy.absolute_max_seconds,
issued_at=session_start,
client_type=client_type,
session_id=str(session_id),
)
await create_login_session(
db,
session_id=session_id,
user_id=db_user.id,
request=request,
login_at=session_start,
)
return Token(access_token=access_token, token_type="bearer")
@@ -284,7 +300,7 @@ async def login_for_access_token(
db_user = await authenticate_encrypted_password(payload, db)
ensure_user_active(db_user)
return issue_user_token(db_user, request)
return await issue_user_token(db_user, request, db)
@router.post("/dev-login", response_model=Token)
@@ -295,7 +311,7 @@ async def dev_login_for_access_token(
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found")
db_user = await authenticate_plain_password(payload, db)
ensure_user_active(db_user)
return issue_user_token(db_user, request)
return await issue_user_token(db_user, request, db)
@router.get("/me", response_model=UserRead)
@@ -339,11 +355,47 @@ async def extend_access_token(
max_age_seconds=policy.absolute_max_seconds,
issued_at=issued_at,
client_type=normalize_session_client_type(payload.get("client_type")),
session_id=str(session_id_from_payload(payload)),
)
expires_at = policy_expires_at(issued_at, session_start, policy)
return ExtendResponse(accessToken=new_token, expiresAt=expires_at)
@router.post("/session/heartbeat")
async def heartbeat_login_session(
request: Request,
token: str = Depends(oauth2_scheme),
current_user=Depends(get_current_user),
db: AsyncSession = Depends(get_db_session),
) -> dict:
session = await touch_login_session(
db,
user_id=current_user.id,
payload=decode_token(token),
request=request,
)
if session is None:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="登录会话已结束")
return {
"status": "online",
"last_seen_at": session.last_seen_at.isoformat(),
}
@router.post("/session/logout", status_code=status.HTTP_204_NO_CONTENT)
async def logout_login_session(
token: str = Depends(oauth2_scheme),
current_user=Depends(get_current_user),
db: AsyncSession = Depends(get_db_session),
) -> Response:
await end_login_session(
db,
user_id=current_user.id,
payload=decode_token(token),
)
return Response(status_code=status.HTTP_204_NO_CONTENT)
@router.patch("/me", response_model=UserRead)
async def update_me(
payload: UserSelfUpdate,
File diff suppressed because it is too large Load Diff
+31 -2
View File
@@ -9,7 +9,8 @@ from app.schemas.common import PaginatedResponse
from app.crud import user as user_crud
from app.crud import member as member_crud
from app.utils.pagination import paginate
from app.schemas.user import UserCreate, UserRead, UserStatus, UserUpdate
from app.schemas.user import UserCreate, UserLoginActivityRead, UserRead, UserStatus, UserUpdate
from app.services.user_login_sessions import get_login_summaries, list_login_activities
router = APIRouter()
@@ -25,7 +26,35 @@ async def list_users(
) -> PaginatedResponse[UserRead]:
users = await user_crud.list_users(db, skip=skip, limit=limit, keyword=keyword, status=user_status)
total_users = await user_crud.count_users(db, keyword=keyword, status=user_status)
return paginate(list(users), total=total_users)
summaries = await get_login_summaries(db, [user.id for user in users])
items = []
for user in users:
summary = summaries.get(user.id)
item = UserRead.model_validate(user).model_dump()
if summary:
item.update(
{
"login_status": summary.status,
"last_login_at": summary.last_login_at,
"last_seen_at": summary.last_seen_at,
"last_client_type": summary.client_type,
"active_session_count": summary.active_session_count,
}
)
items.append(item)
return paginate(items, total=total_users)
@router.get("/{user_id}/login-activities", response_model=list[UserLoginActivityRead])
async def read_user_login_activities(
user_id: uuid.UUID,
limit: int = Query(default=30, ge=1, le=100),
db: AsyncSession = Depends(get_db_session),
current_user=Depends(require_roles(["ADMIN"])),
) -> list[UserLoginActivityRead]:
if not await user_crud.get_by_id(db, user_id):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="用户不存在")
return await list_login_activities(db, user_id=user_id, limit=limit)
@router.post("/", response_model=UserRead, status_code=status.HTTP_201_CREATED)
+12
View File
@@ -33,6 +33,18 @@ class Settings(BaseSettings):
)
IP2REGION_XDB_PATH: Optional[str] = None
IP2REGION_IPV6_XDB_PATH: Optional[str] = None
TRUSTED_PROXY_CIDRS: str = "127.0.0.1/32,::1/128,172.16.0.0/12"
MONITORING_SERVER_PUBLIC_IP: Optional[str] = None
MONITORING_PUBLIC_IP_DISCOVERY_URLS: str = (
"https://api64.ipify.org,https://icanhazip.com"
)
MONITORING_PUBLIC_IP_DISCOVERY_TIMEOUT_SECONDS: float = Field(default=2.5, ge=0.5, le=10)
MONITORING_SERVER_LOCATION_CACHE_SECONDS: int = Field(default=86400, ge=300, le=604800)
MONITORING_ACCESS_LOG_RETENTION_DAYS: int = Field(default=90, ge=7, le=3650)
MONITORING_METRIC_RETENTION_DAYS: int = Field(default=400, ge=30, le=3650)
MONITORING_RETENTION_INTERVAL_SECONDS: int = Field(default=86400, ge=60, le=604800)
USER_LOGIN_ACTIVITY_RETENTION_DAYS: int = Field(default=180, ge=30, le=3650)
USER_SESSION_ONLINE_SECONDS: int = Field(default=300, ge=60, le=3600)
@lru_cache
+13 -3
View File
@@ -286,8 +286,9 @@ def _enqueue_permission_log(
writer = get_log_writer()
if writer:
forwarded = request.headers.get("x-forwarded-for")
ip = forwarded.split(",")[0].strip() if forwarded else (request.client.host if request.client else None)
from app.core.request_context import build_request_audit_context, get_request_audit_context
context = get_request_audit_context() or build_request_audit_context(request)
writer.enqueue({
"study_id": study_id,
"user_id": user_id,
@@ -295,7 +296,16 @@ def _enqueue_permission_log(
"role": role,
"allowed": allowed,
"elapsed_ms": elapsed_ms,
"ip_address": ip,
"ip_address": context.client_ip,
"user_agent": context.user_agent,
"client_type": context.client_type,
"client_version": context.client_version,
"client_platform": context.client_platform,
"build_channel": context.build_channel,
"build_commit": context.build_commit,
"request_headers": context.request_headers,
"request_snapshot": context.request_snapshot,
"request_id": context.request_id,
})
+187 -6
View File
@@ -2,14 +2,19 @@
from __future__ import annotations
import ipaddress
import re
import uuid
from contextvars import ContextVar, Token
from dataclasses import dataclass
from typing import Any
from app.core.config import settings
@dataclass(frozen=True)
class RequestAuditContext:
request_id: str | None = None
client_ip: str | None = None
user_agent: str | None = None
client_type: str | None = None
@@ -17,6 +22,8 @@ class RequestAuditContext:
client_platform: str | None = None
build_channel: str | None = None
build_commit: str | None = None
request_headers: dict[str, str] | None = None
request_snapshot: dict[str, Any] | None = None
_request_audit_context: ContextVar[RequestAuditContext | None] = ContextVar(
@@ -25,11 +32,37 @@ _request_audit_context: ContextVar[RequestAuditContext | None] = ContextVar(
)
_SENSITIVE_TEXT_PATTERN = re.compile(
r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]+|((?:access_)?token|authorization)=([^&\s]+)"
r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]+|((?:access_)?token|authorization|password|passwd|secret|credential|api[-_]?key|session)=([^&\s]+)"
)
_SENSITIVE_HEADER_NAME_PATTERN = re.compile(
r"(?i)(authorization|cookie|set-cookie|token|password|passwd|secret|credential|api[-_]?key|session)"
)
_SAFE_REQUEST_HEADER_NAMES = frozenset(
{
"accept",
"accept-language",
"content-type",
"host",
"origin",
"user-agent",
"x-request-id",
"x-correlation-id",
}
)
_SAFE_REQUEST_HEADER_PREFIXES = ("x-ctms-",)
_SENSITIVE_PARAMETER_NAME_PATTERN = re.compile(
r"(?i)(token|authorization|password|passwd|secret|credential|api[-_]?key|session|"
r"subject|participant|patient|user_?name|full_?name|email|phone|mobile|id_?card|"
r"identity|certificate|contact|address)"
)
_KNOWN_CLIENT_TYPES = frozenset({"web", "desktop"})
_CTMS_CLIENT_SOURCE_TO_TYPE = {
"ctms-web": "web",
"ctms-desktop": "desktop",
}
def _clean_header_value(value: str | None, max_length: int) -> str | None:
def _clean_audit_value(value: str | None, max_length: int) -> str | None:
if value is None:
return None
cleaned = _SENSITIVE_TEXT_PATTERN.sub(lambda m: f"{m.group(1) or m.group(2) + '='}[redacted]", value.strip())
@@ -38,26 +71,174 @@ def _clean_header_value(value: str | None, max_length: int) -> str | None:
return cleaned[:max_length]
def _clean_header_value(value: str | None, max_length: int) -> str | None:
return _clean_audit_value(value, max_length)
def resolve_ctms_client_type(headers: Any) -> str | None:
source = _clean_header_value(headers.get("x-ctms-client-source"), 32)
if source:
mapped_type = _CTMS_CLIENT_SOURCE_TO_TYPE.get(source.strip().lower())
if mapped_type:
return mapped_type
client_type = _clean_header_value(headers.get("x-ctms-client-type"), 16)
if not client_type:
return None
normalized = client_type.strip().lower()
return normalized if normalized in _KNOWN_CLIENT_TYPES else normalized[:16]
def build_sanitized_request_headers(request: Any) -> dict[str, str] | None:
captured: dict[str, str] = {}
for raw_name, raw_value in request.headers.items():
name = str(raw_name).strip().lower()
if not name:
continue
if _SENSITIVE_HEADER_NAME_PATTERN.search(name):
captured[name] = "[redacted]"
continue
if name not in _SAFE_REQUEST_HEADER_NAMES and not name.startswith(_SAFE_REQUEST_HEADER_PREFIXES):
continue
value = _clean_header_value(str(raw_value), 500)
if value:
captured[name] = value
return captured or None
def _sanitize_named_value(raw_name: Any, raw_value: Any, max_length: int = 500) -> dict[str, str] | None:
name = _clean_audit_value(str(raw_name), 120)
if not name:
return None
if _SENSITIVE_PARAMETER_NAME_PATTERN.search(name):
return {"name": name, "value": "[redacted]"}
value = _clean_audit_value(str(raw_value), max_length)
if value is None:
return {"name": name, "value": ""}
return {"name": name, "value": value}
def _query_param_items(request: Any) -> list[dict[str, str]] | None:
query_params = getattr(request, "query_params", None)
if not query_params:
return None
if hasattr(query_params, "multi_items"):
raw_items = query_params.multi_items()
else:
raw_items = query_params.items()
items = [
item
for item in (_sanitize_named_value(name, value) for name, value in raw_items)
if item is not None
][:50]
return items or None
def _sanitized_query_string(items: list[dict[str, str]] | None) -> str | None:
if not items:
return None
value = "&".join(f"{item['name']}={item['value']}" for item in items)
return value[:2000] or None
def _request_client_snapshot(request: Any) -> dict[str, str | int | None] | None:
client = getattr(request, "client", None)
if not client:
return None
host = _clean_audit_value(getattr(client, "host", None), 120)
port = getattr(client, "port", None)
if host is None and port is None:
return None
return {"host": host, "port": port}
def build_request_snapshot(request: Any, *, request_id: str | None = None) -> dict[str, Any]:
headers = getattr(request, "headers", {})
scope = getattr(request, "scope", {}) or {}
url = getattr(request, "url", None)
query_params = _query_param_items(request)
path = getattr(url, "path", None) or scope.get("path")
method = getattr(request, "method", None) or scope.get("method")
snapshot: dict[str, Any] = {
"request_id": request_id,
"method": _clean_audit_value(str(method).upper() if method else None, 12),
"path": _clean_audit_value(path, 500),
"query_string": _sanitized_query_string(query_params),
"query_params": query_params,
"headers": build_sanitized_request_headers(request),
"http_version": _clean_audit_value(scope.get("http_version"), 16),
"scheme": _clean_audit_value(getattr(url, "scheme", None) or scope.get("scheme"), 16),
"client": _request_client_snapshot(request),
"content": {
"type": _clean_audit_value(headers.get("content-type"), 200),
"length": _clean_audit_value(headers.get("content-length"), 32),
},
"body": {
"captured": False,
"reason": "body_not_captured_by_audit_policy",
},
}
return {key: value for key, value in snapshot.items() if value not in (None, {}, [])}
def _parse_ip(value: str | None):
try:
return ipaddress.ip_address((value or "").strip())
except ValueError:
return None
def _trusted_proxy_networks():
networks = []
for value in settings.TRUSTED_PROXY_CIDRS.split(","):
candidate = value.strip()
if not candidate:
continue
try:
networks.append(ipaddress.ip_network(candidate, strict=False))
except ValueError:
continue
return tuple(networks)
def _is_trusted_proxy(value: str | None) -> bool:
address = _parse_ip(value)
return bool(address and any(address in network for network in _trusted_proxy_networks()))
def resolve_client_ip(request: Any) -> str | None:
peer_ip = request.client.host if request.client else None
if not _is_trusted_proxy(peer_ip):
return _clean_header_value(peer_ip, 45)
forwarded = request.headers.get("x-forwarded-for")
if forwarded:
return _clean_header_value(forwarded.split(",")[0].strip(), 45)
chain = [part.strip() for part in forwarded.split(",") if _parse_ip(part)]
chain.append(str(peer_ip))
for candidate in reversed(chain):
if not _is_trusted_proxy(candidate):
return _clean_header_value(candidate, 45)
real_ip = request.headers.get("x-real-ip")
if real_ip:
if _parse_ip(real_ip):
return _clean_header_value(real_ip.strip(), 45)
return _clean_header_value(request.client.host if request.client else None, 45)
return _clean_header_value(peer_ip, 45)
def build_request_audit_context(request: Any) -> RequestAuditContext:
headers = request.headers
request_id = str(uuid.uuid4())
return RequestAuditContext(
request_id=request_id,
client_ip=resolve_client_ip(request),
user_agent=_clean_header_value(headers.get("user-agent"), 500),
client_type=_clean_header_value(headers.get("x-ctms-client-type"), 16),
client_type=resolve_ctms_client_type(headers),
client_version=_clean_header_value(headers.get("x-ctms-client-version"), 32),
client_platform=_clean_header_value(headers.get("x-ctms-client-platform"), 16),
build_channel=_clean_header_value(headers.get("x-ctms-build-channel"), 16),
build_commit=_clean_header_value(headers.get("x-ctms-build-commit"), 64),
request_headers=build_sanitized_request_headers(request),
request_snapshot=build_request_snapshot(request, request_id=request_id),
)
+3
View File
@@ -22,6 +22,7 @@ def create_access_token(
max_age_seconds: Optional[int] = None,
issued_at: Optional[datetime] = None,
client_type: Optional[str] = None,
session_id: Optional[str] = None,
) -> str:
now = issued_at or datetime.now(timezone.utc)
if now.tzinfo is None:
@@ -42,6 +43,8 @@ def create_access_token(
}
if client_type:
to_encode["client_type"] = client_type
if session_id:
to_encode["sid"] = session_id
return jwt.encode(to_encode, settings.JWT_SECRET_KEY, algorithm=ALGORITHM)
+2
View File
@@ -42,6 +42,8 @@ from app.models.permission_access_log import PermissionAccessLog # noqa: F401
from app.models.permission_metric_snapshot import PermissionMetricSnapshot # noqa: F401
from app.models.permission_template import PermissionTemplate, PermissionTemplateVersion # noqa: F401
from app.models.security_access_log import SecurityAccessLog # noqa: F401
from app.models.source_location_snapshot import SourceLocationSnapshot # noqa: F401
from app.models.user_login_session import UserLoginSession # noqa: F401
from app.models.desktop_notification import ( # noqa: F401
DesktopNotificationDelivery,
DesktopNotificationSubscription,
+59 -3
View File
@@ -5,8 +5,9 @@ import time
from contextlib import asynccontextmanager
from collections import defaultdict
from fastapi import FastAPI
from fastapi import Depends, FastAPI
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from sqlalchemy import text
from app.api.v1.router import api_router
@@ -19,15 +20,23 @@ from app.db.session import SessionLocal, engine
from app.services.visit_scheduler import run_daily_lost_visit_job
from app.services.permission_log_writer import start_log_writer, stop_log_writer
from app.services.permission_metric_aggregator import run_hourly_metric_aggregation
from app.services.source_location_aggregator import run_hourly_source_location_aggregation
from app.services.monitoring_server_location import resolve_monitoring_server_location
from app.services.monitoring_retention import run_monitoring_retention
from app.services.security_access_log_writer import (
get_security_log_writer,
start_security_log_writer,
stop_security_log_writer,
)
from app.core.security import decode_token
from app.core.deps import get_db_session
from app.core.request_context import (
build_request_audit_context,
build_request_snapshot,
build_sanitized_request_headers,
get_request_audit_context,
reset_request_audit_context,
resolve_ctms_client_type,
resolve_client_ip,
set_request_audit_context,
)
@@ -42,8 +51,12 @@ setup_config_stats: dict[str, int] = defaultdict(int)
@asynccontextmanager
async def lifespan(_: FastAPI):
stop_event = asyncio.Event()
await resolve_monitoring_server_location()
scheduler_task = asyncio.create_task(run_daily_lost_visit_job(stop_event))
aggregator_task = asyncio.create_task(run_hourly_metric_aggregation(stop_event))
source_location_aggregator_task = asyncio.create_task(
run_hourly_source_location_aggregation(stop_event)
)
await start_log_writer()
await start_security_log_writer()
if settings.ENV == "development":
@@ -52,12 +65,15 @@ async def lifespan(_: FastAPI):
await conn.run_sync(Base.metadata.create_all)
async with SessionLocal() as session:
await ensure_admin_exists(session)
retention_task = asyncio.create_task(run_monitoring_retention(stop_event))
yield
stop_event.set()
await stop_log_writer()
await stop_security_log_writer()
await scheduler_task
await aggregator_task
await source_location_aggregator_task
await retention_task
async def _ensure_legacy_primary_keys(conn) -> None:
@@ -129,19 +145,23 @@ def create_app() -> FastAPI:
"Accept",
"Authorization",
"Content-Type",
"X-CTMS-Client-Source",
"X-CTMS-Client-Type",
"X-CTMS-Client-Version",
"X-CTMS-Client-Platform",
"X-CTMS-Build-Channel",
"X-CTMS-Build-Commit",
"X-Request-ID",
"X-Correlation-ID",
],
expose_headers=["X-Request-ID"],
)
@app.middleware("http")
async def setup_config_monitoring_middleware(request, call_next):
path = request.url.path
is_setup_config_path = "/api/v1/studies/" in path and "/setup-config" in path
should_security_log = path.startswith("/api/")
should_security_log = path.startswith("/api/") and path != "/api/v1/auth/session/heartbeat"
started_at = time.perf_counter()
audit_context = build_request_audit_context(request)
audit_context_token = set_request_audit_context(audit_context)
@@ -167,6 +187,8 @@ def create_app() -> FastAPI:
raise
status_code = int(response.status_code)
if audit_context.request_id:
response.headers["X-Request-ID"] = audit_context.request_id
if is_setup_config_path:
normalized_path = UUID_RE.sub("{study_id}", path)
duration_ms = int((time.perf_counter() - started_at) * 1000)
@@ -229,6 +251,36 @@ def create_app() -> FastAPI:
async def health() -> dict[str, str]:
return {"status": "ok"}
@app.get(
"/readyz",
tags=["health"],
summary="服务就绪检查",
description="验证应用可访问数据库;失败时返回 503。",
)
async def readiness(db=Depends(get_db_session)):
started_at = time.perf_counter()
try:
await db.execute(text("SELECT 1"))
except Exception as exc:
logger.exception("Readiness database check failed")
return JSONResponse(
status_code=503,
content={
"status": "not_ready",
"database": {
"status": "unhealthy",
"error_type": type(exc).__name__,
},
},
)
return {
"status": "ready",
"database": {
"status": "healthy",
"latency_ms": round((time.perf_counter() - started_at) * 1000, 2),
},
}
@app.get(
"/health/setup-config-stats",
tags=["health"],
@@ -271,6 +323,7 @@ def _enqueue_security_access_log(request, path: str, status_code: int, started_a
if not writer:
return
auth_status, user_identifier = _resolve_auth_context(request)
context = get_request_audit_context()
writer.enqueue(
{
"method": request.method,
@@ -279,11 +332,14 @@ def _enqueue_security_access_log(request, path: str, status_code: int, started_a
"elapsed_ms": round((time.perf_counter() - started_at) * 1000, 2),
"client_ip": resolve_client_ip(request),
"user_agent": request.headers.get("user-agent"),
"client_type": request.headers.get("x-ctms-client-type"),
"client_type": resolve_ctms_client_type(request.headers),
"client_version": request.headers.get("x-ctms-client-version"),
"client_platform": request.headers.get("x-ctms-client-platform"),
"build_channel": request.headers.get("x-ctms-build-channel"),
"build_commit": request.headers.get("x-ctms-build-commit"),
"request_headers": context.request_headers if context else build_sanitized_request_headers(request),
"request_snapshot": context.request_snapshot if context else build_request_snapshot(request),
"request_id": context.request_id if context else None,
"auth_status": auth_status,
"user_identifier": user_identifier,
}
+16 -2
View File
@@ -6,13 +6,15 @@ import uuid
from datetime import datetime
from typing import Optional
from sqlalchemy import Boolean, DateTime, Float, ForeignKey, Index, String
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy import JSON, Boolean, DateTime, Float, ForeignKey, Index, String
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.sql import func
from app.db.base_class import Base
JSONB_TYPE = JSON().with_variant(JSONB, "postgresql")
class PermissionAccessLog(Base):
__tablename__ = "permission_access_logs"
@@ -22,6 +24,9 @@ class PermissionAccessLog(Base):
Index("ix_perm_log_endpoint_created", "endpoint_key", "created_at"),
Index("ix_perm_log_created_at", "created_at"),
Index("ix_perm_log_allowed", "allowed", "created_at"),
Index("ix_perm_log_ip_created", "ip_address", "created_at"),
Index("ix_perm_log_client_source_created", "client_type", "created_at"),
Index("ix_perm_log_request_id", "request_id"),
)
id: Mapped[uuid.UUID] = mapped_column(
@@ -38,6 +43,15 @@ class PermissionAccessLog(Base):
allowed: Mapped[bool] = mapped_column(Boolean, nullable=False)
elapsed_ms: Mapped[float] = mapped_column(Float, nullable=False)
ip_address: Mapped[Optional[str]] = mapped_column(String(45), nullable=True)
user_agent: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
client_type: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
client_version: Mapped[Optional[str]] = mapped_column(String(32), nullable=True)
client_platform: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
build_channel: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
build_commit: Mapped[Optional[str]] = mapped_column(String(64), nullable=True)
request_headers: Mapped[Optional[dict]] = mapped_column(JSONB_TYPE, nullable=True)
request_snapshot: Mapped[Optional[dict]] = mapped_column(JSONB_TYPE, nullable=True)
request_id: Mapped[Optional[str]] = mapped_column(String(36), nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
+12 -2
View File
@@ -6,13 +6,15 @@ import uuid
from datetime import datetime
from typing import Optional
from sqlalchemy import DateTime, Float, Index, Integer, String
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy import JSON, DateTime, Float, Index, Integer, String
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.sql import func
from app.db.base_class import Base
JSONB_TYPE = JSON().with_variant(JSONB, "postgresql")
class SecurityAccessLog(Base):
__tablename__ = "security_access_logs"
@@ -21,6 +23,9 @@ class SecurityAccessLog(Base):
Index("ix_security_log_ip_created", "client_ip", "created_at"),
Index("ix_security_log_status_created", "status_code", "created_at"),
Index("ix_security_log_auth_created", "auth_status", "created_at"),
Index("ix_security_log_request_id", "request_id"),
Index("ix_security_log_category_created", "category", "created_at"),
Index("ix_security_log_severity_created", "severity", "created_at"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
@@ -35,8 +40,13 @@ class SecurityAccessLog(Base):
client_platform: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
build_channel: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
build_commit: Mapped[Optional[str]] = mapped_column(String(64), nullable=True)
request_headers: Mapped[Optional[dict]] = mapped_column(JSONB_TYPE, nullable=True)
request_snapshot: Mapped[Optional[dict]] = mapped_column(JSONB_TYPE, nullable=True)
auth_status: Mapped[str] = mapped_column(String(30), nullable=False)
user_identifier: Mapped[Optional[str]] = mapped_column(String(80), nullable=True)
request_id: Mapped[Optional[str]] = mapped_column(String(36), nullable=True)
category: Mapped[Optional[str]] = mapped_column(String(30), nullable=True)
severity: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
@@ -0,0 +1,48 @@
"""Hourly source-location rollup for monitoring analytics."""
from __future__ import annotations
import uuid
from datetime import datetime
from sqlalchemy import DateTime, Float, Index, Integer, String, UniqueConstraint
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.sql import func
from app.db.base_class import Base
class SourceLocationSnapshot(Base):
__tablename__ = "source_location_snapshots"
__table_args__ = (
UniqueConstraint("bucket_time", "ip_hash", "user_hash", name="uq_source_location_bucket_identity"),
Index("ix_source_location_bucket", "bucket_time"),
Index("ix_source_location_country_bucket", "country_code", "bucket_time"),
Index("ix_source_location_risk_bucket", "high_risk_count", "bucket_time"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
bucket_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
ip_hash: Mapped[str] = mapped_column(String(64), nullable=False)
user_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="")
country: Mapped[str] = mapped_column(String(100), nullable=False, default="")
country_code: Mapped[str] = mapped_column(String(16), nullable=False, default="")
province: Mapped[str] = mapped_column(String(100), nullable=False, default="")
region_code: Mapped[str] = mapped_column(String(24), nullable=False, default="")
city: Mapped[str] = mapped_column(String(100), nullable=False, default="")
isp: Mapped[str] = mapped_column(String(160), nullable=False, default="")
location: Mapped[str] = mapped_column(String(320), nullable=False, default="")
longitude: Mapped[float | None] = mapped_column(Float, nullable=True)
latitude: Mapped[float | None] = mapped_column(Float, nullable=True)
accuracy_level: Mapped[str] = mapped_column(String(16), nullable=False, default="unknown")
allowed_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
denied_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
security_event_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
high_risk_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
auth_failure_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
+32
View File
@@ -0,0 +1,32 @@
from __future__ import annotations
import uuid
from datetime import datetime
from sqlalchemy import DateTime, ForeignKey, String, func
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column
from app.db.base_class import Base
class UserLoginSession(Base):
"""Server-side login activity record without storing credentials or raw IPs."""
__tablename__ = "user_login_sessions"
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True)
user_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("users.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
client_type: Mapped[str] = mapped_column(String(16), nullable=False, default="web")
client_platform: Mapped[str | None] = mapped_column(String(32), nullable=True)
client_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
client_source: Mapped[str | None] = mapped_column(String(32), nullable=True)
login_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
end_reason: Mapped[str | None] = mapped_column(String(32), nullable=True)
+19
View File
@@ -6,6 +6,7 @@ from typing import Literal, Optional
from pydantic import BaseModel, ConfigDict, EmailStr, Field, field_validator
UserStatus = Literal["PENDING", "ACTIVE", "REJECTED", "DISABLED"]
LoginStatus = Literal["ONLINE", "OFFLINE"]
PASSWORD_REGEX = re.compile(r"^(?=.*[A-Za-z])(?=.*\d).{8,}$")
@@ -60,6 +61,11 @@ class UserRead(BaseModel):
approved_at: Optional[datetime] = None
approved_by: Optional[uuid.UUID] = None
avatar_url: Optional[str] = None
login_status: LoginStatus = "OFFLINE"
last_login_at: Optional[datetime] = None
last_seen_at: Optional[datetime] = None
last_client_type: Optional[Literal["web", "desktop"]] = None
active_session_count: int = 0
model_config = ConfigDict(from_attributes=True)
@@ -78,6 +84,19 @@ class UserResponse(UserRead):
pass
class UserLoginActivityRead(BaseModel):
id: uuid.UUID
client_type: Literal["web", "desktop"]
client_platform: Optional[str] = None
client_version: Optional[str] = None
login_at: datetime
last_seen_at: datetime
ended_at: Optional[datetime] = None
end_reason: Optional[str] = None
model_config = ConfigDict(from_attributes=True)
class UserSelfUpdate(_PasswordValidator):
full_name: Optional[str] = None
clinical_department: Optional[str] = None
@@ -0,0 +1,175 @@
"""Canonical map metadata for monitoring source locations.
ip2region provides administrative names but no coordinates. This module keeps
the normalization and centroid contract on the server so web and desktop
clients render the same location semantics.
"""
from __future__ import annotations
from dataclasses import dataclass
from app.services.ip_location import IpLocation
@dataclass(frozen=True)
class GeoLocationMetadata:
country: str
country_code: str
region_code: str
longitude: float | None
latitude: float | None
accuracy_level: str
COUNTRY_ALIASES = {
"中国": "China",
"China": "China",
"Mainland China": "China",
"中国香港": "China",
"中国澳门": "China",
"中国台湾": "China",
"美国": "United States",
"United States": "United States",
"United States of America": "United States",
"USA": "United States",
"土耳其": "Türkiye",
"Turkey": "Türkiye",
"Türkiye": "Türkiye",
}
COUNTRY_CODES = {
"China": "CN",
"United States": "US",
"Netherlands": "NL",
"Türkiye": "TR",
"Australia": "AU",
"Japan": "JP",
"Singapore": "SG",
"Germany": "DE",
"France": "FR",
"United Kingdom": "GB",
"Canada": "CA",
"India": "IN",
"Russia": "RU",
}
COUNTRY_CENTROIDS = {
"China": (104.1954, 35.8617),
"United States": (-95.7129, 37.0902),
"Netherlands": (5.2913, 52.1326),
"Türkiye": (35.2433, 38.9637),
"Australia": (133.7751, -25.2744),
"Japan": (138.2529, 36.2048),
"Singapore": (103.8198, 1.3521),
"Germany": (10.4515, 51.1657),
"France": (2.2137, 46.2276),
"United Kingdom": (-3.436, 55.3781),
"Canada": (-106.3468, 56.1304),
"India": (78.9629, 20.5937),
"Russia": (105.3188, 61.524),
}
CHINA_REGION_METADATA = {
"北京市": ("CN-BJ", 116.4074, 39.9042),
"天津市": ("CN-TJ", 117.2008, 39.0842),
"河北省": ("CN-HE", 114.5025, 38.0455),
"山西省": ("CN-SX", 112.5492, 37.857),
"内蒙古自治区": ("CN-NM", 111.6708, 40.8183),
"辽宁省": ("CN-LN", 123.4315, 41.8057),
"吉林省": ("CN-JL", 125.3245, 43.8868),
"黑龙江省": ("CN-HL", 126.6424, 45.7567),
"上海市": ("CN-SH", 121.4737, 31.2304),
"江苏省": ("CN-JS", 118.7633, 32.0617),
"浙江省": ("CN-ZJ", 120.1551, 30.2741),
"安徽省": ("CN-AH", 117.2272, 31.8206),
"福建省": ("CN-FJ", 119.2965, 26.0745),
"江西省": ("CN-JX", 115.8582, 28.682),
"山东省": ("CN-SD", 117.1201, 36.6512),
"河南省": ("CN-HA", 113.6254, 34.7466),
"湖北省": ("CN-HB", 114.3055, 30.5928),
"湖南省": ("CN-HN", 112.9388, 28.2282),
"广东省": ("CN-GD", 113.2644, 23.1291),
"广西壮族自治区": ("CN-GX", 108.3669, 22.817),
"海南省": ("CN-HI", 110.3312, 20.0311),
"重庆": ("CN-CQ", 106.5516, 29.563),
"重庆市": ("CN-CQ", 106.5516, 29.563),
"四川省": ("CN-SC", 104.0665, 30.5723),
"贵州省": ("CN-GZ", 106.6302, 26.647),
"云南省": ("CN-YN", 102.8329, 24.8801),
"西藏自治区": ("CN-XZ", 91.1322, 29.6604),
"陕西省": ("CN-SN", 108.9398, 34.3416),
"甘肃省": ("CN-GS", 103.8343, 36.0611),
"青海省": ("CN-QH", 101.7782, 36.6171),
"宁夏回族自治区": ("CN-NX", 106.2309, 38.4872),
"新疆维吾尔自治区": ("CN-XJ", 87.6168, 43.8256),
"台湾省": ("CN-TW", 121.5654, 25.033),
"香港特别行政区": ("CN-HK", 114.1694, 22.3193),
"澳门特别行政区": ("CN-MO", 113.5439, 22.1987),
}
CITY_CENTROIDS = {
"南京": (118.7969, 32.0603),
"南京市": (118.7969, 32.0603),
"San Jose": (-121.8863, 37.3382),
"South Holland": (4.493, 52.0208),
"Istanbul": (28.9784, 41.0082),
}
def resolve_geo_location_metadata(location: IpLocation) -> GeoLocationMetadata:
if location.location in {"局域网", "本机"}:
return GeoLocationMetadata(
country="",
country_code="PRIVATE",
region_code="PRIVATE",
longitude=None,
latitude=None,
accuracy_level="private",
)
country = COUNTRY_ALIASES.get(location.country, location.country)
country_code = COUNTRY_CODES.get(country, "")
city_coordinate = CITY_CENTROIDS.get(location.city)
if city_coordinate:
return GeoLocationMetadata(
country=country,
country_code=country_code,
region_code="",
longitude=city_coordinate[0],
latitude=city_coordinate[1],
accuracy_level="city",
)
region_metadata = CHINA_REGION_METADATA.get(location.province)
if country in {"China", "中国"} and region_metadata:
region_code, longitude, latitude = region_metadata
return GeoLocationMetadata(
country="China",
country_code="CN",
region_code=region_code,
longitude=longitude,
latitude=latitude,
accuracy_level="region",
)
country_coordinate = COUNTRY_CENTROIDS.get(country)
if country_coordinate:
return GeoLocationMetadata(
country=country,
country_code=country_code,
region_code="",
longitude=country_coordinate[0],
latitude=country_coordinate[1],
accuracy_level="country",
)
return GeoLocationMetadata(
country=country,
country_code=country_code,
region_code="",
longitude=None,
latitude=None,
accuracy_level="unknown",
)
+2
View File
@@ -9,6 +9,7 @@ import ipaddress
import logging
import sys
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
from typing import Optional
@@ -125,5 +126,6 @@ class Ip2RegionResolver:
_resolver = Ip2RegionResolver()
@lru_cache(maxsize=8192)
def resolve_ip_location(ip: str | None) -> IpLocation:
return _resolver.lookup(ip)
@@ -0,0 +1,151 @@
"""监测数据留存清理任务。"""
from __future__ import annotations
import asyncio
import logging
from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import delete
from app.core.config import settings
from app.db.session import SessionLocal
from app.models.permission_access_log import PermissionAccessLog
from app.models.permission_metric_snapshot import PermissionMetricSnapshot
from app.models.security_access_log import SecurityAccessLog
from app.models.source_location_snapshot import SourceLocationSnapshot
from app.models.user_login_session import UserLoginSession
logger = logging.getLogger("ctms.monitoring_retention")
@dataclass
class MonitoringRetentionState:
running: bool = False
last_run_started_at: datetime | None = None
last_success_at: datetime | None = None
last_error_at: datetime | None = None
last_error_type: str | None = None
error_count: int = 0
last_deleted: dict[str, int] = field(default_factory=dict)
total_deleted: dict[str, int] = field(default_factory=dict)
def snapshot(self) -> dict[str, Any]:
return {
"running": self.running,
"access_log_retention_days": settings.MONITORING_ACCESS_LOG_RETENTION_DAYS,
"metric_retention_days": settings.MONITORING_METRIC_RETENTION_DAYS,
"login_activity_retention_days": settings.USER_LOGIN_ACTIVITY_RETENTION_DAYS,
"interval_seconds": settings.MONITORING_RETENTION_INTERVAL_SECONDS,
"last_run_started_at": (
self.last_run_started_at.isoformat() if self.last_run_started_at else None
),
"last_success_at": self.last_success_at.isoformat() if self.last_success_at else None,
"last_error_at": self.last_error_at.isoformat() if self.last_error_at else None,
"last_error_type": self.last_error_type,
"error_count": self.error_count,
"last_deleted": dict(self.last_deleted),
"total_deleted": dict(self.total_deleted),
}
_state = MonitoringRetentionState()
def get_monitoring_retention_status() -> dict[str, Any]:
return _state.snapshot()
def _deleted_count(result: Any) -> int:
rowcount = getattr(result, "rowcount", 0)
return max(0, int(rowcount or 0))
async def purge_expired_monitoring_data(
*, now: datetime | None = None
) -> dict[str, int]:
reference_time = now or datetime.now(timezone.utc)
access_cutoff = reference_time - timedelta(
days=settings.MONITORING_ACCESS_LOG_RETENTION_DAYS
)
metric_cutoff = reference_time - timedelta(
days=settings.MONITORING_METRIC_RETENTION_DAYS
)
login_activity_cutoff = reference_time - timedelta(
days=settings.USER_LOGIN_ACTIVITY_RETENTION_DAYS
)
async with SessionLocal() as session:
permission_result = await session.execute(
delete(PermissionAccessLog).where(PermissionAccessLog.created_at < access_cutoff)
)
security_result = await session.execute(
delete(SecurityAccessLog).where(SecurityAccessLog.created_at < access_cutoff)
)
metric_result = await session.execute(
delete(PermissionMetricSnapshot).where(
PermissionMetricSnapshot.bucket_time < metric_cutoff
)
)
source_location_result = await session.execute(
delete(SourceLocationSnapshot).where(
SourceLocationSnapshot.bucket_time < access_cutoff
)
)
login_session_result = await session.execute(
delete(UserLoginSession).where(UserLoginSession.last_seen_at < login_activity_cutoff)
)
await session.commit()
return {
"permission_access_logs": _deleted_count(permission_result),
"security_access_logs": _deleted_count(security_result),
"permission_metric_snapshots": _deleted_count(metric_result),
"source_location_snapshots": _deleted_count(source_location_result),
"user_login_sessions": _deleted_count(login_session_result),
}
async def run_monitoring_retention_once(
*, now: datetime | None = None
) -> dict[str, int]:
_state.last_run_started_at = now or datetime.now(timezone.utc)
try:
deleted = await purge_expired_monitoring_data(now=now)
except Exception as exc:
_state.last_error_at = datetime.now(timezone.utc)
_state.last_error_type = type(exc).__name__
_state.error_count += 1
raise
_state.last_success_at = datetime.now(timezone.utc)
_state.last_deleted = deleted
for key, count in deleted.items():
_state.total_deleted[key] = _state.total_deleted.get(key, 0) + count
return deleted
async def run_monitoring_retention(stop_event: asyncio.Event) -> None:
logger.info("Monitoring retention task started")
_state.running = True
try:
while not stop_event.is_set():
try:
deleted = await run_monitoring_retention_once()
if any(deleted.values()):
logger.info("Purged expired monitoring data: %s", deleted)
except Exception:
logger.exception("Failed to purge expired monitoring data")
try:
await asyncio.wait_for(
stop_event.wait(),
timeout=settings.MONITORING_RETENTION_INTERVAL_SECONDS,
)
except asyncio.TimeoutError:
continue
finally:
_state.running = False
logger.info("Monitoring retention task stopped")
@@ -0,0 +1,164 @@
"""Resolve the monitoring deployment's public network location.
The map server marker is derived from the deployment's public IP instead of a
frontend or configuration coordinate. An explicit public IP is accepted for
restricted networks; otherwise the public frontend hostname and, finally, a
small set of public-IP discovery endpoints are used. Results are cached so the
monitoring API never performs per-request network discovery.
"""
from __future__ import annotations
import asyncio
import ipaddress
import logging
import socket
import time
from dataclasses import dataclass
from urllib.parse import urlparse
import httpx
from app.core.config import settings
from app.services.geo_location_metadata import resolve_geo_location_metadata
from app.services.ip_location import resolve_ip_location
logger = logging.getLogger("ctms.monitoring_server_location")
@dataclass(frozen=True)
class MonitoringServerLocation:
name: str
longitude: float
latitude: float
accuracy_level: str
resolution_source: str
def to_public_dict(self) -> dict:
return {
"name": self.name,
"longitude": self.longitude,
"latitude": self.latitude,
"accuracy_level": self.accuracy_level,
"resolution_source": self.resolution_source,
}
_cached_location: MonitoringServerLocation | None = None
_cache_initialized = False
_cache_expires_at = 0.0
_cache_lock = asyncio.Lock()
def _normalize_public_ip(value: str | None) -> str | None:
candidate = (value or "").strip().splitlines()[0] if (value or "").strip() else ""
try:
address = ipaddress.ip_address(candidate)
except ValueError:
return None
return str(address) if address.is_global else None
async def _resolve_frontend_public_ip() -> str | None:
hostname = urlparse(settings.FRONTEND_PUBLIC_URL).hostname
if not hostname:
return None
literal_ip = _normalize_public_ip(hostname)
if literal_ip:
return literal_ip
try:
loop = asyncio.get_running_loop()
records = await loop.getaddrinfo(hostname, None, type=socket.SOCK_STREAM)
except (OSError, socket.gaierror):
return None
candidates = []
for _, _, _, _, socket_address in records:
candidate = _normalize_public_ip(str(socket_address[0]))
if candidate and candidate not in candidates:
candidates.append(candidate)
return next((item for item in candidates if ":" not in item), candidates[0] if candidates else None)
async def _discover_public_ip() -> tuple[str | None, str]:
configured_ip = _normalize_public_ip(settings.MONITORING_SERVER_PUBLIC_IP)
if configured_ip:
return configured_ip, "configured_public_ip"
frontend_ip = await _resolve_frontend_public_ip()
if frontend_ip:
return frontend_ip, "frontend_dns"
if settings.ENV == "test":
return None, "unavailable"
urls = [
item.strip()
for item in settings.MONITORING_PUBLIC_IP_DISCOVERY_URLS.split(",")
if item.strip()
]
timeout = httpx.Timeout(settings.MONITORING_PUBLIC_IP_DISCOVERY_TIMEOUT_SECONDS)
async with httpx.AsyncClient(timeout=timeout, follow_redirects=False) as client:
for url in urls:
try:
response = await client.get(url, headers={"Accept": "text/plain"})
response.raise_for_status()
except httpx.HTTPError:
logger.warning("Public IP discovery endpoint unavailable", extra={"endpoint": url})
continue
discovered_ip = _normalize_public_ip(response.text[:128])
if discovered_ip:
return discovered_ip, "public_ip_discovery"
return None, "unavailable"
async def resolve_monitoring_server_location(*, force: bool = False) -> MonitoringServerLocation | None:
global _cached_location, _cache_initialized, _cache_expires_at
now = time.monotonic()
if not force and _cache_initialized and now < _cache_expires_at:
return _cached_location
async with _cache_lock:
now = time.monotonic()
if not force and _cache_initialized and now < _cache_expires_at:
return _cached_location
public_ip, resolution_source = await _discover_public_ip()
location = None
if public_ip:
ip_location = resolve_ip_location(public_ip)
metadata = resolve_geo_location_metadata(ip_location)
if metadata.longitude is not None and metadata.latitude is not None:
country = metadata.country or ip_location.country
name = " / ".join(
part for part in [country, ip_location.province, ip_location.city] if part
) or "公网服务器"
location = MonitoringServerLocation(
name=name,
longitude=metadata.longitude,
latitude=metadata.latitude,
accuracy_level=metadata.accuracy_level,
resolution_source=resolution_source,
)
logger.info(
"Monitoring server location resolved",
extra={"resolution_source": resolution_source, "accuracy_level": metadata.accuracy_level},
)
else:
logger.warning("Deployment public IP has no usable geo coordinates")
else:
logger.warning("Unable to discover the deployment public IP")
_cached_location = location
_cache_initialized = True
cache_seconds = settings.MONITORING_SERVER_LOCATION_CACHE_SECONDS if location else 300
_cache_expires_at = now + cache_seconds
return location
def reset_monitoring_server_location_cache() -> None:
"""Reset process-local state for configuration reloads and tests."""
global _cached_location, _cache_initialized, _cache_expires_at
_cached_location = None
_cache_initialized = False
_cache_expires_at = 0.0
+106 -40
View File
@@ -13,6 +13,7 @@ import asyncio
import logging
import uuid
from datetime import datetime, timezone
from typing import Any
from app.db.session import SessionLocal
from app.models.permission_access_log import PermissionAccessLog
@@ -22,85 +23,150 @@ logger = logging.getLogger("ctms.permission_log_writer")
BATCH_SIZE = 50
FLUSH_INTERVAL = 5.0
QUEUE_MAX_SIZE = 10000
WRITE_ATTEMPTS = 2
_QUEUE_STOP = object()
class PermissionLogWriter:
def __init__(self):
self._queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=QUEUE_MAX_SIZE)
self._queue: asyncio.Queue[dict | object] = asyncio.Queue(maxsize=QUEUE_MAX_SIZE)
self._task: asyncio.Task | None = None
self._stopping = False
self._started_at: datetime | None = None
self._last_success_at: datetime | None = None
self._last_error_at: datetime | None = None
self._last_error_type: str | None = None
self._accepted_entries = 0
self._written_entries = 0
self._dropped_entries = 0
self._failed_batches = 0
self._failed_entries = 0
self._retry_count = 0
def enqueue(self, entry: dict) -> None:
if self._stopping:
self._dropped_entries += 1
logger.warning("Permission log writer is stopping, dropping entry")
return
try:
self._queue.put_nowait(entry)
self._accepted_entries += 1
except asyncio.QueueFull:
self._dropped_entries += 1
logger.warning("Permission log queue full, dropping entry")
async def start(self) -> None:
if self._task and not self._task.done():
return
self._stopping = False
self._started_at = datetime.now(timezone.utc)
self._task = asyncio.create_task(self._flush_loop())
logger.info("PermissionLogWriter started")
async def stop(self) -> None:
if self._task:
self._task.cancel()
try:
await self._task
except asyncio.CancelledError:
pass
await self._drain()
if self._task and not self._task.done():
self._stopping = True
await self._queue.put(_QUEUE_STOP)
await self._task
self._task = None
logger.info("PermissionLogWriter stopped")
async def _flush_loop(self) -> None:
while True:
batch = await self._collect_batch()
batch, should_stop = await self._collect_batch()
if batch:
await self._write_batch(batch)
if should_stop:
break
async def _collect_batch(self) -> list[dict]:
async def _collect_batch(self) -> tuple[list[dict], bool]:
batch: list[dict] = []
try:
first = await asyncio.wait_for(self._queue.get(), timeout=FLUSH_INTERVAL)
if first is _QUEUE_STOP:
return batch, True
batch.append(first)
except asyncio.TimeoutError:
return batch
return batch, False
while len(batch) < BATCH_SIZE:
try:
item = self._queue.get_nowait()
if item is _QUEUE_STOP:
return batch, True
batch.append(item)
except asyncio.QueueEmpty:
break
return batch
return batch, False
async def _write_batch(self, batch: list[dict]) -> None:
try:
async with SessionLocal() as session:
for entry in batch:
log = PermissionAccessLog(
id=uuid.uuid4(),
study_id=entry["study_id"],
user_id=entry["user_id"],
endpoint_key=entry["endpoint_key"],
role=entry["role"],
allowed=entry["allowed"],
elapsed_ms=entry["elapsed_ms"],
ip_address=entry.get("ip_address"),
created_at=entry.get("created_at", datetime.now(timezone.utc)),
)
session.add(log)
await session.commit()
except Exception:
logger.exception("Failed to write permission access log batch (%d entries)", len(batch))
async def _drain(self) -> None:
batch: list[dict] = []
while not self._queue.empty():
async def _write_batch(self, batch: list[dict]) -> bool:
for attempt in range(WRITE_ATTEMPTS):
try:
batch.append(self._queue.get_nowait())
except asyncio.QueueEmpty:
break
if batch:
await self._write_batch(batch)
async with SessionLocal() as session:
for entry in batch:
log = PermissionAccessLog(
id=uuid.uuid4(),
study_id=entry["study_id"],
user_id=entry["user_id"],
endpoint_key=entry["endpoint_key"],
role=entry["role"],
allowed=entry["allowed"],
elapsed_ms=entry["elapsed_ms"],
ip_address=entry.get("ip_address"),
user_agent=entry.get("user_agent"),
client_type=entry.get("client_type"),
client_version=entry.get("client_version"),
client_platform=entry.get("client_platform"),
build_channel=entry.get("build_channel"),
build_commit=entry.get("build_commit"),
request_headers=entry.get("request_headers"),
request_snapshot=entry.get("request_snapshot"),
request_id=entry.get("request_id"),
created_at=entry.get("created_at", datetime.now(timezone.utc)),
)
session.add(log)
await session.commit()
except Exception as exc:
self._last_error_at = datetime.now(timezone.utc)
self._last_error_type = type(exc).__name__
if attempt + 1 < WRITE_ATTEMPTS:
self._retry_count += 1
logger.warning(
"Permission access log batch write failed; retrying (%d entries)",
len(batch),
)
await asyncio.sleep(0)
continue
self._failed_batches += 1
self._failed_entries += len(batch)
logger.exception(
"Failed to write permission access log batch after retries (%d entries)",
len(batch),
)
return False
self._written_entries += len(batch)
self._last_success_at = datetime.now(timezone.utc)
return True
return False
def stats(self) -> dict[str, Any]:
return {
"running": bool(self._task and not self._task.done()),
"stopping": self._stopping,
"queue_size": self._queue.qsize(),
"queue_capacity": self._queue.maxsize,
"accepted_entries": self._accepted_entries,
"written_entries": self._written_entries,
"dropped_entries": self._dropped_entries,
"failed_batches": self._failed_batches,
"failed_entries": self._failed_entries,
"retry_count": self._retry_count,
"started_at": self._started_at.isoformat() if self._started_at else None,
"last_success_at": self._last_success_at.isoformat() if self._last_success_at else None,
"last_error_at": self._last_error_at.isoformat() if self._last_error_at else None,
"last_error_type": self._last_error_type,
}
_writer: PermissionLogWriter | None = None
@@ -6,99 +6,170 @@ import asyncio
import logging
import uuid
from datetime import datetime, timezone
from typing import Any
from app.db.session import SessionLocal
from app.models.security_access_log import SecurityAccessLog
from app.services.ip_location import resolve_ip_location
from app.services.security_events import classify_security_event
logger = logging.getLogger("ctms.security_access_log_writer")
BATCH_SIZE = 100
FLUSH_INTERVAL = 3.0
QUEUE_MAX_SIZE = 20000
WRITE_ATTEMPTS = 2
_QUEUE_STOP = object()
class SecurityAccessLogWriter:
def __init__(self) -> None:
self._queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=QUEUE_MAX_SIZE)
self._queue: asyncio.Queue[dict | object] = asyncio.Queue(maxsize=QUEUE_MAX_SIZE)
self._task: asyncio.Task | None = None
self._stopping = False
self._started_at: datetime | None = None
self._last_success_at: datetime | None = None
self._last_error_at: datetime | None = None
self._last_error_type: str | None = None
self._accepted_entries = 0
self._written_entries = 0
self._dropped_entries = 0
self._failed_batches = 0
self._failed_entries = 0
self._retry_count = 0
def enqueue(self, entry: dict) -> None:
if self._stopping:
self._dropped_entries += 1
logger.warning("Security access log writer is stopping, dropping entry")
return
try:
self._queue.put_nowait(entry)
self._accepted_entries += 1
except asyncio.QueueFull:
self._dropped_entries += 1
logger.warning("Security access log queue full, dropping entry")
async def start(self) -> None:
if self._task and not self._task.done():
return
self._stopping = False
self._started_at = datetime.now(timezone.utc)
self._task = asyncio.create_task(self._flush_loop())
logger.info("SecurityAccessLogWriter started")
async def stop(self) -> None:
if self._task:
self._task.cancel()
try:
await self._task
except asyncio.CancelledError:
pass
await self._drain()
if self._task and not self._task.done():
self._stopping = True
await self._queue.put(_QUEUE_STOP)
await self._task
self._task = None
logger.info("SecurityAccessLogWriter stopped")
async def _flush_loop(self) -> None:
while True:
batch = await self._collect_batch()
batch, should_stop = await self._collect_batch()
if batch:
await self._write_batch(batch)
if should_stop:
break
async def _collect_batch(self) -> list[dict]:
async def _collect_batch(self) -> tuple[list[dict], bool]:
batch: list[dict] = []
try:
first = await asyncio.wait_for(self._queue.get(), timeout=FLUSH_INTERVAL)
if first is _QUEUE_STOP:
return batch, True
batch.append(first)
except asyncio.TimeoutError:
return batch
return batch, False
while len(batch) < BATCH_SIZE:
try:
batch.append(self._queue.get_nowait())
item = self._queue.get_nowait()
if item is _QUEUE_STOP:
return batch, True
batch.append(item)
except asyncio.QueueEmpty:
break
return batch
return batch, False
async def _write_batch(self, batch: list[dict]) -> None:
try:
async with SessionLocal() as session:
for entry in batch:
session.add(
SecurityAccessLog(
id=uuid.uuid4(),
method=entry["method"],
async def _write_batch(self, batch: list[dict]) -> bool:
for attempt in range(WRITE_ATTEMPTS):
try:
async with SessionLocal() as session:
for entry in batch:
classification = classify_security_event(
path=entry["path"],
status_code=entry["status_code"],
elapsed_ms=entry["elapsed_ms"],
client_ip=entry.get("client_ip"),
user_agent=entry.get("user_agent"),
client_type=entry.get("client_type"),
client_version=entry.get("client_version"),
client_platform=entry.get("client_platform"),
build_channel=entry.get("build_channel"),
build_commit=entry.get("build_commit"),
auth_status=entry["auth_status"],
user_identifier=entry.get("user_identifier"),
created_at=entry.get("created_at", datetime.now(timezone.utc)),
ip_location=resolve_ip_location(entry.get("client_ip")),
)
session.add(
SecurityAccessLog(
id=uuid.uuid4(),
method=entry["method"],
path=entry["path"],
status_code=entry["status_code"],
elapsed_ms=entry["elapsed_ms"],
client_ip=entry.get("client_ip"),
user_agent=entry.get("user_agent"),
client_type=entry.get("client_type"),
client_version=entry.get("client_version"),
client_platform=entry.get("client_platform"),
build_channel=entry.get("build_channel"),
build_commit=entry.get("build_commit"),
request_headers=entry.get("request_headers"),
request_snapshot=entry.get("request_snapshot"),
request_id=entry.get("request_id"),
category=classification["category"],
severity=classification["severity"],
auth_status=entry["auth_status"],
user_identifier=entry.get("user_identifier"),
created_at=entry.get("created_at", datetime.now(timezone.utc)),
)
)
await session.commit()
except Exception as exc:
self._last_error_at = datetime.now(timezone.utc)
self._last_error_type = type(exc).__name__
if attempt + 1 < WRITE_ATTEMPTS:
self._retry_count += 1
logger.warning(
"Security access log batch write failed; retrying (%d entries)",
len(batch),
)
await session.commit()
except Exception:
logger.exception("Failed to write security access log batch (%d entries)", len(batch))
await asyncio.sleep(0)
continue
self._failed_batches += 1
self._failed_entries += len(batch)
logger.exception(
"Failed to write security access log batch after retries (%d entries)",
len(batch),
)
return False
self._written_entries += len(batch)
self._last_success_at = datetime.now(timezone.utc)
return True
return False
async def _drain(self) -> None:
batch: list[dict] = []
while not self._queue.empty():
try:
batch.append(self._queue.get_nowait())
except asyncio.QueueEmpty:
break
if batch:
await self._write_batch(batch)
def stats(self) -> dict[str, Any]:
return {
"running": bool(self._task and not self._task.done()),
"stopping": self._stopping,
"queue_size": self._queue.qsize(),
"queue_capacity": self._queue.maxsize,
"accepted_entries": self._accepted_entries,
"written_entries": self._written_entries,
"dropped_entries": self._dropped_entries,
"failed_batches": self._failed_batches,
"failed_entries": self._failed_entries,
"retry_count": self._retry_count,
"started_at": self._started_at.isoformat() if self._started_at else None,
"last_success_at": self._last_success_at.isoformat() if self._last_success_at else None,
"last_error_at": self._last_error_at.isoformat() if self._last_error_at else None,
"last_error_type": self._last_error_type,
}
_writer: SecurityAccessLogWriter | None = None
+37
View File
@@ -0,0 +1,37 @@
"""Security event classification shared by writers and monitoring APIs."""
from __future__ import annotations
from app.services.ip_location import IpLocation
SENSITIVE_PROBE_MARKERS = (
"/.env",
".env",
"/.git",
".git/config",
"backup",
"config.php",
"wp-config",
"database.yml",
)
def classify_security_event(
*,
path: str,
status_code: int,
auth_status: str,
ip_location: IpLocation | None = None,
) -> dict[str, str]:
normalized_path = (path or "").lower()
if any(marker in normalized_path for marker in SENSITIVE_PROBE_MARKERS):
return {"category": "PROBE", "severity": "CRITICAL"}
if status_code >= 500:
return {"category": "SERVER_ERROR", "severity": "HIGH"}
if auth_status == "INVALID_TOKEN":
return {"category": "INVALID_TOKEN", "severity": "MEDIUM"}
if auth_status == "ANONYMOUS" and status_code in {401, 403}:
return {"category": "ANONYMOUS_API", "severity": "MEDIUM"}
if status_code == 404:
return {"category": "NOT_FOUND_NOISE", "severity": "LOW"}
return {"category": "OTHER", "severity": "LOW"}
@@ -0,0 +1,272 @@
"""Hourly source-location aggregation and timeline reads."""
from __future__ import annotations
import asyncio
import hashlib
import hmac
import logging
import uuid
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from typing import Literal
from sqlalchemy import delete, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import settings
from app.db.session import SessionLocal
from app.models.permission_access_log import PermissionAccessLog
from app.models.security_access_log import SecurityAccessLog
from app.models.source_location_snapshot import SourceLocationSnapshot
from app.services.geo_location_metadata import resolve_geo_location_metadata
from app.services.ip_location import resolve_ip_location
logger = logging.getLogger("ctms.source_location_aggregator")
def _normalize_datetime(value: datetime) -> datetime:
return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc)
def _identity_hash(kind: str, value: str | None) -> str:
if not value:
return ""
payload = f"source-location:{kind}:{value}".encode("utf-8")
return hmac.new(settings.JWT_SECRET_KEY.encode("utf-8"), payload, hashlib.sha256).hexdigest()
async def aggregate_source_location_hour(bucket_start: datetime, bucket_end: datetime) -> int:
bucket_start = _normalize_datetime(bucket_start)
bucket_end = _normalize_datetime(bucket_end)
async with SessionLocal() as session:
matching_security_request = select(SecurityAccessLog.id).where(
PermissionAccessLog.request_id.is_not(None),
SecurityAccessLog.request_id == PermissionAccessLog.request_id,
).exists()
permission_rows = (
await session.execute(
select(
PermissionAccessLog.ip_address,
PermissionAccessLog.user_id,
func.count().filter(PermissionAccessLog.allowed.is_(True)).label("allowed_count"),
func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied_count"),
func.min(PermissionAccessLog.created_at).label("first_seen_at"),
func.max(PermissionAccessLog.created_at).label("last_seen_at"),
)
.where(
PermissionAccessLog.created_at >= bucket_start,
PermissionAccessLog.created_at < bucket_end,
PermissionAccessLog.ip_address.is_not(None),
~matching_security_request,
)
.group_by(PermissionAccessLog.ip_address, PermissionAccessLog.user_id)
)
).all()
security_rows = (
await session.execute(
select(
SecurityAccessLog.client_ip,
SecurityAccessLog.user_identifier,
func.count().filter(SecurityAccessLog.status_code < 400).label("allowed_count"),
func.count().filter(SecurityAccessLog.status_code >= 400).label("denied_count"),
func.count()
.filter(
SecurityAccessLog.category.is_not(None),
~SecurityAccessLog.category.in_(("OTHER", "NOT_FOUND_NOISE")),
)
.label("security_event_count"),
func.count()
.filter(SecurityAccessLog.severity.in_(("HIGH", "CRITICAL")))
.label("high_risk_count"),
func.count()
.filter(
SecurityAccessLog.status_code >= 400,
SecurityAccessLog.auth_status.in_(("INVALID_TOKEN", "ANONYMOUS")),
)
.label("auth_failure_count"),
func.min(SecurityAccessLog.created_at).label("first_seen_at"),
func.max(SecurityAccessLog.created_at).label("last_seen_at"),
)
.where(
SecurityAccessLog.created_at >= bucket_start,
SecurityAccessLog.created_at < bucket_end,
SecurityAccessLog.client_ip.is_not(None),
)
.group_by(SecurityAccessLog.client_ip, SecurityAccessLog.user_identifier)
)
).all()
aggregates: dict[tuple[str, str], dict] = {}
def merge_row(
ip_address: str,
user_identity: str,
allowed_count: int,
denied_count: int,
first_seen_at: datetime,
last_seen_at: datetime,
*,
security_event_count: int = 0,
high_risk_count: int = 0,
auth_failure_count: int = 0,
) -> None:
key = (ip_address, user_identity)
row = aggregates.setdefault(
key,
{
"ip_address": ip_address,
"user_identity": user_identity,
"allowed_count": 0,
"denied_count": 0,
"security_event_count": 0,
"high_risk_count": 0,
"auth_failure_count": 0,
"first_seen_at": _normalize_datetime(first_seen_at),
"last_seen_at": _normalize_datetime(last_seen_at),
},
)
row["allowed_count"] += int(allowed_count or 0)
row["denied_count"] += int(denied_count or 0)
row["security_event_count"] += int(security_event_count or 0)
row["high_risk_count"] += int(high_risk_count or 0)
row["auth_failure_count"] += int(auth_failure_count or 0)
row["first_seen_at"] = min(row["first_seen_at"], _normalize_datetime(first_seen_at))
row["last_seen_at"] = max(row["last_seen_at"], _normalize_datetime(last_seen_at))
for ip_address, user_id, allowed, denied, first_seen, last_seen in permission_rows:
merge_row(str(ip_address), str(user_id or ""), allowed, denied, first_seen, last_seen)
for ip_address, user_identifier, allowed, denied, security_events, high_risk, auth_failures, first_seen, last_seen in security_rows:
merge_row(
str(ip_address),
str(user_identifier or ""),
allowed,
denied,
first_seen,
last_seen,
security_event_count=security_events,
high_risk_count=high_risk,
auth_failure_count=auth_failures,
)
await session.execute(
delete(SourceLocationSnapshot).where(SourceLocationSnapshot.bucket_time == bucket_start)
)
for row in aggregates.values():
ip_info = resolve_ip_location(row["ip_address"])
metadata = resolve_geo_location_metadata(ip_info)
longitude = metadata.longitude
latitude = metadata.latitude
country = metadata.country or ip_info.country
location = " / ".join(
part for part in [country, ip_info.province, ip_info.city] if part
) or ip_info.location or "未知"
session.add(
SourceLocationSnapshot(
id=uuid.uuid4(),
bucket_time=bucket_start,
ip_hash=_identity_hash("ip", row["ip_address"]),
user_hash=_identity_hash("user", row["user_identity"]),
country=country,
country_code=metadata.country_code,
province=ip_info.province,
region_code=metadata.region_code,
city=ip_info.city,
isp=ip_info.isp,
location=location,
longitude=longitude,
latitude=latitude,
accuracy_level=metadata.accuracy_level,
allowed_count=row["allowed_count"],
denied_count=row["denied_count"],
security_event_count=row["security_event_count"],
high_risk_count=row["high_risk_count"],
auth_failure_count=row["auth_failure_count"],
first_seen_at=row["first_seen_at"],
last_seen_at=row["last_seen_at"],
)
)
await session.commit()
return len(aggregates)
async def get_source_location_timeline(
db: AsyncSession,
*,
start_at: datetime,
end_at: datetime,
granularity: Literal["hour", "day"],
) -> list[dict]:
rows = (
await db.execute(
select(SourceLocationSnapshot)
.where(
SourceLocationSnapshot.bucket_time >= start_at,
SourceLocationSnapshot.bucket_time < end_at,
)
.order_by(SourceLocationSnapshot.bucket_time)
)
).scalars().all()
buckets: dict[datetime, dict] = defaultdict(
lambda: {
"allowed_count": 0,
"denied_count": 0,
"security_event_count": 0,
"high_risk_count": 0,
"ip_hashes": set(),
"user_hashes": set(),
}
)
for row in rows:
bucket = _normalize_datetime(row.bucket_time)
if granularity == "day":
bucket = bucket.replace(hour=0, minute=0, second=0, microsecond=0)
else:
bucket = bucket.replace(minute=0, second=0, microsecond=0)
item = buckets[bucket]
item["allowed_count"] += row.allowed_count
item["denied_count"] += row.denied_count
item["security_event_count"] += row.security_event_count
item["high_risk_count"] += row.high_risk_count
item["ip_hashes"].add(row.ip_hash)
if row.user_hash:
item["user_hashes"].add(row.user_hash)
return [
{
"bucket_time": bucket.isoformat(),
"total_count": item["allowed_count"] + item["denied_count"],
"allowed_count": item["allowed_count"],
"denied_count": item["denied_count"],
"security_event_count": item["security_event_count"],
"high_risk_count": item["high_risk_count"],
"unique_ip_count": len(item["ip_hashes"]),
"unique_user_count": len(item["user_hashes"]),
}
for bucket, item in sorted(buckets.items())
]
async def run_hourly_source_location_aggregation(stop_event: asyncio.Event) -> None:
logger.info("Source location aggregator started")
now = datetime.now(timezone.utc)
completed_hour = now.replace(minute=0, second=0, microsecond=0)
try:
await aggregate_source_location_hour(completed_hour - timedelta(hours=1), completed_hour)
except Exception:
logger.exception("Failed to backfill source location snapshot for %s", completed_hour)
while not stop_event.is_set():
now = datetime.now(timezone.utc)
next_hour = now.replace(minute=0, second=0, microsecond=0) + timedelta(hours=1)
try:
await asyncio.wait_for(stop_event.wait(), timeout=(next_hour - now).total_seconds())
break
except asyncio.TimeoutError:
pass
try:
await aggregate_source_location_hour(next_hour - timedelta(hours=1), next_hour)
except Exception:
logger.exception("Failed to aggregate source locations for %s", next_hour)
logger.info("Source location aggregator stopped")
+198
View File
@@ -0,0 +1,198 @@
"""Server-authoritative login session activity for the admin account list."""
from __future__ import annotations
import uuid
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Any, Iterable
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import settings
from app.models.user_login_session import UserLoginSession
@dataclass(frozen=True)
class UserLoginSummary:
status: str = "OFFLINE"
last_login_at: datetime | None = None
last_seen_at: datetime | None = None
client_type: str | None = None
active_session_count: int = 0
def _now() -> datetime:
return datetime.now(timezone.utc)
def _as_utc(value: datetime) -> datetime:
return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc)
def _text_header(headers: Any, name: str, limit: int) -> str | None:
value = (headers.get(name) or "").strip()
return value[:limit] or None
def session_id_from_payload(payload: dict[str, Any]) -> uuid.UUID:
raw_session_id = payload.get("sid")
try:
return uuid.UUID(str(raw_session_id))
except (TypeError, ValueError):
# Tokens issued before session tracking are mapped to a deterministic
# legacy session without persisting the token or a token-derived value.
legacy_key = ":".join(
[
str(payload.get("sub") or ""),
str(payload.get("orig_iat") or payload.get("iat") or ""),
str(payload.get("client_type") or "web"),
]
)
return uuid.uuid5(uuid.NAMESPACE_URL, f"ctms:legacy-session:{legacy_key}")
def session_client_type(payload: dict[str, Any], headers: Any) -> str:
from_payload = (payload.get("client_type") or "").strip().lower()
from_header = (headers.get("x-ctms-client-type") or "").strip().lower()
return "desktop" if "desktop" in {from_payload, from_header} else "web"
async def create_login_session(
db: AsyncSession,
*,
session_id: uuid.UUID,
user_id: uuid.UUID,
request: Any,
login_at: datetime | None = None,
) -> UserLoginSession:
occurred_at = login_at or _now()
session = UserLoginSession(
id=session_id,
user_id=user_id,
client_type=session_client_type({}, request.headers),
client_platform=_text_header(request.headers, "x-ctms-client-platform", 32),
client_version=_text_header(request.headers, "x-ctms-client-version", 64),
client_source=_text_header(request.headers, "x-ctms-client-source", 32),
login_at=occurred_at,
last_seen_at=occurred_at,
)
db.add(session)
await db.commit()
await db.refresh(session)
return session
async def touch_login_session(
db: AsyncSession,
*,
user_id: uuid.UUID,
payload: dict[str, Any],
request: Any,
) -> UserLoginSession | None:
session_id = session_id_from_payload(payload)
session = await db.get(UserLoginSession, session_id)
now = _now()
if session is None:
session = UserLoginSession(
id=session_id,
user_id=user_id,
client_type=session_client_type(payload, request.headers),
client_platform=_text_header(request.headers, "x-ctms-client-platform", 32),
client_version=_text_header(request.headers, "x-ctms-client-version", 64),
client_source=_text_header(request.headers, "x-ctms-client-source", 32),
login_at=now,
last_seen_at=now,
)
db.add(session)
elif session.user_id != user_id or session.ended_at is not None:
return None
else:
session.last_seen_at = now
session.client_platform = _text_header(request.headers, "x-ctms-client-platform", 32) or session.client_platform
session.client_version = _text_header(request.headers, "x-ctms-client-version", 64) or session.client_version
await db.commit()
await db.refresh(session)
return session
async def end_login_session(
db: AsyncSession,
*,
user_id: uuid.UUID,
payload: dict[str, Any],
reason: str = "logout",
) -> bool:
session_id = session_id_from_payload(payload)
result = await db.execute(
update(UserLoginSession)
.where(
UserLoginSession.id == session_id,
UserLoginSession.user_id == user_id,
UserLoginSession.ended_at.is_(None),
)
.values(ended_at=_now(), end_reason=reason, last_seen_at=_now())
)
await db.commit()
return bool(result.rowcount)
async def get_login_summaries(
db: AsyncSession,
user_ids: Iterable[uuid.UUID],
) -> dict[uuid.UUID, UserLoginSummary]:
ids = list(user_ids)
if not ids:
return {}
rows = (
await db.execute(
select(UserLoginSession)
.where(UserLoginSession.user_id.in_(ids))
.order_by(UserLoginSession.user_id, UserLoginSession.login_at.desc())
)
).scalars().all()
cutoff = _now() - timedelta(seconds=settings.USER_SESSION_ONLINE_SECONDS)
summaries: dict[uuid.UUID, UserLoginSummary] = {}
mutable: dict[uuid.UUID, dict[str, Any]] = {}
for row in rows:
current = mutable.setdefault(
row.user_id,
{
"last_login_at": row.login_at,
"last_seen_at": row.last_seen_at,
"client_type": row.client_type,
"active_session_count": 0,
},
)
row_last_seen_at = _as_utc(row.last_seen_at)
if row_last_seen_at and (
current["last_seen_at"] is None or row_last_seen_at > _as_utc(current["last_seen_at"])
):
current["last_seen_at"] = row_last_seen_at
if row.ended_at is None and row_last_seen_at >= cutoff:
current["active_session_count"] += 1
for user_id, item in mutable.items():
summaries[user_id] = UserLoginSummary(
status="ONLINE" if item["active_session_count"] else "OFFLINE",
last_login_at=item["last_login_at"],
last_seen_at=item["last_seen_at"],
client_type=item["client_type"],
active_session_count=item["active_session_count"],
)
return summaries
async def list_login_activities(
db: AsyncSession,
*,
user_id: uuid.UUID,
limit: int,
) -> list[UserLoginSession]:
result = await db.execute(
select(UserLoginSession)
.where(UserLoginSession.user_id == user_id)
.order_by(UserLoginSession.login_at.desc())
.limit(limit)
)
return list(result.scalars().all())
+4 -6
View File
@@ -246,7 +246,7 @@ async def test_project_member_can_read_own_effective_permissions(db_session: Asy
@pytest.mark.asyncio
async def test_project_pm_monitoring_scope_is_limited_to_own_projects(db_session: AsyncSession):
async def test_project_pm_cannot_access_system_monitoring(db_session: AsyncSession):
pm_id = uuid.uuid4()
own_study_id = uuid.uuid4()
other_study_id = uuid.uuid4()
@@ -256,12 +256,10 @@ async def test_project_pm_monitoring_scope_is_limited_to_own_projects(db_session
await _seed_member(db_session, own_study_id, pm_id, "PM")
await db_session.commit()
scope = await resolve_monitoring_scope(db_session, UserStub(id=pm_id))
with pytest.raises(HTTPException) as exc_info:
await resolve_monitoring_scope(db_session, UserStub(id=pm_id))
assert scope.is_admin is False
assert scope.study_ids == {own_study_id}
assert scope.can_access_study(own_study_id)
assert not scope.can_access_study(other_study_id)
assert exc_info.value.status_code == 403
@pytest.mark.asyncio
+4
View File
@@ -54,6 +54,9 @@ class _DummyTask:
def cancel(self):
self.cancelled = True
def done(self):
return False
def __await__(self):
async def _wait():
self.awaited = True
@@ -106,6 +109,7 @@ async def test_development_startup_does_not_seed_business_data(monkeypatch):
monkeypatch.setattr(main.settings, "ENV", "development")
monkeypatch.setattr(main, "ensure_admin_exists", ensure_admin_exists)
monkeypatch.setattr(main, "run_daily_lost_visit_job", fake_scheduler)
monkeypatch.setattr(main, "resolve_monitoring_server_location", AsyncMock(return_value=None))
monkeypatch.setattr(main.asyncio, "create_task", fake_create_task)
monkeypatch.setattr(main, "engine", _DummyEngine())
monkeypatch.setattr(main, "SessionLocal", fake_session_local)
+145
View File
@@ -4,6 +4,7 @@ import pytest
from app.core.request_context import (
RequestAuditContext,
build_request_audit_context,
reset_request_audit_context,
set_request_audit_context,
)
@@ -22,6 +23,48 @@ class FakeAsyncSession:
self.flushed = True
class FakeClient:
def __init__(self, host: str = "10.0.0.8") -> None:
self.host = host
self.port = 54321
class FakeUrl:
scheme = "https"
path = "/api/v1/audit"
query = "token=secret-token&view=all&subject_name=Alice"
class FakeQueryParams:
def __init__(self) -> None:
self._items = [
("token", "secret-token"),
("view", "all"),
("subject_name", "Alice"),
]
def __bool__(self) -> bool:
return True
def multi_items(self):
return list(self._items)
class FakeRequest:
def __init__(self, headers: dict[str, str], *, client_host: str = "10.0.0.8") -> None:
self.headers = headers
self.client = FakeClient(client_host)
self.method = "GET"
self.url = FakeUrl()
self.query_params = FakeQueryParams()
self.scope = {
"http_version": "1.1",
"scheme": "https",
"path": "/api/v1/audit",
"method": "GET",
}
@pytest.mark.asyncio
async def test_audit_log_action_captures_request_access_context():
context_token = set_request_audit_context(
@@ -60,3 +103,105 @@ async def test_audit_log_action_captures_request_access_context():
assert log.client_platform == "macos"
assert log.build_channel == "release"
assert log.build_commit == "abcdef1"
def test_request_audit_context_captures_sanitized_headers():
context = build_request_audit_context(
FakeRequest(
{
"user-agent": "Chrome",
"referer": "https://ctms.local/audit?token=secret-token",
"authorization": "Bearer secret-token",
"cookie": "sid=secret",
"x-ctms-client-source": "ctms-web",
"x-ctms-client-type": "web",
"x-ctms-client-version": "0.1.0",
"x-request-id": "req-123",
"x-custom-secret": "hidden",
"x-random-debug": "skip-me",
}
)
)
assert context.user_agent == "Chrome"
assert context.client_type == "web"
assert context.client_ip == "10.0.0.8"
assert uuid.UUID(context.request_id)
assert context.request_headers == {
"authorization": "[redacted]",
"cookie": "[redacted]",
"user-agent": "Chrome",
"x-ctms-client-source": "ctms-web",
"x-ctms-client-type": "web",
"x-ctms-client-version": "0.1.0",
"x-custom-secret": "[redacted]",
"x-request-id": "req-123",
}
assert context.request_snapshot is not None
assert context.request_snapshot["method"] == "GET"
assert context.request_snapshot["request_id"] == context.request_id
assert context.request_snapshot["path"] == "/api/v1/audit"
assert context.request_snapshot["query_string"] == (
"token=[redacted]&view=all&subject_name=[redacted]"
)
assert context.request_snapshot["query_params"] == [
{"name": "token", "value": "[redacted]"},
{"name": "view", "value": "all"},
{"name": "subject_name", "value": "[redacted]"},
]
assert context.request_snapshot["headers"]["authorization"] == "[redacted]"
assert context.request_snapshot["headers"]["cookie"] == "[redacted]"
assert context.request_snapshot["headers"]["x-ctms-client-source"] == "ctms-web"
assert "referer" not in context.request_snapshot["headers"]
assert "x-random-debug" not in context.request_snapshot["headers"]
assert context.request_snapshot["client"] == {"host": "10.0.0.8", "port": 54321}
assert context.request_snapshot["body"] == {
"captured": False,
"reason": "body_not_captured_by_audit_policy",
}
def test_request_audit_context_prefers_explicit_ctms_client_source():
context = build_request_audit_context(
FakeRequest(
{
"x-ctms-client-source": "ctms-desktop",
"x-ctms-client-type": "web",
}
)
)
assert context.client_type == "desktop"
assert context.request_headers == {
"x-ctms-client-source": "ctms-desktop",
"x-ctms-client-type": "web",
}
def test_request_audit_context_uses_proxy_observed_ip_instead_of_spoofed_forwarded_prefix(monkeypatch):
monkeypatch.setattr("app.core.request_context.settings.TRUSTED_PROXY_CIDRS", "10.0.0.0/8")
context = build_request_audit_context(
FakeRequest(
{
"x-real-ip": "203.0.113.20",
"x-forwarded-for": "1.2.3.4, 203.0.113.20",
}
)
)
assert context.client_ip == "203.0.113.20"
def test_request_audit_context_ignores_forwarded_headers_from_untrusted_peer(monkeypatch):
monkeypatch.setattr("app.core.request_context.settings.TRUSTED_PROXY_CIDRS", "10.0.0.0/8")
context = build_request_audit_context(
FakeRequest(
{
"x-real-ip": "203.0.113.20",
"x-forwarded-for": "1.2.3.4",
},
client_host="198.51.100.8",
)
)
assert context.client_ip == "198.51.100.8"
+210
View File
@@ -0,0 +1,210 @@
from __future__ import annotations
import asyncio
import uuid
from datetime import datetime, timedelta, timezone
import pytest
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from app.models.permission_access_log import PermissionAccessLog
from app.models.permission_metric_snapshot import PermissionMetricSnapshot
from app.models.security_access_log import SecurityAccessLog
from app.models.study import Study
from app.models.user import User, UserStatus
from app.services import monitoring_retention
from app.services import permission_log_writer, security_access_log_writer
class _FakeSession:
def __init__(self) -> None:
self.added: list[object] = []
self.committed = False
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return False
def add(self, value: object) -> None:
self.added.append(value)
async def commit(self) -> None:
self.committed = True
@pytest.mark.asyncio
async def test_permission_writer_gracefully_flushes_and_reports_stats(monkeypatch):
session = _FakeSession()
monkeypatch.setattr(permission_log_writer, "SessionLocal", lambda: session)
writer = permission_log_writer.PermissionLogWriter()
await writer.start()
writer.enqueue(
{
"study_id": uuid.uuid4(),
"user_id": uuid.uuid4(),
"endpoint_key": "subjects.read",
"role": "PM",
"allowed": True,
"elapsed_ms": 2.5,
}
)
await writer.stop()
stats = writer.stats()
assert session.committed is True
assert len(session.added) == 1
assert stats["running"] is False
assert stats["accepted_entries"] == 1
assert stats["written_entries"] == 1
assert stats["dropped_entries"] == 0
assert stats["queue_size"] == 0
assert stats["last_success_at"] is not None
@pytest.mark.asyncio
async def test_permission_writer_retries_and_exposes_permanent_failure(monkeypatch):
class _FailingSession:
async def __aenter__(self):
raise RuntimeError("database unavailable")
async def __aexit__(self, exc_type, exc, tb):
return False
monkeypatch.setattr(permission_log_writer, "SessionLocal", _FailingSession)
writer = permission_log_writer.PermissionLogWriter()
batch = [
{
"study_id": uuid.uuid4(),
"user_id": uuid.uuid4(),
"endpoint_key": "subjects.read",
"role": "PM",
"allowed": True,
"elapsed_ms": 2.5,
}
]
assert await writer._write_batch(batch) is False
stats = writer.stats()
assert stats["retry_count"] == 1
assert stats["failed_batches"] == 1
assert stats["failed_entries"] == 1
assert stats["last_error_type"] == "RuntimeError"
def test_writer_queue_overflow_is_counted():
writer = security_access_log_writer.SecurityAccessLogWriter()
writer._queue = asyncio.Queue(maxsize=1)
writer.enqueue({"path": "/first"})
writer.enqueue({"path": "/dropped"})
stats = writer.stats()
assert stats["accepted_entries"] == 1
assert stats["dropped_entries"] == 1
assert stats["queue_size"] == 1
@pytest.mark.asyncio
async def test_retention_deletes_expired_rows_and_preserves_recent_rows(
db_session, test_engine, monkeypatch
):
now = datetime(2026, 7, 10, tzinfo=timezone.utc)
study_id = (await db_session.execute(select(Study.id).limit(1))).scalar_one()
user = User(
email=f"retention-{uuid.uuid4()}@example.com",
password_hash="hash",
full_name="Retention Test",
clinical_department="QA",
status=UserStatus.ACTIVE,
)
db_session.add(user)
await db_session.flush()
old_permission = PermissionAccessLog(
study_id=study_id,
user_id=user.id,
endpoint_key="old",
role="PM",
allowed=True,
elapsed_ms=1,
created_at=now - timedelta(days=91),
)
recent_permission = PermissionAccessLog(
study_id=study_id,
user_id=user.id,
endpoint_key="recent",
role="PM",
allowed=True,
elapsed_ms=1,
created_at=now - timedelta(days=89),
)
old_security = SecurityAccessLog(
method="GET",
path="/old",
status_code=404,
elapsed_ms=1,
auth_status="ANONYMOUS",
created_at=now - timedelta(days=91),
)
recent_security = SecurityAccessLog(
method="GET",
path="/recent",
status_code=200,
elapsed_ms=1,
auth_status="AUTHENTICATED",
created_at=now - timedelta(days=89),
)
old_metric = PermissionMetricSnapshot(
bucket_time=now - timedelta(days=401)
)
recent_metric = PermissionMetricSnapshot(
bucket_time=now - timedelta(days=399)
)
db_session.add_all(
[
old_permission,
recent_permission,
old_security,
recent_security,
old_metric,
recent_metric,
]
)
await db_session.commit()
old_permission_id = old_permission.id
recent_permission_id = recent_permission.id
old_security_id = old_security.id
recent_security_id = recent_security.id
old_metric_id = old_metric.id
recent_metric_id = recent_metric.id
session_factory = async_sessionmaker(
bind=test_engine,
class_=AsyncSession,
expire_on_commit=False,
)
monkeypatch.setattr(monitoring_retention, "SessionLocal", session_factory)
monkeypatch.setattr(
monitoring_retention.settings, "MONITORING_ACCESS_LOG_RETENTION_DAYS", 90
)
monkeypatch.setattr(
monitoring_retention.settings, "MONITORING_METRIC_RETENTION_DAYS", 400
)
deleted = await monitoring_retention.purge_expired_monitoring_data(now=now)
db_session.expire_all()
assert deleted["permission_access_logs"] >= 1
assert deleted["security_access_logs"] >= 1
assert deleted["permission_metric_snapshots"] >= 1
assert await db_session.get(PermissionAccessLog, old_permission_id) is None
assert await db_session.get(SecurityAccessLog, old_security_id) is None
assert await db_session.get(PermissionMetricSnapshot, old_metric_id) is None
assert await db_session.get(PermissionAccessLog, recent_permission_id) is not None
assert await db_session.get(SecurityAccessLog, recent_security_id) is not None
assert await db_session.get(PermissionMetricSnapshot, recent_metric_id) is not None
@@ -0,0 +1,53 @@
from types import SimpleNamespace
import pytest
from app.core.config import settings
from app.services import monitoring_server_location
def test_public_ip_normalization_rejects_private_addresses():
assert monitoring_server_location._normalize_public_ip("8.8.8.8\n") == "8.8.8.8"
assert monitoring_server_location._normalize_public_ip("10.0.0.8") is None
assert monitoring_server_location._normalize_public_ip("127.0.0.1") is None
assert monitoring_server_location._normalize_public_ip("not-an-ip") is None
@pytest.mark.asyncio
async def test_server_location_is_derived_from_configured_public_ip(monkeypatch):
monkeypatch.setattr(settings, "MONITORING_SERVER_PUBLIC_IP", "8.8.8.8")
monkeypatch.setattr(
monitoring_server_location,
"resolve_ip_location",
lambda ip: SimpleNamespace(
location="United States / California / San Jose",
country="United States",
province="California",
city="San Jose",
isp="Google",
),
)
monitoring_server_location.reset_monitoring_server_location_cache()
location = await monitoring_server_location.resolve_monitoring_server_location()
assert location is not None
assert location.name == "United States / California / San Jose"
assert location.longitude == -121.8863
assert location.latitude == 37.3382
assert location.resolution_source == "configured_public_ip"
monitoring_server_location.reset_monitoring_server_location_cache()
@pytest.mark.asyncio
async def test_server_location_returns_none_instead_of_a_hardcoded_fallback(monkeypatch):
async def no_public_ip():
return None, "unavailable"
monkeypatch.setattr(monitoring_server_location, "_discover_public_ip", no_public_ip)
monitoring_server_location.reset_monitoring_server_location_cache()
location = await monitoring_server_location.resolve_monitoring_server_location()
assert location is None
monitoring_server_location.reset_monitoring_server_location_cache()
+554 -22
View File
@@ -1,7 +1,9 @@
"""监控API测试:权限系统监控API端点验证。"""
import inspect
import json
import uuid
from types import SimpleNamespace
import pytest
from fastapi import HTTPException
@@ -10,6 +12,7 @@ from sqlalchemy import text
from app.core.permission_monitor import set_permission_monitor, PermissionMonitor
from app.api.v1 import permission_monitoring
from app.api.v1.system_permissions import list_system_permissions
from app.services.monitoring_server_location import MonitoringServerLocation
class FakeIpInfo:
@@ -30,6 +33,18 @@ class ProjectPmUserStub:
is_admin = False
@pytest.fixture(autouse=True)
def disable_server_public_ip_discovery(monkeypatch):
async def no_server_location():
return None
monkeypatch.setattr(
permission_monitoring,
"resolve_monitoring_server_location",
no_server_location,
)
@pytest.mark.asyncio
async def test_resolve_monitoring_scope_rejects_project_pm(db_session):
"""系统监测模块仅允许系统管理员访问,项目 PM 不再具备监测范围。"""
@@ -103,9 +118,11 @@ async def _seed_permission_log(db_session, study_id: uuid.UUID, user_id: uuid.UU
text(
"""
INSERT INTO permission_access_logs
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, created_at)
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address,
request_snapshot, created_at)
VALUES
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address, CURRENT_TIMESTAMP)
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address,
:request_snapshot, CURRENT_TIMESTAMP)
"""
),
{
@@ -117,6 +134,7 @@ async def _seed_permission_log(db_session, study_id: uuid.UUID, user_id: uuid.UU
"allowed": allowed,
"elapsed_ms": elapsed_ms,
"ip_address": "127.0.0.1",
"request_snapshot": None,
},
)
await db_session.commit()
@@ -262,8 +280,8 @@ async def test_permission_system_health_degraded(db_session):
assert any("权限拒绝率偏高" in issue for issue in data["issues"])
assert any("缓存命中率偏低" in issue for issue in data["issues"])
assert data["status"] == "unhealthy"
assert data["health_score"] == 30
assert data["score_breakdown"]["performance"]["deduction"] == 25
assert data["health_score"] == 35
assert data["score_breakdown"]["performance"]["deduction"] == 20
assert data["score_breakdown"]["deny_rate"]["deduction"] == 20
assert data["score_breakdown"]["cache"]["deduction"] == 25
@@ -279,6 +297,10 @@ async def test_permission_system_health_includes_metrics(db_session):
assert "cache_stats" in data
assert "score_breakdown" in data
assert "sample_sufficient" in data
assert "components" in data
assert "storage" in data
assert "runtime" in data["score_breakdown"]
assert data["components"]["database"]["status"] == "healthy"
assert "total_checks" in data["last_hour"]
assert "cache_metrics" in data["cache_stats"]
@@ -304,6 +326,41 @@ async def test_permission_system_health_ignores_small_cache_sample(db_session):
assert data["health_score"] == 100 - non_cache_deductions
@pytest.mark.asyncio
async def test_permission_system_health_reports_writer_degradation(db_session, monkeypatch):
monitor = PermissionMonitor()
set_permission_monitor(monitor)
writer = SimpleNamespace(
stats=lambda: {
"running": True,
"queue_size": 3,
"queue_capacity": 100,
"failed_batches": 0,
"dropped_entries": 2,
}
)
monkeypatch.setattr(permission_monitoring, "get_log_writer", lambda: writer)
monkeypatch.setattr(permission_monitoring, "get_security_log_writer", lambda: None)
monkeypatch.setattr(
permission_monitoring,
"get_monitoring_retention_status",
lambda: {
"running": False,
"last_run_started_at": None,
"last_success_at": None,
"last_error_at": None,
},
)
data = await permission_monitoring.permission_system_health(
db=db_session, _=AdminUserStub()
)
assert data["components"]["permission_log_writer"]["status"] == "degraded"
assert data["score_breakdown"]["runtime"]["deduction"] == 10
assert any("权限日志写入器" in issue for issue in data["issues"])
@pytest.mark.asyncio
async def test_ip_locations_counts_unique_users_per_location(db_session, monkeypatch):
"""IP 属地统计应按省市聚合访问次数、来源 IP 数和访问用户数。"""
@@ -381,10 +438,26 @@ async def test_ip_locations_counts_unique_users_per_location(db_session, monkeyp
)
await db_session.commit()
resolved_ips = []
def fake_resolve_ip_location(ip):
resolved_ips.append(ip)
return FakeIpInfo("广东省", "深圳市")
async def fake_resolve_monitoring_server_location():
return MonitoringServerLocation(
name="China / 重庆 / 重庆市",
longitude=106.5516,
latitude=29.563,
accuracy_level="city",
resolution_source="public_ip_discovery",
)
monkeypatch.setattr(permission_monitoring, "resolve_ip_location", fake_resolve_ip_location)
monkeypatch.setattr(
permission_monitoring,
"resolve_ip_location",
lambda ip: FakeIpInfo("广东省", "深圳市"),
"resolve_monitoring_server_location",
fake_resolve_monitoring_server_location,
)
result = await permission_monitoring.get_ip_locations(db=db_session, _=AdminUserStub(), days=7, limit=10)
@@ -396,6 +469,56 @@ async def test_ip_locations_counts_unique_users_per_location(db_session, monkeyp
assert result["summary"]["total_count"] == 3
assert result["summary"]["unique_ip_count"] == 2
assert result["summary"]["unique_user_count"] == 2
assert result["items"][0]["country_code"] == "CN"
assert result["items"][0]["region_code"] == "CN-GD"
assert result["items"][0]["longitude"] is not None
assert result["items"][0]["risk_level"] == "attention"
assert result["period"]["mode"] == "rolling"
assert result["period"]["retention_days"] >= 7
assert result["data_quality"]["located_ip_count"] == 2
assert result["server_location"]["longitude"] == 106.5516
assert result["server_location"]["resolution_source"] == "public_ip_discovery"
assert sorted(resolved_ips) == ["10.1.1.1", "10.1.1.2"]
await db_session.execute(
text(
"""
INSERT INTO permission_access_logs
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, created_at)
VALUES
(lower(hex(randomblob(4))) || '-' || lower(hex(randomblob(2))) || '-4' ||
substr(lower(hex(randomblob(2))),2) || '-' ||
substr('89ab', abs(random()) % 4 + 1, 1) ||
substr(lower(hex(randomblob(2))),2) || '-' || lower(hex(randomblob(6))),
:study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address,
datetime('now', '-120 days'))
"""
),
{
"study_id": study_id,
"user_id": user_a,
"endpoint_key": "admin.permissions.read",
"role": "PM",
"allowed": True,
"elapsed_ms": 2.4,
"ip_address": "10.1.1.3",
},
)
await db_session.commit()
resolved_ips.clear()
all_result = await permission_monitoring.get_ip_locations(
db=db_session,
_=AdminUserStub(),
days=7,
limit=10,
all_time=True,
)
assert all_result["days"] is None
assert all_result["summary"]["total_count"] == 4
assert all_result["summary"]["unique_ip_count"] == 3
assert sorted(resolved_ips) == ["10.1.1.1", "10.1.1.2", "10.1.1.3"]
@pytest.mark.asyncio
@@ -530,9 +653,11 @@ async def test_ip_locations_keeps_private_network_location_label(db_session, mon
text(
"""
INSERT INTO permission_access_logs
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, created_at)
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address,
request_snapshot, created_at)
VALUES
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address, CURRENT_TIMESTAMP)
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address,
:request_snapshot, CURRENT_TIMESTAMP)
"""
),
{
@@ -544,6 +669,7 @@ async def test_ip_locations_keeps_private_network_location_label(db_session, mon
"allowed": True,
"elapsed_ms": 3.2,
"ip_address": "192.168.1.10",
"request_snapshot": None,
},
)
await db_session.commit()
@@ -556,8 +682,8 @@ async def test_ip_locations_keeps_private_network_location_label(db_session, mon
@pytest.mark.asyncio
async def test_ip_locations_includes_abnormal_security_ips(db_session, monkeypatch):
"""来源分析应同时统计安全事件中的异常来源 IP"""
async def test_ip_locations_scores_security_sources_by_behavior(db_session, monkeypatch):
"""来源分析应按安全行为而非地域评估来源风险"""
await db_session.execute(text("DELETE FROM permission_access_logs"))
await db_session.execute(text("DELETE FROM security_access_logs"))
await db_session.commit()
@@ -603,9 +729,11 @@ async def test_ip_locations_includes_abnormal_security_ips(db_session, monkeypat
text(
"""
INSERT INTO permission_access_logs
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, created_at)
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address,
request_snapshot, created_at)
VALUES
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address, CURRENT_TIMESTAMP)
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address,
:request_snapshot, CURRENT_TIMESTAMP)
"""
),
{
@@ -617,9 +745,10 @@ async def test_ip_locations_includes_abnormal_security_ips(db_session, monkeypat
"allowed": True,
"elapsed_ms": 3.2,
"ip_address": "10.3.1.1",
"request_snapshot": None,
},
)
for ip_address in ["8.8.8.8", "1.1.1.1"]:
for ip_address, status_code in [("8.8.8.8", 403), ("1.1.1.1", 200)]:
await db_session.execute(
text(
"""
@@ -633,7 +762,7 @@ async def test_ip_locations_includes_abnormal_security_ips(db_session, monkeypat
"id": str(uuid.uuid4()),
"method": "GET",
"path": "/api/v1/admin",
"status_code": 403,
"status_code": status_code,
"elapsed_ms": 8.5,
"client_ip": ip_address,
"user_agent": "pytest",
@@ -658,18 +787,23 @@ async def test_ip_locations_includes_abnormal_security_ips(db_session, monkeypat
locations = {item["country"]: item for item in result["items"]}
assert result["summary"]["total_count"] == 3
assert result["summary"]["denied_count"] == 2
assert result["summary"]["denied_count"] == 1
assert result["summary"]["unique_ip_count"] == 3
assert result["summary"]["unique_user_count"] == 1
assert locations["United States"]["total_count"] == 1
assert locations["United States"]["denied_count"] == 1
assert locations["United States"]["risk_level"] == "high"
assert locations["United States"]["country_code"] == "US"
assert locations["Australia"]["total_count"] == 1
assert locations["Australia"]["denied_count"] == 0
assert locations["Australia"]["risk_level"] == "normal"
@pytest.mark.asyncio
async def test_access_logs_include_user_behavior_summary(db_session):
"""访问日志应返回用户行为审计汇总和用户排行。"""
await db_session.execute(text("DELETE FROM permission_access_logs"))
await db_session.execute(text("DELETE FROM security_access_logs"))
await db_session.commit()
study_id = "00000000-0000-0000-0000-000000000201"
@@ -784,10 +918,300 @@ def test_access_logs_user_stats_query_avoids_postgresql_uuid_max():
assert "func.max(PermissionAccessLog.user_id)" not in source
@pytest.mark.asyncio
async def test_access_logs_filter_by_account_location_and_client_source(db_session, monkeypatch):
"""访问审计应支持按账号、IP 属地和客户端来源排查。"""
await db_session.execute(text("DELETE FROM permission_access_logs"))
await db_session.execute(text("DELETE FROM security_access_logs"))
await db_session.commit()
study_id = "00000000-0000-0000-0000-000000001401"
user_a = "00000000-0000-0000-0000-000000001501"
user_b = "00000000-0000-0000-0000-000000001502"
await db_session.execute(
text(
"""
INSERT INTO studies (id, code, name, status, is_locked, visit_schedule, active_roles)
VALUES (:id, :code, :name, :status, :is_locked, :visit_schedule, :active_roles)
"""
),
{
"id": study_id,
"code": "ACCESS-CONTEXT-STUDY",
"name": "Access Context Study",
"status": "ACTIVE",
"is_locked": False,
"visit_schedule": "[]",
"active_roles": "[]",
},
)
for user_id, email, name in [
(user_a, "audit-shanghai@example.com", "上海审计用户"),
(user_b, "audit-beijing@example.com", "北京审计用户"),
]:
await db_session.execute(
text(
"""
INSERT INTO users (id, email, password_hash, full_name, clinical_department, is_admin, status)
VALUES (:id, :email, :password_hash, :full_name, :clinical_department, :is_admin, :status)
"""
),
{
"id": user_id,
"email": email,
"password_hash": "hash",
"full_name": name,
"clinical_department": "临床运营",
"is_admin": False,
"status": "ACTIVE",
},
)
rows = [
(user_a, "203.0.113.10", "web", "Chrome", "macos", {"user-agent": "Chrome", "x-request-id": "req-web"}),
(user_b, "198.51.100.20", "desktop", "CTMS Desktop", "windows", {"user-agent": "CTMS Desktop"}),
(user_a, "10.10.10.10", None, "curl/8.0", None, {"user-agent": "curl/8.0"}),
]
for user, ip_address, client_type, user_agent, platform, request_headers in rows:
await db_session.execute(
text(
"""
INSERT INTO permission_access_logs
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address,
user_agent, client_type, client_version, client_platform, build_channel, build_commit, request_headers, created_at)
VALUES
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address,
:user_agent, :client_type, :client_version, :client_platform, :build_channel, :build_commit, :request_headers, CURRENT_TIMESTAMP)
"""
),
{
"id": str(uuid.uuid4()),
"study_id": study_id,
"user_id": user,
"endpoint_key": "admin.permissions.read",
"role": "PM",
"allowed": True,
"elapsed_ms": 4.0,
"ip_address": ip_address,
"user_agent": user_agent,
"client_type": client_type,
"client_version": "0.1.0" if client_type else None,
"client_platform": platform,
"build_channel": "local" if client_type else None,
"build_commit": "abcdef1" if client_type else None,
"request_headers": json.dumps(request_headers),
},
)
await db_session.commit()
ip_info_by_address = {
"203.0.113.10": FakeIpInfo("上海市", "上海市", "电信"),
"198.51.100.20": FakeIpInfo("北京市", "北京市", "联通"),
"10.10.10.10": FakeIpInfo("", "", "", "局域网"),
}
monkeypatch.setattr(
permission_monitoring,
"resolve_ip_location",
lambda ip: ip_info_by_address[ip],
)
result = await permission_monitoring.get_access_logs(
db=db_session,
_=AdminUserStub(),
study_id=None,
user_id=None,
endpoint_key=None,
role=None,
allowed=None,
start_time=None,
end_time=None,
client_ip=None,
client_type="web",
keyword="上海",
page=1,
page_size=50,
)
assert result["total"] == 1
assert result["items"][0]["user_email"] == "audit-shanghai@example.com"
assert result["items"][0]["client_type"] == "web"
assert result["items"][0]["client_platform"] == "macos"
assert result["items"][0]["user_agent"] == "Chrome"
assert result["items"][0]["request_headers"]["x-request-id"] == "req-web"
assert result["items"][0]["ip_city"] == "上海市"
unknown_result = await permission_monitoring.get_access_logs(
db=db_session,
_=AdminUserStub(),
study_id=None,
user_id=None,
endpoint_key=None,
role=None,
allowed=None,
start_time=None,
end_time=None,
client_ip=None,
client_type="unknown",
keyword="curl",
page=1,
page_size=50,
)
assert unknown_result["total"] == 1
assert unknown_result["items"][0]["ip_address"] == "10.10.10.10"
@pytest.mark.asyncio
async def test_access_logs_include_security_events_for_admin(db_session, monkeypatch):
"""访问日志应合并业务权限日志和安全访问日志,避免匿名/探测请求审计盲区。"""
await db_session.execute(text("DELETE FROM permission_access_logs"))
await db_session.execute(text("DELETE FROM security_access_logs"))
await db_session.commit()
study_id = "00000000-0000-0000-0000-000000001601"
user_id = "00000000-0000-0000-0000-000000001701"
await db_session.execute(
text(
"""
INSERT INTO studies (id, code, name, status, is_locked, visit_schedule, active_roles)
VALUES (:id, :code, :name, :status, :is_locked, :visit_schedule, :active_roles)
"""
),
{
"id": study_id,
"code": "ACCESS-SECURITY-FEED",
"name": "Access Security Feed",
"status": "ACTIVE",
"is_locked": False,
"visit_schedule": "[]",
"active_roles": "[]",
},
)
await db_session.execute(
text(
"""
INSERT INTO users (id, email, password_hash, full_name, clinical_department, is_admin, status)
VALUES (:id, :email, :password_hash, :full_name, :clinical_department, :is_admin, :status)
"""
),
{
"id": user_id,
"email": "access-feed@example.com",
"password_hash": "hash",
"full_name": "统一访问用户",
"clinical_department": "临床运营",
"is_admin": False,
"status": "ACTIVE",
},
)
await db_session.execute(
text(
"""
INSERT INTO permission_access_logs
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address,
request_snapshot, created_at)
VALUES
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address,
:request_snapshot, CURRENT_TIMESTAMP)
"""
),
{
"id": str(uuid.uuid4()),
"study_id": study_id,
"user_id": user_id,
"endpoint_key": "project_overview:read",
"role": "PM",
"allowed": True,
"elapsed_ms": 3.0,
"ip_address": "192.168.10.2",
"request_snapshot": json.dumps(
{
"method": "GET",
"path": "/api/v1/projects/overview",
"query_string": "tab=summary",
"headers": {"user-agent": "Chrome", "x-request-id": "perm-1"},
"body": {"captured": False, "reason": "body_not_captured_by_audit_policy"},
}
),
},
)
await db_session.execute(
text(
"""
INSERT INTO security_access_logs
(id, method, path, status_code, elapsed_ms, client_ip, user_agent, auth_status,
user_identifier, request_headers, request_snapshot, created_at)
VALUES
(:id, :method, :path, :status_code, :elapsed_ms, :client_ip, :user_agent, :auth_status,
:user_identifier, :request_headers, :request_snapshot, CURRENT_TIMESTAMP)
"""
),
{
"id": str(uuid.uuid4()),
"method": "GET",
"path": "/api/.git/config",
"status_code": 404,
"elapsed_ms": 1.5,
"client_ip": "45.148.10.95",
"user_agent": "probe-bot/1.0",
"auth_status": "ANONYMOUS",
"user_identifier": None,
"request_headers": json.dumps({"user-agent": "probe-bot/1.0", "x-request-id": "sec-1"}),
"request_snapshot": json.dumps(
{
"method": "GET",
"path": "/api/.git/config",
"query_string": "token=[redacted]",
"headers": {"user-agent": "probe-bot/1.0", "authorization": "[redacted]"},
"body": {"captured": False, "reason": "body_not_captured_by_audit_policy"},
}
),
},
)
await db_session.commit()
ip_info_by_address = {
"192.168.10.2": FakeIpInfo("", "", "", "局域网"),
"45.148.10.95": FakeIpInfo("South Holland", "", "", "Netherlands"),
}
monkeypatch.setattr(permission_monitoring, "resolve_ip_location", lambda ip: ip_info_by_address[ip])
result = await permission_monitoring.get_access_logs(
db=db_session,
_=AdminUserStub(),
study_id=None,
user_id=None,
endpoint_key=None,
role=None,
allowed=None,
start_time=None,
end_time=None,
client_ip=None,
client_type=None,
keyword=None,
page=1,
page_size=50,
)
assert result["total"] == 2
assert result["summary"]["total_count"] == 2
assert result["summary"]["denied_count"] == 1
items_by_type = {item["event_type"]: item for item in result["items"]}
assert items_by_type["permission"]["endpoint_key"] == "project_overview:read"
assert items_by_type["security"]["status_code"] == 404
assert items_by_type["security"]["allowed"] is False
assert items_by_type["security"]["category"] == "PROBE"
assert items_by_type["security"]["severity"] == "CRITICAL"
assert items_by_type["security"]["request_headers"]["x-request-id"] == "sec-1"
assert items_by_type["permission"]["request_snapshot"]["path"] == "/api/v1/projects/overview"
assert items_by_type["security"]["request_snapshot"]["headers"]["authorization"] == "[redacted]"
@pytest.mark.asyncio
async def test_access_logs_ip_ranking_aggregates_by_ip_total(db_session):
"""IP 访问排行应按同一 IP 的整体访问量聚合排序,而不是按用户/IP/角色拆分。"""
await db_session.execute(text("DELETE FROM permission_access_logs"))
await db_session.execute(text("DELETE FROM security_access_logs"))
await db_session.commit()
study_id = "00000000-0000-0000-0000-000000001201"
@@ -896,10 +1320,13 @@ async def test_security_access_logs_include_anonymous_ip_attempts(db_session, mo
text(
"""
INSERT INTO security_access_logs
(id, method, path, status_code, elapsed_ms, client_ip, user_agent, auth_status, user_identifier, created_at)
(id, method, path, status_code, elapsed_ms, client_ip, user_agent, auth_status,
user_identifier, request_snapshot, created_at)
VALUES
('00000000-0000-4000-8000-000000000501', 'POST', '/api/v1/auth/login', 401, 12.5,
'203.0.113.10', 'attack-bot/1.0', 'ANONYMOUS', NULL, CURRENT_TIMESTAMP)
'203.0.113.10', 'attack-bot/1.0', 'ANONYMOUS', NULL,
'{"method":"POST","path":"/api/v1/auth/login","headers":{"authorization":"[redacted]"}}',
CURRENT_TIMESTAMP)
"""
)
)
@@ -925,6 +1352,7 @@ async def test_security_access_logs_include_anonymous_ip_attempts(db_session, mo
assert result["items"][0]["ip_isp"] == "联通"
assert result["items"][0]["account_label"] == "未知账号"
assert result["items"][0]["status_code"] == 401
assert result["items"][0]["request_snapshot"]["headers"]["authorization"] == "[redacted]"
@pytest.mark.asyncio
@@ -993,8 +1421,8 @@ async def test_security_access_logs_prioritize_sensitive_probe_over_foreign_ip(d
@pytest.mark.asyncio
async def test_security_access_logs_classify_non_china_ip_as_abnormal(db_session, monkeypatch):
"""安全事件明细应把非中国公网 IP 归类为异常 IP"""
async def test_security_access_logs_classify_foreign_invalid_token_by_behavior(db_session, monkeypatch):
"""海外来源不应单凭地域升级风险,仍按认证行为分类"""
await db_session.execute(text("DELETE FROM security_access_logs"))
monkeypatch.setattr(
permission_monitoring,
@@ -1024,8 +1452,8 @@ async def test_security_access_logs_classify_non_china_ip_as_abnormal(db_session
)
item = result["items"][0]
assert item["category"] == "ABNORMAL_IP"
assert item["severity"] == "HIGH"
assert item["category"] == "INVALID_TOKEN"
assert item["severity"] == "MEDIUM"
assert item["ip_location"] == "United States / California / San Jose / Google"
assert item["ip_country"] == "United States"
@@ -1062,4 +1490,108 @@ async def test_security_access_logs_resolve_public_ip_with_packaged_ip2region(db
assert item["ip_country"] == "United States"
assert item["ip_province"] == "California"
assert item["ip_city"] == "San Jose"
assert item["category"] == "ABNORMAL_IP"
assert item["category"] == "NOT_FOUND_NOISE"
@pytest.mark.asyncio
async def test_access_logs_deduplicate_permission_and_security_rows_by_request_id(db_session):
await db_session.execute(text("DELETE FROM permission_access_logs"))
await db_session.execute(text("DELETE FROM security_access_logs"))
request_id = str(uuid.uuid4())
study_id = str(uuid.uuid4())
user_id = str(uuid.uuid4())
await _seed_permission_log(db_session, uuid.UUID(study_id), uuid.UUID(user_id), allowed=False, elapsed_ms=4.0)
await db_session.execute(
text("UPDATE permission_access_logs SET request_id = :request_id"),
{"request_id": request_id},
)
await db_session.execute(
text(
"""
INSERT INTO security_access_logs
(id, method, path, status_code, elapsed_ms, client_ip, auth_status, user_identifier,
request_id, category, severity, created_at)
VALUES
(:id, 'GET', '/api/v1/studies/denied', 403, 5.0, '203.0.113.20', 'AUTHENTICATED', :user_id,
:request_id, 'OTHER', 'LOW', CURRENT_TIMESTAMP)
"""
),
{"id": str(uuid.uuid4()), "user_id": user_id, "request_id": request_id},
)
await db_session.commit()
result = await permission_monitoring.get_access_logs(
db=db_session,
_=AdminUserStub(),
study_id=None,
user_id=None,
endpoint_key=None,
role=None,
allowed=None,
start_time=None,
end_time=None,
client_ip=None,
client_type=None,
keyword=None,
page=1,
page_size=50,
)
assert result["total"] == 1
assert result["items"][0]["event_type"] == "security"
assert result["items"][0]["request_id"] == request_id
@pytest.mark.asyncio
async def test_security_events_only_includes_persisted_probe(db_session):
await db_session.execute(text("DELETE FROM security_access_logs"))
for category, ip in [("PROBE", "52.53.218.145"), ("OTHER", "203.0.113.20")]:
await db_session.execute(
text(
"""
INSERT INTO security_access_logs
(id, method, path, status_code, elapsed_ms, client_ip, auth_status,
category, severity, created_at)
VALUES
(:id, 'GET', '/api/v1/studies/', 200, 3.0, :ip, 'AUTHENTICATED',
:category, :severity, CURRENT_TIMESTAMP)
"""
),
{
"id": str(uuid.uuid4()),
"ip": ip,
"category": category,
"severity": "CRITICAL" if category == "PROBE" else "LOW",
},
)
await db_session.commit()
result = await permission_monitoring.get_security_access_logs(
db=db_session,
_=AdminUserStub(),
status_min=None,
auth_status=None,
events_only=True,
page=1,
page_size=20,
)
assert result["total"] == 1
assert result["items"][0]["category"] == "PROBE"
assert result["summary"]["high_risk_count"] == 1
def test_server_error_classification_takes_priority_over_foreign_ip():
log = SimpleNamespace(
path="/api/v1/studies/",
status_code=503,
auth_status="AUTHENTICATED",
category=None,
severity=None,
)
location = FakeIpInfo("California", "San Jose", "Google", "United States")
assert permission_monitoring._classify_security_access_log(log, location) == {
"category": "SERVER_ERROR",
"severity": "HIGH",
}
+13
View File
@@ -12,6 +12,7 @@ from datetime import datetime, timedelta, timezone
import base64
import json
import os
import uuid
from app.main import create_app
from app.core.config import settings
@@ -397,6 +398,7 @@ async def test_root_returns_service_metadata(client_and_db):
client, _ = client_and_db
resp = await client.get("/")
assert resp.status_code == 200
assert uuid.UUID(resp.headers["x-request-id"])
assert resp.json() == {
"service": "ctms-backend",
"status": "ok",
@@ -406,6 +408,17 @@ async def test_root_returns_service_metadata(client_and_db):
}
@pytest.mark.asyncio
async def test_readiness_checks_database(client_and_db):
client, _ = client_and_db
resp = await client.get("/readyz")
assert resp.status_code == 200
payload = resp.json()
assert payload["status"] == "ready"
assert payload["database"]["status"] == "healthy"
assert payload["database"]["latency_ms"] >= 0
@pytest.mark.asyncio
async def test_registered_user_can_login_after_email_verification(client_and_db):
client, SessionLocal = client_and_db
@@ -0,0 +1,68 @@
from datetime import datetime, timedelta, timezone
import pytest
from app.models.source_location_snapshot import SourceLocationSnapshot
from app.services.source_location_aggregator import _identity_hash, get_source_location_timeline
def _snapshot(bucket_time: datetime, *, allowed: int, denied: int) -> SourceLocationSnapshot:
return SourceLocationSnapshot(
bucket_time=bucket_time,
ip_hash=_identity_hash("ip", "203.0.113.10"),
user_hash=_identity_hash("user", "user-1"),
country="United States",
country_code="US",
province="California",
region_code="",
city="San Jose",
isp="Example ISP",
location="United States / California / San Jose",
longitude=-121.8863,
latitude=37.3382,
accuracy_level="city",
allowed_count=allowed,
denied_count=denied,
security_event_count=denied,
high_risk_count=0,
auth_failure_count=denied,
first_seen_at=bucket_time,
last_seen_at=bucket_time + timedelta(minutes=30),
)
def test_source_location_identity_hash_is_stable_and_does_not_expose_ip():
first = _identity_hash("ip", "203.0.113.10")
second = _identity_hash("ip", "203.0.113.10")
assert first == second
assert len(first) == 64
assert "203.0.113.10" not in first
@pytest.mark.asyncio
async def test_source_location_timeline_supports_hour_and_day_rollups(db_session):
start = datetime(2026, 7, 10, 8, tzinfo=timezone.utc)
db_session.add(_snapshot(start, allowed=4, denied=1))
db_session.add(_snapshot(start + timedelta(hours=1), allowed=3, denied=2))
await db_session.commit()
hourly = await get_source_location_timeline(
db_session,
start_at=start,
end_at=start + timedelta(hours=2),
granularity="hour",
)
daily = await get_source_location_timeline(
db_session,
start_at=start,
end_at=start + timedelta(hours=2),
granularity="day",
)
assert [item["total_count"] for item in hourly] == [5, 5]
assert len(daily) == 1
assert daily[0]["total_count"] == 10
assert daily[0]["denied_count"] == 3
assert daily[0]["unique_ip_count"] == 1
assert daily[0]["unique_user_count"] == 1
+61
View File
@@ -0,0 +1,61 @@
from datetime import datetime, timedelta, timezone
import uuid
import pytest
from app.core.config import settings
from app.models.user import User, UserStatus
from app.models.user_login_session import UserLoginSession
from app.services.user_login_sessions import get_login_summaries, session_id_from_payload
def test_legacy_session_id_is_stable_without_storing_a_token():
payload = {"sub": "00000000-0000-0000-0000-000000000101", "orig_iat": 1_700_000_000, "client_type": "web"}
first = session_id_from_payload(payload)
second = session_id_from_payload(payload)
assert first == second
assert isinstance(first, uuid.UUID)
@pytest.mark.asyncio
async def test_login_summary_marks_recent_unended_sessions_online(db_session, monkeypatch):
now = datetime.now(timezone.utc)
monkeypatch.setattr(settings, "USER_SESSION_ONLINE_SECONDS", 300)
user = User(
id=uuid.uuid4(),
email="login-summary@example.com",
password_hash="hash",
full_name="Login Summary",
clinical_department="IT",
status=UserStatus.ACTIVE,
)
db_session.add(user)
db_session.add_all(
[
UserLoginSession(
id=uuid.uuid4(),
user_id=user.id,
client_type="desktop",
login_at=now - timedelta(minutes=10),
last_seen_at=now - timedelta(seconds=20),
),
UserLoginSession(
id=uuid.uuid4(),
user_id=user.id,
client_type="web",
login_at=now - timedelta(days=1),
last_seen_at=now - timedelta(days=1),
ended_at=now - timedelta(days=1),
end_reason="logout",
),
]
)
await db_session.commit()
summaries = await get_login_summaries(db_session, [user.id])
assert summaries[user.id].status == "ONLINE"
assert summaries[user.id].active_session_count == 1
assert summaries[user.id].client_type == "desktop"