feat(监控): 完善系统监控、登录状态与访问审计能力
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
"""add access context to permission access logs
|
||||
|
||||
Revision ID: 20260709_02
|
||||
Revises: 20260709_01
|
||||
Create Date: 2026-07-09 15:45:00.000000
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
|
||||
revision: str = "20260709_02"
|
||||
down_revision: Union[str, None] = "20260709_01"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("permission_access_logs", sa.Column("user_agent", sa.String(500), nullable=True))
|
||||
op.add_column("permission_access_logs", sa.Column("client_type", sa.String(16), nullable=True))
|
||||
op.add_column("permission_access_logs", sa.Column("client_version", sa.String(32), nullable=True))
|
||||
op.add_column("permission_access_logs", sa.Column("client_platform", sa.String(16), nullable=True))
|
||||
op.add_column("permission_access_logs", sa.Column("build_channel", sa.String(16), nullable=True))
|
||||
op.add_column("permission_access_logs", sa.Column("build_commit", sa.String(64), nullable=True))
|
||||
op.create_index("ix_perm_log_ip_created", "permission_access_logs", ["ip_address", "created_at"])
|
||||
op.create_index("ix_perm_log_client_source_created", "permission_access_logs", ["client_type", "created_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_perm_log_client_source_created", table_name="permission_access_logs")
|
||||
op.drop_index("ix_perm_log_ip_created", table_name="permission_access_logs")
|
||||
for column in (
|
||||
"build_commit",
|
||||
"build_channel",
|
||||
"client_platform",
|
||||
"client_version",
|
||||
"client_type",
|
||||
"user_agent",
|
||||
):
|
||||
op.drop_column("permission_access_logs", column)
|
||||
@@ -0,0 +1,35 @@
|
||||
"""add request headers to permission access logs
|
||||
|
||||
Revision ID: 20260709_03
|
||||
Revises: 20260709_02
|
||||
Create Date: 2026-07-09 16:58:00.000000
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
|
||||
revision: str = "20260709_03"
|
||||
down_revision: Union[str, None] = "20260709_02"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
REQUEST_HEADERS_TYPE = sa.JSON().with_variant(
|
||||
postgresql.JSONB(astext_type=sa.Text()),
|
||||
"postgresql",
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"permission_access_logs",
|
||||
sa.Column("request_headers", REQUEST_HEADERS_TYPE, nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("permission_access_logs", "request_headers")
|
||||
@@ -0,0 +1,35 @@
|
||||
"""add request headers to security access logs
|
||||
|
||||
Revision ID: 20260709_04
|
||||
Revises: 20260709_03
|
||||
Create Date: 2026-07-09 17:30:00.000000
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
|
||||
revision: str = "20260709_04"
|
||||
down_revision: Union[str, None] = "20260709_03"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
REQUEST_HEADERS_TYPE = sa.JSON().with_variant(
|
||||
postgresql.JSONB(astext_type=sa.Text()),
|
||||
"postgresql",
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"security_access_logs",
|
||||
sa.Column("request_headers", REQUEST_HEADERS_TYPE, nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("security_access_logs", "request_headers")
|
||||
@@ -0,0 +1,40 @@
|
||||
"""add request snapshots to access logs
|
||||
|
||||
Revision ID: 20260709_05
|
||||
Revises: 20260709_04
|
||||
Create Date: 2026-07-09 18:00:00.000000
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
|
||||
revision: str = "20260709_05"
|
||||
down_revision: Union[str, None] = "20260709_04"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
REQUEST_SNAPSHOT_TYPE = sa.JSON().with_variant(
|
||||
postgresql.JSONB(astext_type=sa.Text()),
|
||||
"postgresql",
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"permission_access_logs",
|
||||
sa.Column("request_snapshot", REQUEST_SNAPSHOT_TYPE, nullable=True),
|
||||
)
|
||||
op.add_column(
|
||||
"security_access_logs",
|
||||
sa.Column("request_snapshot", REQUEST_SNAPSHOT_TYPE, nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("security_access_logs", "request_snapshot")
|
||||
op.drop_column("permission_access_logs", "request_snapshot")
|
||||
@@ -0,0 +1,67 @@
|
||||
"""add monitoring event identity and persisted security classification
|
||||
|
||||
Revision ID: 20260710_01
|
||||
Revises: 20260709_05
|
||||
Create Date: 2026-07-10 09:30:00.000000
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
|
||||
revision: str = "20260710_01"
|
||||
down_revision: Union[str, None] = "20260709_05"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("permission_access_logs", sa.Column("request_id", sa.String(36), nullable=True))
|
||||
op.add_column("security_access_logs", sa.Column("request_id", sa.String(36), nullable=True))
|
||||
op.add_column("security_access_logs", sa.Column("category", sa.String(30), nullable=True))
|
||||
op.add_column("security_access_logs", sa.Column("severity", sa.String(16), nullable=True))
|
||||
|
||||
op.create_index("ix_perm_log_request_id", "permission_access_logs", ["request_id"])
|
||||
op.create_index("ix_security_log_request_id", "security_access_logs", ["request_id"])
|
||||
op.create_index("ix_security_log_category_created", "security_access_logs", ["category", "created_at"])
|
||||
op.create_index("ix_security_log_severity_created", "security_access_logs", ["severity", "created_at"])
|
||||
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE security_access_logs
|
||||
SET category = CASE
|
||||
WHEN lower(path) LIKE '%/.env%' OR lower(path) LIKE '%.git%'
|
||||
OR lower(path) LIKE '%backup%' OR lower(path) LIKE '%config.php%'
|
||||
OR lower(path) LIKE '%wp-config%' OR lower(path) LIKE '%database.yml%'
|
||||
THEN 'PROBE'
|
||||
WHEN status_code >= 500 THEN 'SERVER_ERROR'
|
||||
WHEN auth_status = 'INVALID_TOKEN' THEN 'INVALID_TOKEN'
|
||||
WHEN auth_status = 'ANONYMOUS' AND status_code IN (401, 403) THEN 'ANONYMOUS_API'
|
||||
WHEN status_code = 404 THEN 'NOT_FOUND_NOISE'
|
||||
ELSE 'OTHER'
|
||||
END,
|
||||
severity = CASE
|
||||
WHEN lower(path) LIKE '%/.env%' OR lower(path) LIKE '%.git%'
|
||||
OR lower(path) LIKE '%backup%' OR lower(path) LIKE '%config.php%'
|
||||
OR lower(path) LIKE '%wp-config%' OR lower(path) LIKE '%database.yml%'
|
||||
THEN 'CRITICAL'
|
||||
WHEN status_code >= 500 THEN 'HIGH'
|
||||
WHEN auth_status = 'INVALID_TOKEN' THEN 'MEDIUM'
|
||||
WHEN auth_status = 'ANONYMOUS' AND status_code IN (401, 403) THEN 'MEDIUM'
|
||||
ELSE 'LOW'
|
||||
END
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_security_log_severity_created", table_name="security_access_logs")
|
||||
op.drop_index("ix_security_log_category_created", table_name="security_access_logs")
|
||||
op.drop_index("ix_security_log_request_id", table_name="security_access_logs")
|
||||
op.drop_index("ix_perm_log_request_id", table_name="permission_access_logs")
|
||||
op.drop_column("security_access_logs", "severity")
|
||||
op.drop_column("security_access_logs", "category")
|
||||
op.drop_column("security_access_logs", "request_id")
|
||||
op.drop_column("permission_access_logs", "request_id")
|
||||
@@ -0,0 +1,93 @@
|
||||
"""add source location hourly snapshots
|
||||
|
||||
Revision ID: 20260710_02
|
||||
Revises: 20260710_01
|
||||
Create Date: 2026-07-10 15:30:00.000000
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
|
||||
revision: str = "20260710_02"
|
||||
down_revision: Union[str, None] = "20260710_01"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"source_location_snapshots",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("bucket_time", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("ip_hash", sa.String(length=64), nullable=False),
|
||||
sa.Column("user_hash", sa.String(length=64), nullable=False, server_default=""),
|
||||
sa.Column("country", sa.String(length=100), nullable=False, server_default=""),
|
||||
sa.Column("country_code", sa.String(length=16), nullable=False, server_default=""),
|
||||
sa.Column("province", sa.String(length=100), nullable=False, server_default=""),
|
||||
sa.Column("region_code", sa.String(length=24), nullable=False, server_default=""),
|
||||
sa.Column("city", sa.String(length=100), nullable=False, server_default=""),
|
||||
sa.Column("isp", sa.String(length=160), nullable=False, server_default=""),
|
||||
sa.Column("location", sa.String(length=320), nullable=False, server_default=""),
|
||||
sa.Column("longitude", sa.Float(), nullable=True),
|
||||
sa.Column("latitude", sa.Float(), nullable=True),
|
||||
sa.Column("accuracy_level", sa.String(length=16), nullable=False, server_default="unknown"),
|
||||
sa.Column("allowed_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("denied_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("security_event_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("high_risk_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("auth_failure_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("bucket_time", "ip_hash", "user_hash", name="uq_source_location_bucket_identity"),
|
||||
)
|
||||
op.create_index("ix_source_location_bucket", "source_location_snapshots", ["bucket_time"])
|
||||
op.create_index(
|
||||
"ix_source_location_country_bucket",
|
||||
"source_location_snapshots",
|
||||
["country_code", "bucket_time"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_source_location_risk_bucket",
|
||||
"source_location_snapshots",
|
||||
["high_risk_count", "bucket_time"],
|
||||
)
|
||||
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE security_access_logs
|
||||
SET category = CASE
|
||||
WHEN lower(path) LIKE '%/.env%' OR lower(path) LIKE '%.git%'
|
||||
OR lower(path) LIKE '%backup%' OR lower(path) LIKE '%config.php%'
|
||||
OR lower(path) LIKE '%wp-config%' OR lower(path) LIKE '%database.yml%'
|
||||
THEN 'PROBE'
|
||||
WHEN status_code >= 500 THEN 'SERVER_ERROR'
|
||||
WHEN auth_status = 'INVALID_TOKEN' THEN 'INVALID_TOKEN'
|
||||
WHEN auth_status = 'ANONYMOUS' AND status_code IN (401, 403) THEN 'ANONYMOUS_API'
|
||||
WHEN status_code = 404 THEN 'NOT_FOUND_NOISE'
|
||||
ELSE 'OTHER'
|
||||
END,
|
||||
severity = CASE
|
||||
WHEN lower(path) LIKE '%/.env%' OR lower(path) LIKE '%.git%'
|
||||
OR lower(path) LIKE '%backup%' OR lower(path) LIKE '%config.php%'
|
||||
OR lower(path) LIKE '%wp-config%' OR lower(path) LIKE '%database.yml%'
|
||||
THEN 'CRITICAL'
|
||||
WHEN status_code >= 500 THEN 'HIGH'
|
||||
WHEN auth_status = 'INVALID_TOKEN' THEN 'MEDIUM'
|
||||
WHEN auth_status = 'ANONYMOUS' AND status_code IN (401, 403) THEN 'MEDIUM'
|
||||
ELSE 'LOW'
|
||||
END
|
||||
WHERE category = 'ABNORMAL_IP'
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_source_location_risk_bucket", table_name="source_location_snapshots")
|
||||
op.drop_index("ix_source_location_country_bucket", table_name="source_location_snapshots")
|
||||
op.drop_index("ix_source_location_bucket", table_name="source_location_snapshots")
|
||||
op.drop_table("source_location_snapshots")
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Remove deployment-specific coordinates from private source snapshots.
|
||||
|
||||
Revision ID: 20260710_03
|
||||
Revises: 20260710_02
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
|
||||
|
||||
revision = "20260710_03"
|
||||
down_revision = "20260710_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE source_location_snapshots
|
||||
SET longitude = NULL,
|
||||
latitude = NULL
|
||||
WHERE accuracy_level = 'private'
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# The previous coordinates were a hardcoded deployment assumption and
|
||||
# cannot be restored without reintroducing incorrect data.
|
||||
pass
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Add user login session activity records.
|
||||
|
||||
Revision ID: 20260710_04
|
||||
Revises: 20260710_03
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
|
||||
revision = "20260710_04"
|
||||
down_revision = "20260710_03"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"user_login_sessions",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("user_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("client_type", sa.String(length=16), nullable=False, server_default="web"),
|
||||
sa.Column("client_platform", sa.String(length=32), nullable=True),
|
||||
sa.Column("client_version", sa.String(length=64), nullable=True),
|
||||
sa.Column("client_source", sa.String(length=32), nullable=True),
|
||||
sa.Column("login_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("CURRENT_TIMESTAMP")),
|
||||
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("CURRENT_TIMESTAMP")),
|
||||
sa.Column("ended_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("end_reason", sa.String(length=32), nullable=True),
|
||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index("ix_user_login_sessions_user_login", "user_login_sessions", ["user_id", "login_at"])
|
||||
op.create_index("ix_user_login_sessions_last_seen", "user_login_sessions", ["last_seen_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_user_login_sessions_last_seen", table_name="user_login_sessions")
|
||||
op.drop_index("ix_user_login_sessions_user_login", table_name="user_login_sessions")
|
||||
op.drop_table("user_login_sessions")
|
||||
@@ -9,7 +9,8 @@ import uuid
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.login_crypto import create_login_challenge, decrypt_login_payload, get_public_key_pem
|
||||
from app.core.security import create_access_token, decode_token_allow_expired, oauth2_scheme, verify_password
|
||||
from app.core.request_context import resolve_ctms_client_type
|
||||
from app.core.security import create_access_token, decode_token, decode_token_allow_expired, oauth2_scheme, verify_password
|
||||
from app.core.deps import get_current_user, get_db_session
|
||||
from app.crud import user as user_crud
|
||||
from app.models.user import UserStatus
|
||||
@@ -26,6 +27,12 @@ from app.schemas.email_settings import (
|
||||
)
|
||||
from app.schemas.user import Token, UserRead, UserRegisterRequest, UserSelfUpdate, UserUpdate
|
||||
from app.services import email_service
|
||||
from app.services.user_login_sessions import (
|
||||
create_login_session,
|
||||
end_login_session,
|
||||
session_id_from_payload,
|
||||
touch_login_session,
|
||||
)
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
|
||||
@@ -95,7 +102,7 @@ def get_session_policy_for_client_type(client_type: str) -> SessionPolicy:
|
||||
|
||||
|
||||
def get_request_session_client_type(request: Request) -> str:
|
||||
return normalize_session_client_type(request.headers.get("x-ctms-client-type"))
|
||||
return normalize_session_client_type(resolve_ctms_client_type(request.headers))
|
||||
|
||||
|
||||
def policy_expires_at(issued_at: datetime, session_start: datetime, policy: SessionPolicy) -> datetime:
|
||||
@@ -104,8 +111,9 @@ def policy_expires_at(issued_at: datetime, session_start: datetime, policy: Sess
|
||||
return min(access_expires_at, session_expires_at)
|
||||
|
||||
|
||||
def issue_user_token(db_user, request: Request) -> Token:
|
||||
async def issue_user_token(db_user, request: Request, db: AsyncSession) -> Token:
|
||||
session_start = datetime.now(timezone.utc)
|
||||
session_id = uuid.uuid4()
|
||||
client_type = get_request_session_client_type(request)
|
||||
policy = get_session_policy_for_client_type(client_type)
|
||||
access_token = create_access_token(
|
||||
@@ -115,6 +123,14 @@ def issue_user_token(db_user, request: Request) -> Token:
|
||||
max_age_seconds=policy.absolute_max_seconds,
|
||||
issued_at=session_start,
|
||||
client_type=client_type,
|
||||
session_id=str(session_id),
|
||||
)
|
||||
await create_login_session(
|
||||
db,
|
||||
session_id=session_id,
|
||||
user_id=db_user.id,
|
||||
request=request,
|
||||
login_at=session_start,
|
||||
)
|
||||
return Token(access_token=access_token, token_type="bearer")
|
||||
|
||||
@@ -284,7 +300,7 @@ async def login_for_access_token(
|
||||
db_user = await authenticate_encrypted_password(payload, db)
|
||||
ensure_user_active(db_user)
|
||||
|
||||
return issue_user_token(db_user, request)
|
||||
return await issue_user_token(db_user, request, db)
|
||||
|
||||
|
||||
@router.post("/dev-login", response_model=Token)
|
||||
@@ -295,7 +311,7 @@ async def dev_login_for_access_token(
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found")
|
||||
db_user = await authenticate_plain_password(payload, db)
|
||||
ensure_user_active(db_user)
|
||||
return issue_user_token(db_user, request)
|
||||
return await issue_user_token(db_user, request, db)
|
||||
|
||||
|
||||
@router.get("/me", response_model=UserRead)
|
||||
@@ -339,11 +355,47 @@ async def extend_access_token(
|
||||
max_age_seconds=policy.absolute_max_seconds,
|
||||
issued_at=issued_at,
|
||||
client_type=normalize_session_client_type(payload.get("client_type")),
|
||||
session_id=str(session_id_from_payload(payload)),
|
||||
)
|
||||
expires_at = policy_expires_at(issued_at, session_start, policy)
|
||||
return ExtendResponse(accessToken=new_token, expiresAt=expires_at)
|
||||
|
||||
|
||||
@router.post("/session/heartbeat")
|
||||
async def heartbeat_login_session(
|
||||
request: Request,
|
||||
token: str = Depends(oauth2_scheme),
|
||||
current_user=Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db_session),
|
||||
) -> dict:
|
||||
session = await touch_login_session(
|
||||
db,
|
||||
user_id=current_user.id,
|
||||
payload=decode_token(token),
|
||||
request=request,
|
||||
)
|
||||
if session is None:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="登录会话已结束")
|
||||
return {
|
||||
"status": "online",
|
||||
"last_seen_at": session.last_seen_at.isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/session/logout", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def logout_login_session(
|
||||
token: str = Depends(oauth2_scheme),
|
||||
current_user=Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db_session),
|
||||
) -> Response:
|
||||
await end_login_session(
|
||||
db,
|
||||
user_id=current_user.id,
|
||||
payload=decode_token(token),
|
||||
)
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
|
||||
@router.patch("/me", response_model=UserRead)
|
||||
async def update_me(
|
||||
payload: UserSelfUpdate,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -9,7 +9,8 @@ from app.schemas.common import PaginatedResponse
|
||||
from app.crud import user as user_crud
|
||||
from app.crud import member as member_crud
|
||||
from app.utils.pagination import paginate
|
||||
from app.schemas.user import UserCreate, UserRead, UserStatus, UserUpdate
|
||||
from app.schemas.user import UserCreate, UserLoginActivityRead, UserRead, UserStatus, UserUpdate
|
||||
from app.services.user_login_sessions import get_login_summaries, list_login_activities
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -25,7 +26,35 @@ async def list_users(
|
||||
) -> PaginatedResponse[UserRead]:
|
||||
users = await user_crud.list_users(db, skip=skip, limit=limit, keyword=keyword, status=user_status)
|
||||
total_users = await user_crud.count_users(db, keyword=keyword, status=user_status)
|
||||
return paginate(list(users), total=total_users)
|
||||
summaries = await get_login_summaries(db, [user.id for user in users])
|
||||
items = []
|
||||
for user in users:
|
||||
summary = summaries.get(user.id)
|
||||
item = UserRead.model_validate(user).model_dump()
|
||||
if summary:
|
||||
item.update(
|
||||
{
|
||||
"login_status": summary.status,
|
||||
"last_login_at": summary.last_login_at,
|
||||
"last_seen_at": summary.last_seen_at,
|
||||
"last_client_type": summary.client_type,
|
||||
"active_session_count": summary.active_session_count,
|
||||
}
|
||||
)
|
||||
items.append(item)
|
||||
return paginate(items, total=total_users)
|
||||
|
||||
|
||||
@router.get("/{user_id}/login-activities", response_model=list[UserLoginActivityRead])
|
||||
async def read_user_login_activities(
|
||||
user_id: uuid.UUID,
|
||||
limit: int = Query(default=30, ge=1, le=100),
|
||||
db: AsyncSession = Depends(get_db_session),
|
||||
current_user=Depends(require_roles(["ADMIN"])),
|
||||
) -> list[UserLoginActivityRead]:
|
||||
if not await user_crud.get_by_id(db, user_id):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="用户不存在")
|
||||
return await list_login_activities(db, user_id=user_id, limit=limit)
|
||||
|
||||
|
||||
@router.post("/", response_model=UserRead, status_code=status.HTTP_201_CREATED)
|
||||
|
||||
@@ -33,6 +33,18 @@ class Settings(BaseSettings):
|
||||
)
|
||||
IP2REGION_XDB_PATH: Optional[str] = None
|
||||
IP2REGION_IPV6_XDB_PATH: Optional[str] = None
|
||||
TRUSTED_PROXY_CIDRS: str = "127.0.0.1/32,::1/128,172.16.0.0/12"
|
||||
MONITORING_SERVER_PUBLIC_IP: Optional[str] = None
|
||||
MONITORING_PUBLIC_IP_DISCOVERY_URLS: str = (
|
||||
"https://api64.ipify.org,https://icanhazip.com"
|
||||
)
|
||||
MONITORING_PUBLIC_IP_DISCOVERY_TIMEOUT_SECONDS: float = Field(default=2.5, ge=0.5, le=10)
|
||||
MONITORING_SERVER_LOCATION_CACHE_SECONDS: int = Field(default=86400, ge=300, le=604800)
|
||||
MONITORING_ACCESS_LOG_RETENTION_DAYS: int = Field(default=90, ge=7, le=3650)
|
||||
MONITORING_METRIC_RETENTION_DAYS: int = Field(default=400, ge=30, le=3650)
|
||||
MONITORING_RETENTION_INTERVAL_SECONDS: int = Field(default=86400, ge=60, le=604800)
|
||||
USER_LOGIN_ACTIVITY_RETENTION_DAYS: int = Field(default=180, ge=30, le=3650)
|
||||
USER_SESSION_ONLINE_SECONDS: int = Field(default=300, ge=60, le=3600)
|
||||
|
||||
|
||||
@lru_cache
|
||||
|
||||
@@ -286,8 +286,9 @@ def _enqueue_permission_log(
|
||||
|
||||
writer = get_log_writer()
|
||||
if writer:
|
||||
forwarded = request.headers.get("x-forwarded-for")
|
||||
ip = forwarded.split(",")[0].strip() if forwarded else (request.client.host if request.client else None)
|
||||
from app.core.request_context import build_request_audit_context, get_request_audit_context
|
||||
|
||||
context = get_request_audit_context() or build_request_audit_context(request)
|
||||
writer.enqueue({
|
||||
"study_id": study_id,
|
||||
"user_id": user_id,
|
||||
@@ -295,7 +296,16 @@ def _enqueue_permission_log(
|
||||
"role": role,
|
||||
"allowed": allowed,
|
||||
"elapsed_ms": elapsed_ms,
|
||||
"ip_address": ip,
|
||||
"ip_address": context.client_ip,
|
||||
"user_agent": context.user_agent,
|
||||
"client_type": context.client_type,
|
||||
"client_version": context.client_version,
|
||||
"client_platform": context.client_platform,
|
||||
"build_channel": context.build_channel,
|
||||
"build_commit": context.build_commit,
|
||||
"request_headers": context.request_headers,
|
||||
"request_snapshot": context.request_snapshot,
|
||||
"request_id": context.request_id,
|
||||
})
|
||||
|
||||
|
||||
|
||||
@@ -2,14 +2,19 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import re
|
||||
import uuid
|
||||
from contextvars import ContextVar, Token
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RequestAuditContext:
|
||||
request_id: str | None = None
|
||||
client_ip: str | None = None
|
||||
user_agent: str | None = None
|
||||
client_type: str | None = None
|
||||
@@ -17,6 +22,8 @@ class RequestAuditContext:
|
||||
client_platform: str | None = None
|
||||
build_channel: str | None = None
|
||||
build_commit: str | None = None
|
||||
request_headers: dict[str, str] | None = None
|
||||
request_snapshot: dict[str, Any] | None = None
|
||||
|
||||
|
||||
_request_audit_context: ContextVar[RequestAuditContext | None] = ContextVar(
|
||||
@@ -25,11 +32,37 @@ _request_audit_context: ContextVar[RequestAuditContext | None] = ContextVar(
|
||||
)
|
||||
|
||||
_SENSITIVE_TEXT_PATTERN = re.compile(
|
||||
r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]+|((?:access_)?token|authorization)=([^&\s]+)"
|
||||
r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]+|((?:access_)?token|authorization|password|passwd|secret|credential|api[-_]?key|session)=([^&\s]+)"
|
||||
)
|
||||
_SENSITIVE_HEADER_NAME_PATTERN = re.compile(
|
||||
r"(?i)(authorization|cookie|set-cookie|token|password|passwd|secret|credential|api[-_]?key|session)"
|
||||
)
|
||||
_SAFE_REQUEST_HEADER_NAMES = frozenset(
|
||||
{
|
||||
"accept",
|
||||
"accept-language",
|
||||
"content-type",
|
||||
"host",
|
||||
"origin",
|
||||
"user-agent",
|
||||
"x-request-id",
|
||||
"x-correlation-id",
|
||||
}
|
||||
)
|
||||
_SAFE_REQUEST_HEADER_PREFIXES = ("x-ctms-",)
|
||||
_SENSITIVE_PARAMETER_NAME_PATTERN = re.compile(
|
||||
r"(?i)(token|authorization|password|passwd|secret|credential|api[-_]?key|session|"
|
||||
r"subject|participant|patient|user_?name|full_?name|email|phone|mobile|id_?card|"
|
||||
r"identity|certificate|contact|address)"
|
||||
)
|
||||
_KNOWN_CLIENT_TYPES = frozenset({"web", "desktop"})
|
||||
_CTMS_CLIENT_SOURCE_TO_TYPE = {
|
||||
"ctms-web": "web",
|
||||
"ctms-desktop": "desktop",
|
||||
}
|
||||
|
||||
|
||||
def _clean_header_value(value: str | None, max_length: int) -> str | None:
|
||||
def _clean_audit_value(value: str | None, max_length: int) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
cleaned = _SENSITIVE_TEXT_PATTERN.sub(lambda m: f"{m.group(1) or m.group(2) + '='}[redacted]", value.strip())
|
||||
@@ -38,26 +71,174 @@ def _clean_header_value(value: str | None, max_length: int) -> str | None:
|
||||
return cleaned[:max_length]
|
||||
|
||||
|
||||
def _clean_header_value(value: str | None, max_length: int) -> str | None:
|
||||
return _clean_audit_value(value, max_length)
|
||||
|
||||
|
||||
def resolve_ctms_client_type(headers: Any) -> str | None:
|
||||
source = _clean_header_value(headers.get("x-ctms-client-source"), 32)
|
||||
if source:
|
||||
mapped_type = _CTMS_CLIENT_SOURCE_TO_TYPE.get(source.strip().lower())
|
||||
if mapped_type:
|
||||
return mapped_type
|
||||
|
||||
client_type = _clean_header_value(headers.get("x-ctms-client-type"), 16)
|
||||
if not client_type:
|
||||
return None
|
||||
normalized = client_type.strip().lower()
|
||||
return normalized if normalized in _KNOWN_CLIENT_TYPES else normalized[:16]
|
||||
|
||||
|
||||
def build_sanitized_request_headers(request: Any) -> dict[str, str] | None:
|
||||
captured: dict[str, str] = {}
|
||||
for raw_name, raw_value in request.headers.items():
|
||||
name = str(raw_name).strip().lower()
|
||||
if not name:
|
||||
continue
|
||||
if _SENSITIVE_HEADER_NAME_PATTERN.search(name):
|
||||
captured[name] = "[redacted]"
|
||||
continue
|
||||
if name not in _SAFE_REQUEST_HEADER_NAMES and not name.startswith(_SAFE_REQUEST_HEADER_PREFIXES):
|
||||
continue
|
||||
value = _clean_header_value(str(raw_value), 500)
|
||||
if value:
|
||||
captured[name] = value
|
||||
return captured or None
|
||||
|
||||
|
||||
def _sanitize_named_value(raw_name: Any, raw_value: Any, max_length: int = 500) -> dict[str, str] | None:
|
||||
name = _clean_audit_value(str(raw_name), 120)
|
||||
if not name:
|
||||
return None
|
||||
if _SENSITIVE_PARAMETER_NAME_PATTERN.search(name):
|
||||
return {"name": name, "value": "[redacted]"}
|
||||
value = _clean_audit_value(str(raw_value), max_length)
|
||||
if value is None:
|
||||
return {"name": name, "value": ""}
|
||||
return {"name": name, "value": value}
|
||||
|
||||
|
||||
def _query_param_items(request: Any) -> list[dict[str, str]] | None:
|
||||
query_params = getattr(request, "query_params", None)
|
||||
if not query_params:
|
||||
return None
|
||||
if hasattr(query_params, "multi_items"):
|
||||
raw_items = query_params.multi_items()
|
||||
else:
|
||||
raw_items = query_params.items()
|
||||
items = [
|
||||
item
|
||||
for item in (_sanitize_named_value(name, value) for name, value in raw_items)
|
||||
if item is not None
|
||||
][:50]
|
||||
return items or None
|
||||
|
||||
|
||||
def _sanitized_query_string(items: list[dict[str, str]] | None) -> str | None:
|
||||
if not items:
|
||||
return None
|
||||
value = "&".join(f"{item['name']}={item['value']}" for item in items)
|
||||
return value[:2000] or None
|
||||
|
||||
|
||||
def _request_client_snapshot(request: Any) -> dict[str, str | int | None] | None:
|
||||
client = getattr(request, "client", None)
|
||||
if not client:
|
||||
return None
|
||||
host = _clean_audit_value(getattr(client, "host", None), 120)
|
||||
port = getattr(client, "port", None)
|
||||
if host is None and port is None:
|
||||
return None
|
||||
return {"host": host, "port": port}
|
||||
|
||||
|
||||
def build_request_snapshot(request: Any, *, request_id: str | None = None) -> dict[str, Any]:
|
||||
headers = getattr(request, "headers", {})
|
||||
scope = getattr(request, "scope", {}) or {}
|
||||
url = getattr(request, "url", None)
|
||||
query_params = _query_param_items(request)
|
||||
path = getattr(url, "path", None) or scope.get("path")
|
||||
method = getattr(request, "method", None) or scope.get("method")
|
||||
snapshot: dict[str, Any] = {
|
||||
"request_id": request_id,
|
||||
"method": _clean_audit_value(str(method).upper() if method else None, 12),
|
||||
"path": _clean_audit_value(path, 500),
|
||||
"query_string": _sanitized_query_string(query_params),
|
||||
"query_params": query_params,
|
||||
"headers": build_sanitized_request_headers(request),
|
||||
"http_version": _clean_audit_value(scope.get("http_version"), 16),
|
||||
"scheme": _clean_audit_value(getattr(url, "scheme", None) or scope.get("scheme"), 16),
|
||||
"client": _request_client_snapshot(request),
|
||||
"content": {
|
||||
"type": _clean_audit_value(headers.get("content-type"), 200),
|
||||
"length": _clean_audit_value(headers.get("content-length"), 32),
|
||||
},
|
||||
"body": {
|
||||
"captured": False,
|
||||
"reason": "body_not_captured_by_audit_policy",
|
||||
},
|
||||
}
|
||||
return {key: value for key, value in snapshot.items() if value not in (None, {}, [])}
|
||||
|
||||
|
||||
def _parse_ip(value: str | None):
|
||||
try:
|
||||
return ipaddress.ip_address((value or "").strip())
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _trusted_proxy_networks():
|
||||
networks = []
|
||||
for value in settings.TRUSTED_PROXY_CIDRS.split(","):
|
||||
candidate = value.strip()
|
||||
if not candidate:
|
||||
continue
|
||||
try:
|
||||
networks.append(ipaddress.ip_network(candidate, strict=False))
|
||||
except ValueError:
|
||||
continue
|
||||
return tuple(networks)
|
||||
|
||||
|
||||
def _is_trusted_proxy(value: str | None) -> bool:
|
||||
address = _parse_ip(value)
|
||||
return bool(address and any(address in network for network in _trusted_proxy_networks()))
|
||||
|
||||
|
||||
def resolve_client_ip(request: Any) -> str | None:
|
||||
peer_ip = request.client.host if request.client else None
|
||||
if not _is_trusted_proxy(peer_ip):
|
||||
return _clean_header_value(peer_ip, 45)
|
||||
|
||||
forwarded = request.headers.get("x-forwarded-for")
|
||||
if forwarded:
|
||||
return _clean_header_value(forwarded.split(",")[0].strip(), 45)
|
||||
chain = [part.strip() for part in forwarded.split(",") if _parse_ip(part)]
|
||||
chain.append(str(peer_ip))
|
||||
for candidate in reversed(chain):
|
||||
if not _is_trusted_proxy(candidate):
|
||||
return _clean_header_value(candidate, 45)
|
||||
|
||||
real_ip = request.headers.get("x-real-ip")
|
||||
if real_ip:
|
||||
if _parse_ip(real_ip):
|
||||
return _clean_header_value(real_ip.strip(), 45)
|
||||
return _clean_header_value(request.client.host if request.client else None, 45)
|
||||
return _clean_header_value(peer_ip, 45)
|
||||
|
||||
|
||||
def build_request_audit_context(request: Any) -> RequestAuditContext:
|
||||
headers = request.headers
|
||||
request_id = str(uuid.uuid4())
|
||||
return RequestAuditContext(
|
||||
request_id=request_id,
|
||||
client_ip=resolve_client_ip(request),
|
||||
user_agent=_clean_header_value(headers.get("user-agent"), 500),
|
||||
client_type=_clean_header_value(headers.get("x-ctms-client-type"), 16),
|
||||
client_type=resolve_ctms_client_type(headers),
|
||||
client_version=_clean_header_value(headers.get("x-ctms-client-version"), 32),
|
||||
client_platform=_clean_header_value(headers.get("x-ctms-client-platform"), 16),
|
||||
build_channel=_clean_header_value(headers.get("x-ctms-build-channel"), 16),
|
||||
build_commit=_clean_header_value(headers.get("x-ctms-build-commit"), 64),
|
||||
request_headers=build_sanitized_request_headers(request),
|
||||
request_snapshot=build_request_snapshot(request, request_id=request_id),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ def create_access_token(
|
||||
max_age_seconds: Optional[int] = None,
|
||||
issued_at: Optional[datetime] = None,
|
||||
client_type: Optional[str] = None,
|
||||
session_id: Optional[str] = None,
|
||||
) -> str:
|
||||
now = issued_at or datetime.now(timezone.utc)
|
||||
if now.tzinfo is None:
|
||||
@@ -42,6 +43,8 @@ def create_access_token(
|
||||
}
|
||||
if client_type:
|
||||
to_encode["client_type"] = client_type
|
||||
if session_id:
|
||||
to_encode["sid"] = session_id
|
||||
return jwt.encode(to_encode, settings.JWT_SECRET_KEY, algorithm=ALGORITHM)
|
||||
|
||||
|
||||
|
||||
@@ -42,6 +42,8 @@ from app.models.permission_access_log import PermissionAccessLog # noqa: F401
|
||||
from app.models.permission_metric_snapshot import PermissionMetricSnapshot # noqa: F401
|
||||
from app.models.permission_template import PermissionTemplate, PermissionTemplateVersion # noqa: F401
|
||||
from app.models.security_access_log import SecurityAccessLog # noqa: F401
|
||||
from app.models.source_location_snapshot import SourceLocationSnapshot # noqa: F401
|
||||
from app.models.user_login_session import UserLoginSession # noqa: F401
|
||||
from app.models.desktop_notification import ( # noqa: F401
|
||||
DesktopNotificationDelivery,
|
||||
DesktopNotificationSubscription,
|
||||
|
||||
+59
-3
@@ -5,8 +5,9 @@ import time
|
||||
from contextlib import asynccontextmanager
|
||||
from collections import defaultdict
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi import Depends, FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.api.v1.router import api_router
|
||||
@@ -19,15 +20,23 @@ from app.db.session import SessionLocal, engine
|
||||
from app.services.visit_scheduler import run_daily_lost_visit_job
|
||||
from app.services.permission_log_writer import start_log_writer, stop_log_writer
|
||||
from app.services.permission_metric_aggregator import run_hourly_metric_aggregation
|
||||
from app.services.source_location_aggregator import run_hourly_source_location_aggregation
|
||||
from app.services.monitoring_server_location import resolve_monitoring_server_location
|
||||
from app.services.monitoring_retention import run_monitoring_retention
|
||||
from app.services.security_access_log_writer import (
|
||||
get_security_log_writer,
|
||||
start_security_log_writer,
|
||||
stop_security_log_writer,
|
||||
)
|
||||
from app.core.security import decode_token
|
||||
from app.core.deps import get_db_session
|
||||
from app.core.request_context import (
|
||||
build_request_audit_context,
|
||||
build_request_snapshot,
|
||||
build_sanitized_request_headers,
|
||||
get_request_audit_context,
|
||||
reset_request_audit_context,
|
||||
resolve_ctms_client_type,
|
||||
resolve_client_ip,
|
||||
set_request_audit_context,
|
||||
)
|
||||
@@ -42,8 +51,12 @@ setup_config_stats: dict[str, int] = defaultdict(int)
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI):
|
||||
stop_event = asyncio.Event()
|
||||
await resolve_monitoring_server_location()
|
||||
scheduler_task = asyncio.create_task(run_daily_lost_visit_job(stop_event))
|
||||
aggregator_task = asyncio.create_task(run_hourly_metric_aggregation(stop_event))
|
||||
source_location_aggregator_task = asyncio.create_task(
|
||||
run_hourly_source_location_aggregation(stop_event)
|
||||
)
|
||||
await start_log_writer()
|
||||
await start_security_log_writer()
|
||||
if settings.ENV == "development":
|
||||
@@ -52,12 +65,15 @@ async def lifespan(_: FastAPI):
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
async with SessionLocal() as session:
|
||||
await ensure_admin_exists(session)
|
||||
retention_task = asyncio.create_task(run_monitoring_retention(stop_event))
|
||||
yield
|
||||
stop_event.set()
|
||||
await stop_log_writer()
|
||||
await stop_security_log_writer()
|
||||
await scheduler_task
|
||||
await aggregator_task
|
||||
await source_location_aggregator_task
|
||||
await retention_task
|
||||
|
||||
|
||||
async def _ensure_legacy_primary_keys(conn) -> None:
|
||||
@@ -129,19 +145,23 @@ def create_app() -> FastAPI:
|
||||
"Accept",
|
||||
"Authorization",
|
||||
"Content-Type",
|
||||
"X-CTMS-Client-Source",
|
||||
"X-CTMS-Client-Type",
|
||||
"X-CTMS-Client-Version",
|
||||
"X-CTMS-Client-Platform",
|
||||
"X-CTMS-Build-Channel",
|
||||
"X-CTMS-Build-Commit",
|
||||
"X-Request-ID",
|
||||
"X-Correlation-ID",
|
||||
],
|
||||
expose_headers=["X-Request-ID"],
|
||||
)
|
||||
|
||||
@app.middleware("http")
|
||||
async def setup_config_monitoring_middleware(request, call_next):
|
||||
path = request.url.path
|
||||
is_setup_config_path = "/api/v1/studies/" in path and "/setup-config" in path
|
||||
should_security_log = path.startswith("/api/")
|
||||
should_security_log = path.startswith("/api/") and path != "/api/v1/auth/session/heartbeat"
|
||||
started_at = time.perf_counter()
|
||||
audit_context = build_request_audit_context(request)
|
||||
audit_context_token = set_request_audit_context(audit_context)
|
||||
@@ -167,6 +187,8 @@ def create_app() -> FastAPI:
|
||||
raise
|
||||
|
||||
status_code = int(response.status_code)
|
||||
if audit_context.request_id:
|
||||
response.headers["X-Request-ID"] = audit_context.request_id
|
||||
if is_setup_config_path:
|
||||
normalized_path = UUID_RE.sub("{study_id}", path)
|
||||
duration_ms = int((time.perf_counter() - started_at) * 1000)
|
||||
@@ -229,6 +251,36 @@ def create_app() -> FastAPI:
|
||||
async def health() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.get(
|
||||
"/readyz",
|
||||
tags=["health"],
|
||||
summary="服务就绪检查",
|
||||
description="验证应用可访问数据库;失败时返回 503。",
|
||||
)
|
||||
async def readiness(db=Depends(get_db_session)):
|
||||
started_at = time.perf_counter()
|
||||
try:
|
||||
await db.execute(text("SELECT 1"))
|
||||
except Exception as exc:
|
||||
logger.exception("Readiness database check failed")
|
||||
return JSONResponse(
|
||||
status_code=503,
|
||||
content={
|
||||
"status": "not_ready",
|
||||
"database": {
|
||||
"status": "unhealthy",
|
||||
"error_type": type(exc).__name__,
|
||||
},
|
||||
},
|
||||
)
|
||||
return {
|
||||
"status": "ready",
|
||||
"database": {
|
||||
"status": "healthy",
|
||||
"latency_ms": round((time.perf_counter() - started_at) * 1000, 2),
|
||||
},
|
||||
}
|
||||
|
||||
@app.get(
|
||||
"/health/setup-config-stats",
|
||||
tags=["health"],
|
||||
@@ -271,6 +323,7 @@ def _enqueue_security_access_log(request, path: str, status_code: int, started_a
|
||||
if not writer:
|
||||
return
|
||||
auth_status, user_identifier = _resolve_auth_context(request)
|
||||
context = get_request_audit_context()
|
||||
writer.enqueue(
|
||||
{
|
||||
"method": request.method,
|
||||
@@ -279,11 +332,14 @@ def _enqueue_security_access_log(request, path: str, status_code: int, started_a
|
||||
"elapsed_ms": round((time.perf_counter() - started_at) * 1000, 2),
|
||||
"client_ip": resolve_client_ip(request),
|
||||
"user_agent": request.headers.get("user-agent"),
|
||||
"client_type": request.headers.get("x-ctms-client-type"),
|
||||
"client_type": resolve_ctms_client_type(request.headers),
|
||||
"client_version": request.headers.get("x-ctms-client-version"),
|
||||
"client_platform": request.headers.get("x-ctms-client-platform"),
|
||||
"build_channel": request.headers.get("x-ctms-build-channel"),
|
||||
"build_commit": request.headers.get("x-ctms-build-commit"),
|
||||
"request_headers": context.request_headers if context else build_sanitized_request_headers(request),
|
||||
"request_snapshot": context.request_snapshot if context else build_request_snapshot(request),
|
||||
"request_id": context.request_id if context else None,
|
||||
"auth_status": auth_status,
|
||||
"user_identifier": user_identifier,
|
||||
}
|
||||
|
||||
@@ -6,13 +6,15 @@ import uuid
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, Float, ForeignKey, Index, String
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy import JSON, Boolean, DateTime, Float, ForeignKey, Index, String
|
||||
from sqlalchemy.dialects.postgresql import JSONB, UUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from app.db.base_class import Base
|
||||
|
||||
JSONB_TYPE = JSON().with_variant(JSONB, "postgresql")
|
||||
|
||||
|
||||
class PermissionAccessLog(Base):
|
||||
__tablename__ = "permission_access_logs"
|
||||
@@ -22,6 +24,9 @@ class PermissionAccessLog(Base):
|
||||
Index("ix_perm_log_endpoint_created", "endpoint_key", "created_at"),
|
||||
Index("ix_perm_log_created_at", "created_at"),
|
||||
Index("ix_perm_log_allowed", "allowed", "created_at"),
|
||||
Index("ix_perm_log_ip_created", "ip_address", "created_at"),
|
||||
Index("ix_perm_log_client_source_created", "client_type", "created_at"),
|
||||
Index("ix_perm_log_request_id", "request_id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
@@ -38,6 +43,15 @@ class PermissionAccessLog(Base):
|
||||
allowed: Mapped[bool] = mapped_column(Boolean, nullable=False)
|
||||
elapsed_ms: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
ip_address: Mapped[Optional[str]] = mapped_column(String(45), nullable=True)
|
||||
user_agent: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
|
||||
client_type: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
|
||||
client_version: Mapped[Optional[str]] = mapped_column(String(32), nullable=True)
|
||||
client_platform: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
|
||||
build_channel: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
|
||||
build_commit: Mapped[Optional[str]] = mapped_column(String(64), nullable=True)
|
||||
request_headers: Mapped[Optional[dict]] = mapped_column(JSONB_TYPE, nullable=True)
|
||||
request_snapshot: Mapped[Optional[dict]] = mapped_column(JSONB_TYPE, nullable=True)
|
||||
request_id: Mapped[Optional[str]] = mapped_column(String(36), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
@@ -6,13 +6,15 @@ import uuid
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import DateTime, Float, Index, Integer, String
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy import JSON, DateTime, Float, Index, Integer, String
|
||||
from sqlalchemy.dialects.postgresql import JSONB, UUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from app.db.base_class import Base
|
||||
|
||||
JSONB_TYPE = JSON().with_variant(JSONB, "postgresql")
|
||||
|
||||
|
||||
class SecurityAccessLog(Base):
|
||||
__tablename__ = "security_access_logs"
|
||||
@@ -21,6 +23,9 @@ class SecurityAccessLog(Base):
|
||||
Index("ix_security_log_ip_created", "client_ip", "created_at"),
|
||||
Index("ix_security_log_status_created", "status_code", "created_at"),
|
||||
Index("ix_security_log_auth_created", "auth_status", "created_at"),
|
||||
Index("ix_security_log_request_id", "request_id"),
|
||||
Index("ix_security_log_category_created", "category", "created_at"),
|
||||
Index("ix_security_log_severity_created", "severity", "created_at"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
@@ -35,8 +40,13 @@ class SecurityAccessLog(Base):
|
||||
client_platform: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
|
||||
build_channel: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
|
||||
build_commit: Mapped[Optional[str]] = mapped_column(String(64), nullable=True)
|
||||
request_headers: Mapped[Optional[dict]] = mapped_column(JSONB_TYPE, nullable=True)
|
||||
request_snapshot: Mapped[Optional[dict]] = mapped_column(JSONB_TYPE, nullable=True)
|
||||
auth_status: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||
user_identifier: Mapped[Optional[str]] = mapped_column(String(80), nullable=True)
|
||||
request_id: Mapped[Optional[str]] = mapped_column(String(36), nullable=True)
|
||||
category: Mapped[Optional[str]] = mapped_column(String(30), nullable=True)
|
||||
severity: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Hourly source-location rollup for monitoring analytics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, Float, Index, Integer, String, UniqueConstraint
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from app.db.base_class import Base
|
||||
|
||||
|
||||
class SourceLocationSnapshot(Base):
|
||||
__tablename__ = "source_location_snapshots"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("bucket_time", "ip_hash", "user_hash", name="uq_source_location_bucket_identity"),
|
||||
Index("ix_source_location_bucket", "bucket_time"),
|
||||
Index("ix_source_location_country_bucket", "country_code", "bucket_time"),
|
||||
Index("ix_source_location_risk_bucket", "high_risk_count", "bucket_time"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
bucket_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
ip_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
user_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
country: Mapped[str] = mapped_column(String(100), nullable=False, default="")
|
||||
country_code: Mapped[str] = mapped_column(String(16), nullable=False, default="")
|
||||
province: Mapped[str] = mapped_column(String(100), nullable=False, default="")
|
||||
region_code: Mapped[str] = mapped_column(String(24), nullable=False, default="")
|
||||
city: Mapped[str] = mapped_column(String(100), nullable=False, default="")
|
||||
isp: Mapped[str] = mapped_column(String(160), nullable=False, default="")
|
||||
location: Mapped[str] = mapped_column(String(320), nullable=False, default="")
|
||||
longitude: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
latitude: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
accuracy_level: Mapped[str] = mapped_column(String(16), nullable=False, default="unknown")
|
||||
allowed_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
denied_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
security_event_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
high_risk_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
auth_failure_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -0,0 +1,32 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, String, func
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.db.base_class import Base
|
||||
|
||||
|
||||
class UserLoginSession(Base):
|
||||
"""Server-side login activity record without storing credentials or raw IPs."""
|
||||
|
||||
__tablename__ = "user_login_sessions"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
client_type: Mapped[str] = mapped_column(String(16), nullable=False, default="web")
|
||||
client_platform: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
client_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
client_source: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
login_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||
ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
end_reason: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
@@ -6,6 +6,7 @@ from typing import Literal, Optional
|
||||
from pydantic import BaseModel, ConfigDict, EmailStr, Field, field_validator
|
||||
|
||||
UserStatus = Literal["PENDING", "ACTIVE", "REJECTED", "DISABLED"]
|
||||
LoginStatus = Literal["ONLINE", "OFFLINE"]
|
||||
|
||||
PASSWORD_REGEX = re.compile(r"^(?=.*[A-Za-z])(?=.*\d).{8,}$")
|
||||
|
||||
@@ -60,6 +61,11 @@ class UserRead(BaseModel):
|
||||
approved_at: Optional[datetime] = None
|
||||
approved_by: Optional[uuid.UUID] = None
|
||||
avatar_url: Optional[str] = None
|
||||
login_status: LoginStatus = "OFFLINE"
|
||||
last_login_at: Optional[datetime] = None
|
||||
last_seen_at: Optional[datetime] = None
|
||||
last_client_type: Optional[Literal["web", "desktop"]] = None
|
||||
active_session_count: int = 0
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
@@ -78,6 +84,19 @@ class UserResponse(UserRead):
|
||||
pass
|
||||
|
||||
|
||||
class UserLoginActivityRead(BaseModel):
|
||||
id: uuid.UUID
|
||||
client_type: Literal["web", "desktop"]
|
||||
client_platform: Optional[str] = None
|
||||
client_version: Optional[str] = None
|
||||
login_at: datetime
|
||||
last_seen_at: datetime
|
||||
ended_at: Optional[datetime] = None
|
||||
end_reason: Optional[str] = None
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class UserSelfUpdate(_PasswordValidator):
|
||||
full_name: Optional[str] = None
|
||||
clinical_department: Optional[str] = None
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Canonical map metadata for monitoring source locations.
|
||||
|
||||
ip2region provides administrative names but no coordinates. This module keeps
|
||||
the normalization and centroid contract on the server so web and desktop
|
||||
clients render the same location semantics.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.services.ip_location import IpLocation
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GeoLocationMetadata:
|
||||
country: str
|
||||
country_code: str
|
||||
region_code: str
|
||||
longitude: float | None
|
||||
latitude: float | None
|
||||
accuracy_level: str
|
||||
|
||||
|
||||
COUNTRY_ALIASES = {
|
||||
"中国": "China",
|
||||
"China": "China",
|
||||
"Mainland China": "China",
|
||||
"中国香港": "China",
|
||||
"中国澳门": "China",
|
||||
"中国台湾": "China",
|
||||
"美国": "United States",
|
||||
"United States": "United States",
|
||||
"United States of America": "United States",
|
||||
"USA": "United States",
|
||||
"土耳其": "Türkiye",
|
||||
"Turkey": "Türkiye",
|
||||
"Türkiye": "Türkiye",
|
||||
}
|
||||
|
||||
COUNTRY_CODES = {
|
||||
"China": "CN",
|
||||
"United States": "US",
|
||||
"Netherlands": "NL",
|
||||
"Türkiye": "TR",
|
||||
"Australia": "AU",
|
||||
"Japan": "JP",
|
||||
"Singapore": "SG",
|
||||
"Germany": "DE",
|
||||
"France": "FR",
|
||||
"United Kingdom": "GB",
|
||||
"Canada": "CA",
|
||||
"India": "IN",
|
||||
"Russia": "RU",
|
||||
}
|
||||
|
||||
COUNTRY_CENTROIDS = {
|
||||
"China": (104.1954, 35.8617),
|
||||
"United States": (-95.7129, 37.0902),
|
||||
"Netherlands": (5.2913, 52.1326),
|
||||
"Türkiye": (35.2433, 38.9637),
|
||||
"Australia": (133.7751, -25.2744),
|
||||
"Japan": (138.2529, 36.2048),
|
||||
"Singapore": (103.8198, 1.3521),
|
||||
"Germany": (10.4515, 51.1657),
|
||||
"France": (2.2137, 46.2276),
|
||||
"United Kingdom": (-3.436, 55.3781),
|
||||
"Canada": (-106.3468, 56.1304),
|
||||
"India": (78.9629, 20.5937),
|
||||
"Russia": (105.3188, 61.524),
|
||||
}
|
||||
|
||||
CHINA_REGION_METADATA = {
|
||||
"北京市": ("CN-BJ", 116.4074, 39.9042),
|
||||
"天津市": ("CN-TJ", 117.2008, 39.0842),
|
||||
"河北省": ("CN-HE", 114.5025, 38.0455),
|
||||
"山西省": ("CN-SX", 112.5492, 37.857),
|
||||
"内蒙古自治区": ("CN-NM", 111.6708, 40.8183),
|
||||
"辽宁省": ("CN-LN", 123.4315, 41.8057),
|
||||
"吉林省": ("CN-JL", 125.3245, 43.8868),
|
||||
"黑龙江省": ("CN-HL", 126.6424, 45.7567),
|
||||
"上海市": ("CN-SH", 121.4737, 31.2304),
|
||||
"江苏省": ("CN-JS", 118.7633, 32.0617),
|
||||
"浙江省": ("CN-ZJ", 120.1551, 30.2741),
|
||||
"安徽省": ("CN-AH", 117.2272, 31.8206),
|
||||
"福建省": ("CN-FJ", 119.2965, 26.0745),
|
||||
"江西省": ("CN-JX", 115.8582, 28.682),
|
||||
"山东省": ("CN-SD", 117.1201, 36.6512),
|
||||
"河南省": ("CN-HA", 113.6254, 34.7466),
|
||||
"湖北省": ("CN-HB", 114.3055, 30.5928),
|
||||
"湖南省": ("CN-HN", 112.9388, 28.2282),
|
||||
"广东省": ("CN-GD", 113.2644, 23.1291),
|
||||
"广西壮族自治区": ("CN-GX", 108.3669, 22.817),
|
||||
"海南省": ("CN-HI", 110.3312, 20.0311),
|
||||
"重庆": ("CN-CQ", 106.5516, 29.563),
|
||||
"重庆市": ("CN-CQ", 106.5516, 29.563),
|
||||
"四川省": ("CN-SC", 104.0665, 30.5723),
|
||||
"贵州省": ("CN-GZ", 106.6302, 26.647),
|
||||
"云南省": ("CN-YN", 102.8329, 24.8801),
|
||||
"西藏自治区": ("CN-XZ", 91.1322, 29.6604),
|
||||
"陕西省": ("CN-SN", 108.9398, 34.3416),
|
||||
"甘肃省": ("CN-GS", 103.8343, 36.0611),
|
||||
"青海省": ("CN-QH", 101.7782, 36.6171),
|
||||
"宁夏回族自治区": ("CN-NX", 106.2309, 38.4872),
|
||||
"新疆维吾尔自治区": ("CN-XJ", 87.6168, 43.8256),
|
||||
"台湾省": ("CN-TW", 121.5654, 25.033),
|
||||
"香港特别行政区": ("CN-HK", 114.1694, 22.3193),
|
||||
"澳门特别行政区": ("CN-MO", 113.5439, 22.1987),
|
||||
}
|
||||
|
||||
CITY_CENTROIDS = {
|
||||
"南京": (118.7969, 32.0603),
|
||||
"南京市": (118.7969, 32.0603),
|
||||
"San Jose": (-121.8863, 37.3382),
|
||||
"South Holland": (4.493, 52.0208),
|
||||
"Istanbul": (28.9784, 41.0082),
|
||||
}
|
||||
|
||||
|
||||
def resolve_geo_location_metadata(location: IpLocation) -> GeoLocationMetadata:
|
||||
if location.location in {"局域网", "本机"}:
|
||||
return GeoLocationMetadata(
|
||||
country="",
|
||||
country_code="PRIVATE",
|
||||
region_code="PRIVATE",
|
||||
longitude=None,
|
||||
latitude=None,
|
||||
accuracy_level="private",
|
||||
)
|
||||
|
||||
country = COUNTRY_ALIASES.get(location.country, location.country)
|
||||
country_code = COUNTRY_CODES.get(country, "")
|
||||
|
||||
city_coordinate = CITY_CENTROIDS.get(location.city)
|
||||
if city_coordinate:
|
||||
return GeoLocationMetadata(
|
||||
country=country,
|
||||
country_code=country_code,
|
||||
region_code="",
|
||||
longitude=city_coordinate[0],
|
||||
latitude=city_coordinate[1],
|
||||
accuracy_level="city",
|
||||
)
|
||||
|
||||
region_metadata = CHINA_REGION_METADATA.get(location.province)
|
||||
if country in {"China", "中国"} and region_metadata:
|
||||
region_code, longitude, latitude = region_metadata
|
||||
return GeoLocationMetadata(
|
||||
country="China",
|
||||
country_code="CN",
|
||||
region_code=region_code,
|
||||
longitude=longitude,
|
||||
latitude=latitude,
|
||||
accuracy_level="region",
|
||||
)
|
||||
|
||||
country_coordinate = COUNTRY_CENTROIDS.get(country)
|
||||
if country_coordinate:
|
||||
return GeoLocationMetadata(
|
||||
country=country,
|
||||
country_code=country_code,
|
||||
region_code="",
|
||||
longitude=country_coordinate[0],
|
||||
latitude=country_coordinate[1],
|
||||
accuracy_level="country",
|
||||
)
|
||||
|
||||
return GeoLocationMetadata(
|
||||
country=country,
|
||||
country_code=country_code,
|
||||
region_code="",
|
||||
longitude=None,
|
||||
latitude=None,
|
||||
accuracy_level="unknown",
|
||||
)
|
||||
@@ -9,6 +9,7 @@ import ipaddress
|
||||
import logging
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
@@ -125,5 +126,6 @@ class Ip2RegionResolver:
|
||||
_resolver = Ip2RegionResolver()
|
||||
|
||||
|
||||
@lru_cache(maxsize=8192)
|
||||
def resolve_ip_location(ip: str | None) -> IpLocation:
|
||||
return _resolver.lookup(ip)
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
"""监测数据留存清理任务。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete
|
||||
|
||||
from app.core.config import settings
|
||||
from app.db.session import SessionLocal
|
||||
from app.models.permission_access_log import PermissionAccessLog
|
||||
from app.models.permission_metric_snapshot import PermissionMetricSnapshot
|
||||
from app.models.security_access_log import SecurityAccessLog
|
||||
from app.models.source_location_snapshot import SourceLocationSnapshot
|
||||
from app.models.user_login_session import UserLoginSession
|
||||
|
||||
logger = logging.getLogger("ctms.monitoring_retention")
|
||||
|
||||
|
||||
@dataclass
|
||||
class MonitoringRetentionState:
|
||||
running: bool = False
|
||||
last_run_started_at: datetime | None = None
|
||||
last_success_at: datetime | None = None
|
||||
last_error_at: datetime | None = None
|
||||
last_error_type: str | None = None
|
||||
error_count: int = 0
|
||||
last_deleted: dict[str, int] = field(default_factory=dict)
|
||||
total_deleted: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
def snapshot(self) -> dict[str, Any]:
|
||||
return {
|
||||
"running": self.running,
|
||||
"access_log_retention_days": settings.MONITORING_ACCESS_LOG_RETENTION_DAYS,
|
||||
"metric_retention_days": settings.MONITORING_METRIC_RETENTION_DAYS,
|
||||
"login_activity_retention_days": settings.USER_LOGIN_ACTIVITY_RETENTION_DAYS,
|
||||
"interval_seconds": settings.MONITORING_RETENTION_INTERVAL_SECONDS,
|
||||
"last_run_started_at": (
|
||||
self.last_run_started_at.isoformat() if self.last_run_started_at else None
|
||||
),
|
||||
"last_success_at": self.last_success_at.isoformat() if self.last_success_at else None,
|
||||
"last_error_at": self.last_error_at.isoformat() if self.last_error_at else None,
|
||||
"last_error_type": self.last_error_type,
|
||||
"error_count": self.error_count,
|
||||
"last_deleted": dict(self.last_deleted),
|
||||
"total_deleted": dict(self.total_deleted),
|
||||
}
|
||||
|
||||
|
||||
_state = MonitoringRetentionState()
|
||||
|
||||
|
||||
def get_monitoring_retention_status() -> dict[str, Any]:
|
||||
return _state.snapshot()
|
||||
|
||||
|
||||
def _deleted_count(result: Any) -> int:
|
||||
rowcount = getattr(result, "rowcount", 0)
|
||||
return max(0, int(rowcount or 0))
|
||||
|
||||
|
||||
async def purge_expired_monitoring_data(
|
||||
*, now: datetime | None = None
|
||||
) -> dict[str, int]:
|
||||
reference_time = now or datetime.now(timezone.utc)
|
||||
access_cutoff = reference_time - timedelta(
|
||||
days=settings.MONITORING_ACCESS_LOG_RETENTION_DAYS
|
||||
)
|
||||
metric_cutoff = reference_time - timedelta(
|
||||
days=settings.MONITORING_METRIC_RETENTION_DAYS
|
||||
)
|
||||
login_activity_cutoff = reference_time - timedelta(
|
||||
days=settings.USER_LOGIN_ACTIVITY_RETENTION_DAYS
|
||||
)
|
||||
|
||||
async with SessionLocal() as session:
|
||||
permission_result = await session.execute(
|
||||
delete(PermissionAccessLog).where(PermissionAccessLog.created_at < access_cutoff)
|
||||
)
|
||||
security_result = await session.execute(
|
||||
delete(SecurityAccessLog).where(SecurityAccessLog.created_at < access_cutoff)
|
||||
)
|
||||
metric_result = await session.execute(
|
||||
delete(PermissionMetricSnapshot).where(
|
||||
PermissionMetricSnapshot.bucket_time < metric_cutoff
|
||||
)
|
||||
)
|
||||
source_location_result = await session.execute(
|
||||
delete(SourceLocationSnapshot).where(
|
||||
SourceLocationSnapshot.bucket_time < access_cutoff
|
||||
)
|
||||
)
|
||||
login_session_result = await session.execute(
|
||||
delete(UserLoginSession).where(UserLoginSession.last_seen_at < login_activity_cutoff)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
return {
|
||||
"permission_access_logs": _deleted_count(permission_result),
|
||||
"security_access_logs": _deleted_count(security_result),
|
||||
"permission_metric_snapshots": _deleted_count(metric_result),
|
||||
"source_location_snapshots": _deleted_count(source_location_result),
|
||||
"user_login_sessions": _deleted_count(login_session_result),
|
||||
}
|
||||
|
||||
|
||||
async def run_monitoring_retention_once(
|
||||
*, now: datetime | None = None
|
||||
) -> dict[str, int]:
|
||||
_state.last_run_started_at = now or datetime.now(timezone.utc)
|
||||
try:
|
||||
deleted = await purge_expired_monitoring_data(now=now)
|
||||
except Exception as exc:
|
||||
_state.last_error_at = datetime.now(timezone.utc)
|
||||
_state.last_error_type = type(exc).__name__
|
||||
_state.error_count += 1
|
||||
raise
|
||||
|
||||
_state.last_success_at = datetime.now(timezone.utc)
|
||||
_state.last_deleted = deleted
|
||||
for key, count in deleted.items():
|
||||
_state.total_deleted[key] = _state.total_deleted.get(key, 0) + count
|
||||
return deleted
|
||||
|
||||
|
||||
async def run_monitoring_retention(stop_event: asyncio.Event) -> None:
|
||||
logger.info("Monitoring retention task started")
|
||||
_state.running = True
|
||||
try:
|
||||
while not stop_event.is_set():
|
||||
try:
|
||||
deleted = await run_monitoring_retention_once()
|
||||
if any(deleted.values()):
|
||||
logger.info("Purged expired monitoring data: %s", deleted)
|
||||
except Exception:
|
||||
logger.exception("Failed to purge expired monitoring data")
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
stop_event.wait(),
|
||||
timeout=settings.MONITORING_RETENTION_INTERVAL_SECONDS,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
finally:
|
||||
_state.running = False
|
||||
logger.info("Monitoring retention task stopped")
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Resolve the monitoring deployment's public network location.
|
||||
|
||||
The map server marker is derived from the deployment's public IP instead of a
|
||||
frontend or configuration coordinate. An explicit public IP is accepted for
|
||||
restricted networks; otherwise the public frontend hostname and, finally, a
|
||||
small set of public-IP discovery endpoints are used. Results are cached so the
|
||||
monitoring API never performs per-request network discovery.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import logging
|
||||
import socket
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.geo_location_metadata import resolve_geo_location_metadata
|
||||
from app.services.ip_location import resolve_ip_location
|
||||
|
||||
logger = logging.getLogger("ctms.monitoring_server_location")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MonitoringServerLocation:
|
||||
name: str
|
||||
longitude: float
|
||||
latitude: float
|
||||
accuracy_level: str
|
||||
resolution_source: str
|
||||
|
||||
def to_public_dict(self) -> dict:
|
||||
return {
|
||||
"name": self.name,
|
||||
"longitude": self.longitude,
|
||||
"latitude": self.latitude,
|
||||
"accuracy_level": self.accuracy_level,
|
||||
"resolution_source": self.resolution_source,
|
||||
}
|
||||
|
||||
|
||||
_cached_location: MonitoringServerLocation | None = None
|
||||
_cache_initialized = False
|
||||
_cache_expires_at = 0.0
|
||||
_cache_lock = asyncio.Lock()
|
||||
|
||||
|
||||
def _normalize_public_ip(value: str | None) -> str | None:
|
||||
candidate = (value or "").strip().splitlines()[0] if (value or "").strip() else ""
|
||||
try:
|
||||
address = ipaddress.ip_address(candidate)
|
||||
except ValueError:
|
||||
return None
|
||||
return str(address) if address.is_global else None
|
||||
|
||||
|
||||
async def _resolve_frontend_public_ip() -> str | None:
|
||||
hostname = urlparse(settings.FRONTEND_PUBLIC_URL).hostname
|
||||
if not hostname:
|
||||
return None
|
||||
literal_ip = _normalize_public_ip(hostname)
|
||||
if literal_ip:
|
||||
return literal_ip
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
records = await loop.getaddrinfo(hostname, None, type=socket.SOCK_STREAM)
|
||||
except (OSError, socket.gaierror):
|
||||
return None
|
||||
candidates = []
|
||||
for _, _, _, _, socket_address in records:
|
||||
candidate = _normalize_public_ip(str(socket_address[0]))
|
||||
if candidate and candidate not in candidates:
|
||||
candidates.append(candidate)
|
||||
return next((item for item in candidates if ":" not in item), candidates[0] if candidates else None)
|
||||
|
||||
|
||||
async def _discover_public_ip() -> tuple[str | None, str]:
|
||||
configured_ip = _normalize_public_ip(settings.MONITORING_SERVER_PUBLIC_IP)
|
||||
if configured_ip:
|
||||
return configured_ip, "configured_public_ip"
|
||||
|
||||
frontend_ip = await _resolve_frontend_public_ip()
|
||||
if frontend_ip:
|
||||
return frontend_ip, "frontend_dns"
|
||||
|
||||
if settings.ENV == "test":
|
||||
return None, "unavailable"
|
||||
|
||||
urls = [
|
||||
item.strip()
|
||||
for item in settings.MONITORING_PUBLIC_IP_DISCOVERY_URLS.split(",")
|
||||
if item.strip()
|
||||
]
|
||||
timeout = httpx.Timeout(settings.MONITORING_PUBLIC_IP_DISCOVERY_TIMEOUT_SECONDS)
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=False) as client:
|
||||
for url in urls:
|
||||
try:
|
||||
response = await client.get(url, headers={"Accept": "text/plain"})
|
||||
response.raise_for_status()
|
||||
except httpx.HTTPError:
|
||||
logger.warning("Public IP discovery endpoint unavailable", extra={"endpoint": url})
|
||||
continue
|
||||
discovered_ip = _normalize_public_ip(response.text[:128])
|
||||
if discovered_ip:
|
||||
return discovered_ip, "public_ip_discovery"
|
||||
return None, "unavailable"
|
||||
|
||||
|
||||
async def resolve_monitoring_server_location(*, force: bool = False) -> MonitoringServerLocation | None:
|
||||
global _cached_location, _cache_initialized, _cache_expires_at
|
||||
|
||||
now = time.monotonic()
|
||||
if not force and _cache_initialized and now < _cache_expires_at:
|
||||
return _cached_location
|
||||
|
||||
async with _cache_lock:
|
||||
now = time.monotonic()
|
||||
if not force and _cache_initialized and now < _cache_expires_at:
|
||||
return _cached_location
|
||||
|
||||
public_ip, resolution_source = await _discover_public_ip()
|
||||
location = None
|
||||
if public_ip:
|
||||
ip_location = resolve_ip_location(public_ip)
|
||||
metadata = resolve_geo_location_metadata(ip_location)
|
||||
if metadata.longitude is not None and metadata.latitude is not None:
|
||||
country = metadata.country or ip_location.country
|
||||
name = " / ".join(
|
||||
part for part in [country, ip_location.province, ip_location.city] if part
|
||||
) or "公网服务器"
|
||||
location = MonitoringServerLocation(
|
||||
name=name,
|
||||
longitude=metadata.longitude,
|
||||
latitude=metadata.latitude,
|
||||
accuracy_level=metadata.accuracy_level,
|
||||
resolution_source=resolution_source,
|
||||
)
|
||||
logger.info(
|
||||
"Monitoring server location resolved",
|
||||
extra={"resolution_source": resolution_source, "accuracy_level": metadata.accuracy_level},
|
||||
)
|
||||
else:
|
||||
logger.warning("Deployment public IP has no usable geo coordinates")
|
||||
else:
|
||||
logger.warning("Unable to discover the deployment public IP")
|
||||
|
||||
_cached_location = location
|
||||
_cache_initialized = True
|
||||
cache_seconds = settings.MONITORING_SERVER_LOCATION_CACHE_SECONDS if location else 300
|
||||
_cache_expires_at = now + cache_seconds
|
||||
return location
|
||||
|
||||
|
||||
def reset_monitoring_server_location_cache() -> None:
|
||||
"""Reset process-local state for configuration reloads and tests."""
|
||||
global _cached_location, _cache_initialized, _cache_expires_at
|
||||
_cached_location = None
|
||||
_cache_initialized = False
|
||||
_cache_expires_at = 0.0
|
||||
@@ -13,6 +13,7 @@ import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from app.db.session import SessionLocal
|
||||
from app.models.permission_access_log import PermissionAccessLog
|
||||
@@ -22,85 +23,150 @@ logger = logging.getLogger("ctms.permission_log_writer")
|
||||
BATCH_SIZE = 50
|
||||
FLUSH_INTERVAL = 5.0
|
||||
QUEUE_MAX_SIZE = 10000
|
||||
WRITE_ATTEMPTS = 2
|
||||
_QUEUE_STOP = object()
|
||||
|
||||
|
||||
class PermissionLogWriter:
|
||||
|
||||
def __init__(self):
|
||||
self._queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=QUEUE_MAX_SIZE)
|
||||
self._queue: asyncio.Queue[dict | object] = asyncio.Queue(maxsize=QUEUE_MAX_SIZE)
|
||||
self._task: asyncio.Task | None = None
|
||||
self._stopping = False
|
||||
self._started_at: datetime | None = None
|
||||
self._last_success_at: datetime | None = None
|
||||
self._last_error_at: datetime | None = None
|
||||
self._last_error_type: str | None = None
|
||||
self._accepted_entries = 0
|
||||
self._written_entries = 0
|
||||
self._dropped_entries = 0
|
||||
self._failed_batches = 0
|
||||
self._failed_entries = 0
|
||||
self._retry_count = 0
|
||||
|
||||
def enqueue(self, entry: dict) -> None:
|
||||
if self._stopping:
|
||||
self._dropped_entries += 1
|
||||
logger.warning("Permission log writer is stopping, dropping entry")
|
||||
return
|
||||
try:
|
||||
self._queue.put_nowait(entry)
|
||||
self._accepted_entries += 1
|
||||
except asyncio.QueueFull:
|
||||
self._dropped_entries += 1
|
||||
logger.warning("Permission log queue full, dropping entry")
|
||||
|
||||
async def start(self) -> None:
|
||||
if self._task and not self._task.done():
|
||||
return
|
||||
self._stopping = False
|
||||
self._started_at = datetime.now(timezone.utc)
|
||||
self._task = asyncio.create_task(self._flush_loop())
|
||||
logger.info("PermissionLogWriter started")
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
await self._drain()
|
||||
if self._task and not self._task.done():
|
||||
self._stopping = True
|
||||
await self._queue.put(_QUEUE_STOP)
|
||||
await self._task
|
||||
self._task = None
|
||||
logger.info("PermissionLogWriter stopped")
|
||||
|
||||
async def _flush_loop(self) -> None:
|
||||
while True:
|
||||
batch = await self._collect_batch()
|
||||
batch, should_stop = await self._collect_batch()
|
||||
if batch:
|
||||
await self._write_batch(batch)
|
||||
if should_stop:
|
||||
break
|
||||
|
||||
async def _collect_batch(self) -> list[dict]:
|
||||
async def _collect_batch(self) -> tuple[list[dict], bool]:
|
||||
batch: list[dict] = []
|
||||
try:
|
||||
first = await asyncio.wait_for(self._queue.get(), timeout=FLUSH_INTERVAL)
|
||||
if first is _QUEUE_STOP:
|
||||
return batch, True
|
||||
batch.append(first)
|
||||
except asyncio.TimeoutError:
|
||||
return batch
|
||||
return batch, False
|
||||
|
||||
while len(batch) < BATCH_SIZE:
|
||||
try:
|
||||
item = self._queue.get_nowait()
|
||||
if item is _QUEUE_STOP:
|
||||
return batch, True
|
||||
batch.append(item)
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
return batch
|
||||
return batch, False
|
||||
|
||||
async def _write_batch(self, batch: list[dict]) -> None:
|
||||
try:
|
||||
async with SessionLocal() as session:
|
||||
for entry in batch:
|
||||
log = PermissionAccessLog(
|
||||
id=uuid.uuid4(),
|
||||
study_id=entry["study_id"],
|
||||
user_id=entry["user_id"],
|
||||
endpoint_key=entry["endpoint_key"],
|
||||
role=entry["role"],
|
||||
allowed=entry["allowed"],
|
||||
elapsed_ms=entry["elapsed_ms"],
|
||||
ip_address=entry.get("ip_address"),
|
||||
created_at=entry.get("created_at", datetime.now(timezone.utc)),
|
||||
)
|
||||
session.add(log)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
logger.exception("Failed to write permission access log batch (%d entries)", len(batch))
|
||||
|
||||
async def _drain(self) -> None:
|
||||
batch: list[dict] = []
|
||||
while not self._queue.empty():
|
||||
async def _write_batch(self, batch: list[dict]) -> bool:
|
||||
for attempt in range(WRITE_ATTEMPTS):
|
||||
try:
|
||||
batch.append(self._queue.get_nowait())
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
if batch:
|
||||
await self._write_batch(batch)
|
||||
async with SessionLocal() as session:
|
||||
for entry in batch:
|
||||
log = PermissionAccessLog(
|
||||
id=uuid.uuid4(),
|
||||
study_id=entry["study_id"],
|
||||
user_id=entry["user_id"],
|
||||
endpoint_key=entry["endpoint_key"],
|
||||
role=entry["role"],
|
||||
allowed=entry["allowed"],
|
||||
elapsed_ms=entry["elapsed_ms"],
|
||||
ip_address=entry.get("ip_address"),
|
||||
user_agent=entry.get("user_agent"),
|
||||
client_type=entry.get("client_type"),
|
||||
client_version=entry.get("client_version"),
|
||||
client_platform=entry.get("client_platform"),
|
||||
build_channel=entry.get("build_channel"),
|
||||
build_commit=entry.get("build_commit"),
|
||||
request_headers=entry.get("request_headers"),
|
||||
request_snapshot=entry.get("request_snapshot"),
|
||||
request_id=entry.get("request_id"),
|
||||
created_at=entry.get("created_at", datetime.now(timezone.utc)),
|
||||
)
|
||||
session.add(log)
|
||||
await session.commit()
|
||||
except Exception as exc:
|
||||
self._last_error_at = datetime.now(timezone.utc)
|
||||
self._last_error_type = type(exc).__name__
|
||||
if attempt + 1 < WRITE_ATTEMPTS:
|
||||
self._retry_count += 1
|
||||
logger.warning(
|
||||
"Permission access log batch write failed; retrying (%d entries)",
|
||||
len(batch),
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
continue
|
||||
self._failed_batches += 1
|
||||
self._failed_entries += len(batch)
|
||||
logger.exception(
|
||||
"Failed to write permission access log batch after retries (%d entries)",
|
||||
len(batch),
|
||||
)
|
||||
return False
|
||||
self._written_entries += len(batch)
|
||||
self._last_success_at = datetime.now(timezone.utc)
|
||||
return True
|
||||
return False
|
||||
|
||||
def stats(self) -> dict[str, Any]:
|
||||
return {
|
||||
"running": bool(self._task and not self._task.done()),
|
||||
"stopping": self._stopping,
|
||||
"queue_size": self._queue.qsize(),
|
||||
"queue_capacity": self._queue.maxsize,
|
||||
"accepted_entries": self._accepted_entries,
|
||||
"written_entries": self._written_entries,
|
||||
"dropped_entries": self._dropped_entries,
|
||||
"failed_batches": self._failed_batches,
|
||||
"failed_entries": self._failed_entries,
|
||||
"retry_count": self._retry_count,
|
||||
"started_at": self._started_at.isoformat() if self._started_at else None,
|
||||
"last_success_at": self._last_success_at.isoformat() if self._last_success_at else None,
|
||||
"last_error_at": self._last_error_at.isoformat() if self._last_error_at else None,
|
||||
"last_error_type": self._last_error_type,
|
||||
}
|
||||
|
||||
|
||||
_writer: PermissionLogWriter | None = None
|
||||
|
||||
@@ -6,99 +6,170 @@ import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from app.db.session import SessionLocal
|
||||
from app.models.security_access_log import SecurityAccessLog
|
||||
from app.services.ip_location import resolve_ip_location
|
||||
from app.services.security_events import classify_security_event
|
||||
|
||||
logger = logging.getLogger("ctms.security_access_log_writer")
|
||||
|
||||
BATCH_SIZE = 100
|
||||
FLUSH_INTERVAL = 3.0
|
||||
QUEUE_MAX_SIZE = 20000
|
||||
WRITE_ATTEMPTS = 2
|
||||
_QUEUE_STOP = object()
|
||||
|
||||
|
||||
class SecurityAccessLogWriter:
|
||||
def __init__(self) -> None:
|
||||
self._queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=QUEUE_MAX_SIZE)
|
||||
self._queue: asyncio.Queue[dict | object] = asyncio.Queue(maxsize=QUEUE_MAX_SIZE)
|
||||
self._task: asyncio.Task | None = None
|
||||
self._stopping = False
|
||||
self._started_at: datetime | None = None
|
||||
self._last_success_at: datetime | None = None
|
||||
self._last_error_at: datetime | None = None
|
||||
self._last_error_type: str | None = None
|
||||
self._accepted_entries = 0
|
||||
self._written_entries = 0
|
||||
self._dropped_entries = 0
|
||||
self._failed_batches = 0
|
||||
self._failed_entries = 0
|
||||
self._retry_count = 0
|
||||
|
||||
def enqueue(self, entry: dict) -> None:
|
||||
if self._stopping:
|
||||
self._dropped_entries += 1
|
||||
logger.warning("Security access log writer is stopping, dropping entry")
|
||||
return
|
||||
try:
|
||||
self._queue.put_nowait(entry)
|
||||
self._accepted_entries += 1
|
||||
except asyncio.QueueFull:
|
||||
self._dropped_entries += 1
|
||||
logger.warning("Security access log queue full, dropping entry")
|
||||
|
||||
async def start(self) -> None:
|
||||
if self._task and not self._task.done():
|
||||
return
|
||||
self._stopping = False
|
||||
self._started_at = datetime.now(timezone.utc)
|
||||
self._task = asyncio.create_task(self._flush_loop())
|
||||
logger.info("SecurityAccessLogWriter started")
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
await self._drain()
|
||||
if self._task and not self._task.done():
|
||||
self._stopping = True
|
||||
await self._queue.put(_QUEUE_STOP)
|
||||
await self._task
|
||||
self._task = None
|
||||
logger.info("SecurityAccessLogWriter stopped")
|
||||
|
||||
async def _flush_loop(self) -> None:
|
||||
while True:
|
||||
batch = await self._collect_batch()
|
||||
batch, should_stop = await self._collect_batch()
|
||||
if batch:
|
||||
await self._write_batch(batch)
|
||||
if should_stop:
|
||||
break
|
||||
|
||||
async def _collect_batch(self) -> list[dict]:
|
||||
async def _collect_batch(self) -> tuple[list[dict], bool]:
|
||||
batch: list[dict] = []
|
||||
try:
|
||||
first = await asyncio.wait_for(self._queue.get(), timeout=FLUSH_INTERVAL)
|
||||
if first is _QUEUE_STOP:
|
||||
return batch, True
|
||||
batch.append(first)
|
||||
except asyncio.TimeoutError:
|
||||
return batch
|
||||
return batch, False
|
||||
|
||||
while len(batch) < BATCH_SIZE:
|
||||
try:
|
||||
batch.append(self._queue.get_nowait())
|
||||
item = self._queue.get_nowait()
|
||||
if item is _QUEUE_STOP:
|
||||
return batch, True
|
||||
batch.append(item)
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
return batch
|
||||
return batch, False
|
||||
|
||||
async def _write_batch(self, batch: list[dict]) -> None:
|
||||
try:
|
||||
async with SessionLocal() as session:
|
||||
for entry in batch:
|
||||
session.add(
|
||||
SecurityAccessLog(
|
||||
id=uuid.uuid4(),
|
||||
method=entry["method"],
|
||||
async def _write_batch(self, batch: list[dict]) -> bool:
|
||||
for attempt in range(WRITE_ATTEMPTS):
|
||||
try:
|
||||
async with SessionLocal() as session:
|
||||
for entry in batch:
|
||||
classification = classify_security_event(
|
||||
path=entry["path"],
|
||||
status_code=entry["status_code"],
|
||||
elapsed_ms=entry["elapsed_ms"],
|
||||
client_ip=entry.get("client_ip"),
|
||||
user_agent=entry.get("user_agent"),
|
||||
client_type=entry.get("client_type"),
|
||||
client_version=entry.get("client_version"),
|
||||
client_platform=entry.get("client_platform"),
|
||||
build_channel=entry.get("build_channel"),
|
||||
build_commit=entry.get("build_commit"),
|
||||
auth_status=entry["auth_status"],
|
||||
user_identifier=entry.get("user_identifier"),
|
||||
created_at=entry.get("created_at", datetime.now(timezone.utc)),
|
||||
ip_location=resolve_ip_location(entry.get("client_ip")),
|
||||
)
|
||||
session.add(
|
||||
SecurityAccessLog(
|
||||
id=uuid.uuid4(),
|
||||
method=entry["method"],
|
||||
path=entry["path"],
|
||||
status_code=entry["status_code"],
|
||||
elapsed_ms=entry["elapsed_ms"],
|
||||
client_ip=entry.get("client_ip"),
|
||||
user_agent=entry.get("user_agent"),
|
||||
client_type=entry.get("client_type"),
|
||||
client_version=entry.get("client_version"),
|
||||
client_platform=entry.get("client_platform"),
|
||||
build_channel=entry.get("build_channel"),
|
||||
build_commit=entry.get("build_commit"),
|
||||
request_headers=entry.get("request_headers"),
|
||||
request_snapshot=entry.get("request_snapshot"),
|
||||
request_id=entry.get("request_id"),
|
||||
category=classification["category"],
|
||||
severity=classification["severity"],
|
||||
auth_status=entry["auth_status"],
|
||||
user_identifier=entry.get("user_identifier"),
|
||||
created_at=entry.get("created_at", datetime.now(timezone.utc)),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
except Exception as exc:
|
||||
self._last_error_at = datetime.now(timezone.utc)
|
||||
self._last_error_type = type(exc).__name__
|
||||
if attempt + 1 < WRITE_ATTEMPTS:
|
||||
self._retry_count += 1
|
||||
logger.warning(
|
||||
"Security access log batch write failed; retrying (%d entries)",
|
||||
len(batch),
|
||||
)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
logger.exception("Failed to write security access log batch (%d entries)", len(batch))
|
||||
await asyncio.sleep(0)
|
||||
continue
|
||||
self._failed_batches += 1
|
||||
self._failed_entries += len(batch)
|
||||
logger.exception(
|
||||
"Failed to write security access log batch after retries (%d entries)",
|
||||
len(batch),
|
||||
)
|
||||
return False
|
||||
self._written_entries += len(batch)
|
||||
self._last_success_at = datetime.now(timezone.utc)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def _drain(self) -> None:
|
||||
batch: list[dict] = []
|
||||
while not self._queue.empty():
|
||||
try:
|
||||
batch.append(self._queue.get_nowait())
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
if batch:
|
||||
await self._write_batch(batch)
|
||||
def stats(self) -> dict[str, Any]:
|
||||
return {
|
||||
"running": bool(self._task and not self._task.done()),
|
||||
"stopping": self._stopping,
|
||||
"queue_size": self._queue.qsize(),
|
||||
"queue_capacity": self._queue.maxsize,
|
||||
"accepted_entries": self._accepted_entries,
|
||||
"written_entries": self._written_entries,
|
||||
"dropped_entries": self._dropped_entries,
|
||||
"failed_batches": self._failed_batches,
|
||||
"failed_entries": self._failed_entries,
|
||||
"retry_count": self._retry_count,
|
||||
"started_at": self._started_at.isoformat() if self._started_at else None,
|
||||
"last_success_at": self._last_success_at.isoformat() if self._last_success_at else None,
|
||||
"last_error_at": self._last_error_at.isoformat() if self._last_error_at else None,
|
||||
"last_error_type": self._last_error_type,
|
||||
}
|
||||
|
||||
|
||||
_writer: SecurityAccessLogWriter | None = None
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Security event classification shared by writers and monitoring APIs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.services.ip_location import IpLocation
|
||||
|
||||
|
||||
SENSITIVE_PROBE_MARKERS = (
|
||||
"/.env",
|
||||
".env",
|
||||
"/.git",
|
||||
".git/config",
|
||||
"backup",
|
||||
"config.php",
|
||||
"wp-config",
|
||||
"database.yml",
|
||||
)
|
||||
|
||||
def classify_security_event(
|
||||
*,
|
||||
path: str,
|
||||
status_code: int,
|
||||
auth_status: str,
|
||||
ip_location: IpLocation | None = None,
|
||||
) -> dict[str, str]:
|
||||
normalized_path = (path or "").lower()
|
||||
if any(marker in normalized_path for marker in SENSITIVE_PROBE_MARKERS):
|
||||
return {"category": "PROBE", "severity": "CRITICAL"}
|
||||
if status_code >= 500:
|
||||
return {"category": "SERVER_ERROR", "severity": "HIGH"}
|
||||
if auth_status == "INVALID_TOKEN":
|
||||
return {"category": "INVALID_TOKEN", "severity": "MEDIUM"}
|
||||
if auth_status == "ANONYMOUS" and status_code in {401, 403}:
|
||||
return {"category": "ANONYMOUS_API", "severity": "MEDIUM"}
|
||||
if status_code == 404:
|
||||
return {"category": "NOT_FOUND_NOISE", "severity": "LOW"}
|
||||
return {"category": "OTHER", "severity": "LOW"}
|
||||
@@ -0,0 +1,272 @@
|
||||
"""Hourly source-location aggregation and timeline reads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import hmac
|
||||
import logging
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Literal
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import settings
|
||||
from app.db.session import SessionLocal
|
||||
from app.models.permission_access_log import PermissionAccessLog
|
||||
from app.models.security_access_log import SecurityAccessLog
|
||||
from app.models.source_location_snapshot import SourceLocationSnapshot
|
||||
from app.services.geo_location_metadata import resolve_geo_location_metadata
|
||||
from app.services.ip_location import resolve_ip_location
|
||||
|
||||
logger = logging.getLogger("ctms.source_location_aggregator")
|
||||
|
||||
|
||||
def _normalize_datetime(value: datetime) -> datetime:
|
||||
return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _identity_hash(kind: str, value: str | None) -> str:
|
||||
if not value:
|
||||
return ""
|
||||
payload = f"source-location:{kind}:{value}".encode("utf-8")
|
||||
return hmac.new(settings.JWT_SECRET_KEY.encode("utf-8"), payload, hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
async def aggregate_source_location_hour(bucket_start: datetime, bucket_end: datetime) -> int:
|
||||
bucket_start = _normalize_datetime(bucket_start)
|
||||
bucket_end = _normalize_datetime(bucket_end)
|
||||
async with SessionLocal() as session:
|
||||
matching_security_request = select(SecurityAccessLog.id).where(
|
||||
PermissionAccessLog.request_id.is_not(None),
|
||||
SecurityAccessLog.request_id == PermissionAccessLog.request_id,
|
||||
).exists()
|
||||
permission_rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
PermissionAccessLog.ip_address,
|
||||
PermissionAccessLog.user_id,
|
||||
func.count().filter(PermissionAccessLog.allowed.is_(True)).label("allowed_count"),
|
||||
func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied_count"),
|
||||
func.min(PermissionAccessLog.created_at).label("first_seen_at"),
|
||||
func.max(PermissionAccessLog.created_at).label("last_seen_at"),
|
||||
)
|
||||
.where(
|
||||
PermissionAccessLog.created_at >= bucket_start,
|
||||
PermissionAccessLog.created_at < bucket_end,
|
||||
PermissionAccessLog.ip_address.is_not(None),
|
||||
~matching_security_request,
|
||||
)
|
||||
.group_by(PermissionAccessLog.ip_address, PermissionAccessLog.user_id)
|
||||
)
|
||||
).all()
|
||||
security_rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
SecurityAccessLog.client_ip,
|
||||
SecurityAccessLog.user_identifier,
|
||||
func.count().filter(SecurityAccessLog.status_code < 400).label("allowed_count"),
|
||||
func.count().filter(SecurityAccessLog.status_code >= 400).label("denied_count"),
|
||||
func.count()
|
||||
.filter(
|
||||
SecurityAccessLog.category.is_not(None),
|
||||
~SecurityAccessLog.category.in_(("OTHER", "NOT_FOUND_NOISE")),
|
||||
)
|
||||
.label("security_event_count"),
|
||||
func.count()
|
||||
.filter(SecurityAccessLog.severity.in_(("HIGH", "CRITICAL")))
|
||||
.label("high_risk_count"),
|
||||
func.count()
|
||||
.filter(
|
||||
SecurityAccessLog.status_code >= 400,
|
||||
SecurityAccessLog.auth_status.in_(("INVALID_TOKEN", "ANONYMOUS")),
|
||||
)
|
||||
.label("auth_failure_count"),
|
||||
func.min(SecurityAccessLog.created_at).label("first_seen_at"),
|
||||
func.max(SecurityAccessLog.created_at).label("last_seen_at"),
|
||||
)
|
||||
.where(
|
||||
SecurityAccessLog.created_at >= bucket_start,
|
||||
SecurityAccessLog.created_at < bucket_end,
|
||||
SecurityAccessLog.client_ip.is_not(None),
|
||||
)
|
||||
.group_by(SecurityAccessLog.client_ip, SecurityAccessLog.user_identifier)
|
||||
)
|
||||
).all()
|
||||
|
||||
aggregates: dict[tuple[str, str], dict] = {}
|
||||
|
||||
def merge_row(
|
||||
ip_address: str,
|
||||
user_identity: str,
|
||||
allowed_count: int,
|
||||
denied_count: int,
|
||||
first_seen_at: datetime,
|
||||
last_seen_at: datetime,
|
||||
*,
|
||||
security_event_count: int = 0,
|
||||
high_risk_count: int = 0,
|
||||
auth_failure_count: int = 0,
|
||||
) -> None:
|
||||
key = (ip_address, user_identity)
|
||||
row = aggregates.setdefault(
|
||||
key,
|
||||
{
|
||||
"ip_address": ip_address,
|
||||
"user_identity": user_identity,
|
||||
"allowed_count": 0,
|
||||
"denied_count": 0,
|
||||
"security_event_count": 0,
|
||||
"high_risk_count": 0,
|
||||
"auth_failure_count": 0,
|
||||
"first_seen_at": _normalize_datetime(first_seen_at),
|
||||
"last_seen_at": _normalize_datetime(last_seen_at),
|
||||
},
|
||||
)
|
||||
row["allowed_count"] += int(allowed_count or 0)
|
||||
row["denied_count"] += int(denied_count or 0)
|
||||
row["security_event_count"] += int(security_event_count or 0)
|
||||
row["high_risk_count"] += int(high_risk_count or 0)
|
||||
row["auth_failure_count"] += int(auth_failure_count or 0)
|
||||
row["first_seen_at"] = min(row["first_seen_at"], _normalize_datetime(first_seen_at))
|
||||
row["last_seen_at"] = max(row["last_seen_at"], _normalize_datetime(last_seen_at))
|
||||
|
||||
for ip_address, user_id, allowed, denied, first_seen, last_seen in permission_rows:
|
||||
merge_row(str(ip_address), str(user_id or ""), allowed, denied, first_seen, last_seen)
|
||||
for ip_address, user_identifier, allowed, denied, security_events, high_risk, auth_failures, first_seen, last_seen in security_rows:
|
||||
merge_row(
|
||||
str(ip_address),
|
||||
str(user_identifier or ""),
|
||||
allowed,
|
||||
denied,
|
||||
first_seen,
|
||||
last_seen,
|
||||
security_event_count=security_events,
|
||||
high_risk_count=high_risk,
|
||||
auth_failure_count=auth_failures,
|
||||
)
|
||||
|
||||
await session.execute(
|
||||
delete(SourceLocationSnapshot).where(SourceLocationSnapshot.bucket_time == bucket_start)
|
||||
)
|
||||
for row in aggregates.values():
|
||||
ip_info = resolve_ip_location(row["ip_address"])
|
||||
metadata = resolve_geo_location_metadata(ip_info)
|
||||
longitude = metadata.longitude
|
||||
latitude = metadata.latitude
|
||||
country = metadata.country or ip_info.country
|
||||
location = " / ".join(
|
||||
part for part in [country, ip_info.province, ip_info.city] if part
|
||||
) or ip_info.location or "未知"
|
||||
session.add(
|
||||
SourceLocationSnapshot(
|
||||
id=uuid.uuid4(),
|
||||
bucket_time=bucket_start,
|
||||
ip_hash=_identity_hash("ip", row["ip_address"]),
|
||||
user_hash=_identity_hash("user", row["user_identity"]),
|
||||
country=country,
|
||||
country_code=metadata.country_code,
|
||||
province=ip_info.province,
|
||||
region_code=metadata.region_code,
|
||||
city=ip_info.city,
|
||||
isp=ip_info.isp,
|
||||
location=location,
|
||||
longitude=longitude,
|
||||
latitude=latitude,
|
||||
accuracy_level=metadata.accuracy_level,
|
||||
allowed_count=row["allowed_count"],
|
||||
denied_count=row["denied_count"],
|
||||
security_event_count=row["security_event_count"],
|
||||
high_risk_count=row["high_risk_count"],
|
||||
auth_failure_count=row["auth_failure_count"],
|
||||
first_seen_at=row["first_seen_at"],
|
||||
last_seen_at=row["last_seen_at"],
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return len(aggregates)
|
||||
|
||||
|
||||
async def get_source_location_timeline(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
start_at: datetime,
|
||||
end_at: datetime,
|
||||
granularity: Literal["hour", "day"],
|
||||
) -> list[dict]:
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(SourceLocationSnapshot)
|
||||
.where(
|
||||
SourceLocationSnapshot.bucket_time >= start_at,
|
||||
SourceLocationSnapshot.bucket_time < end_at,
|
||||
)
|
||||
.order_by(SourceLocationSnapshot.bucket_time)
|
||||
)
|
||||
).scalars().all()
|
||||
buckets: dict[datetime, dict] = defaultdict(
|
||||
lambda: {
|
||||
"allowed_count": 0,
|
||||
"denied_count": 0,
|
||||
"security_event_count": 0,
|
||||
"high_risk_count": 0,
|
||||
"ip_hashes": set(),
|
||||
"user_hashes": set(),
|
||||
}
|
||||
)
|
||||
for row in rows:
|
||||
bucket = _normalize_datetime(row.bucket_time)
|
||||
if granularity == "day":
|
||||
bucket = bucket.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
else:
|
||||
bucket = bucket.replace(minute=0, second=0, microsecond=0)
|
||||
item = buckets[bucket]
|
||||
item["allowed_count"] += row.allowed_count
|
||||
item["denied_count"] += row.denied_count
|
||||
item["security_event_count"] += row.security_event_count
|
||||
item["high_risk_count"] += row.high_risk_count
|
||||
item["ip_hashes"].add(row.ip_hash)
|
||||
if row.user_hash:
|
||||
item["user_hashes"].add(row.user_hash)
|
||||
|
||||
return [
|
||||
{
|
||||
"bucket_time": bucket.isoformat(),
|
||||
"total_count": item["allowed_count"] + item["denied_count"],
|
||||
"allowed_count": item["allowed_count"],
|
||||
"denied_count": item["denied_count"],
|
||||
"security_event_count": item["security_event_count"],
|
||||
"high_risk_count": item["high_risk_count"],
|
||||
"unique_ip_count": len(item["ip_hashes"]),
|
||||
"unique_user_count": len(item["user_hashes"]),
|
||||
}
|
||||
for bucket, item in sorted(buckets.items())
|
||||
]
|
||||
|
||||
|
||||
async def run_hourly_source_location_aggregation(stop_event: asyncio.Event) -> None:
|
||||
logger.info("Source location aggregator started")
|
||||
now = datetime.now(timezone.utc)
|
||||
completed_hour = now.replace(minute=0, second=0, microsecond=0)
|
||||
try:
|
||||
await aggregate_source_location_hour(completed_hour - timedelta(hours=1), completed_hour)
|
||||
except Exception:
|
||||
logger.exception("Failed to backfill source location snapshot for %s", completed_hour)
|
||||
|
||||
while not stop_event.is_set():
|
||||
now = datetime.now(timezone.utc)
|
||||
next_hour = now.replace(minute=0, second=0, microsecond=0) + timedelta(hours=1)
|
||||
try:
|
||||
await asyncio.wait_for(stop_event.wait(), timeout=(next_hour - now).total_seconds())
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
try:
|
||||
await aggregate_source_location_hour(next_hour - timedelta(hours=1), next_hour)
|
||||
except Exception:
|
||||
logger.exception("Failed to aggregate source locations for %s", next_hour)
|
||||
|
||||
logger.info("Source location aggregator stopped")
|
||||
@@ -0,0 +1,198 @@
|
||||
"""Server-authoritative login session activity for the admin account list."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Iterable
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import settings
|
||||
from app.models.user_login_session import UserLoginSession
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UserLoginSummary:
|
||||
status: str = "OFFLINE"
|
||||
last_login_at: datetime | None = None
|
||||
last_seen_at: datetime | None = None
|
||||
client_type: str | None = None
|
||||
active_session_count: int = 0
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _as_utc(value: datetime) -> datetime:
|
||||
return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _text_header(headers: Any, name: str, limit: int) -> str | None:
|
||||
value = (headers.get(name) or "").strip()
|
||||
return value[:limit] or None
|
||||
|
||||
|
||||
def session_id_from_payload(payload: dict[str, Any]) -> uuid.UUID:
|
||||
raw_session_id = payload.get("sid")
|
||||
try:
|
||||
return uuid.UUID(str(raw_session_id))
|
||||
except (TypeError, ValueError):
|
||||
# Tokens issued before session tracking are mapped to a deterministic
|
||||
# legacy session without persisting the token or a token-derived value.
|
||||
legacy_key = ":".join(
|
||||
[
|
||||
str(payload.get("sub") or ""),
|
||||
str(payload.get("orig_iat") or payload.get("iat") or ""),
|
||||
str(payload.get("client_type") or "web"),
|
||||
]
|
||||
)
|
||||
return uuid.uuid5(uuid.NAMESPACE_URL, f"ctms:legacy-session:{legacy_key}")
|
||||
|
||||
|
||||
def session_client_type(payload: dict[str, Any], headers: Any) -> str:
|
||||
from_payload = (payload.get("client_type") or "").strip().lower()
|
||||
from_header = (headers.get("x-ctms-client-type") or "").strip().lower()
|
||||
return "desktop" if "desktop" in {from_payload, from_header} else "web"
|
||||
|
||||
|
||||
async def create_login_session(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
session_id: uuid.UUID,
|
||||
user_id: uuid.UUID,
|
||||
request: Any,
|
||||
login_at: datetime | None = None,
|
||||
) -> UserLoginSession:
|
||||
occurred_at = login_at or _now()
|
||||
session = UserLoginSession(
|
||||
id=session_id,
|
||||
user_id=user_id,
|
||||
client_type=session_client_type({}, request.headers),
|
||||
client_platform=_text_header(request.headers, "x-ctms-client-platform", 32),
|
||||
client_version=_text_header(request.headers, "x-ctms-client-version", 64),
|
||||
client_source=_text_header(request.headers, "x-ctms-client-source", 32),
|
||||
login_at=occurred_at,
|
||||
last_seen_at=occurred_at,
|
||||
)
|
||||
db.add(session)
|
||||
await db.commit()
|
||||
await db.refresh(session)
|
||||
return session
|
||||
|
||||
|
||||
async def touch_login_session(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
user_id: uuid.UUID,
|
||||
payload: dict[str, Any],
|
||||
request: Any,
|
||||
) -> UserLoginSession | None:
|
||||
session_id = session_id_from_payload(payload)
|
||||
session = await db.get(UserLoginSession, session_id)
|
||||
now = _now()
|
||||
if session is None:
|
||||
session = UserLoginSession(
|
||||
id=session_id,
|
||||
user_id=user_id,
|
||||
client_type=session_client_type(payload, request.headers),
|
||||
client_platform=_text_header(request.headers, "x-ctms-client-platform", 32),
|
||||
client_version=_text_header(request.headers, "x-ctms-client-version", 64),
|
||||
client_source=_text_header(request.headers, "x-ctms-client-source", 32),
|
||||
login_at=now,
|
||||
last_seen_at=now,
|
||||
)
|
||||
db.add(session)
|
||||
elif session.user_id != user_id or session.ended_at is not None:
|
||||
return None
|
||||
else:
|
||||
session.last_seen_at = now
|
||||
session.client_platform = _text_header(request.headers, "x-ctms-client-platform", 32) or session.client_platform
|
||||
session.client_version = _text_header(request.headers, "x-ctms-client-version", 64) or session.client_version
|
||||
await db.commit()
|
||||
await db.refresh(session)
|
||||
return session
|
||||
|
||||
|
||||
async def end_login_session(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
user_id: uuid.UUID,
|
||||
payload: dict[str, Any],
|
||||
reason: str = "logout",
|
||||
) -> bool:
|
||||
session_id = session_id_from_payload(payload)
|
||||
result = await db.execute(
|
||||
update(UserLoginSession)
|
||||
.where(
|
||||
UserLoginSession.id == session_id,
|
||||
UserLoginSession.user_id == user_id,
|
||||
UserLoginSession.ended_at.is_(None),
|
||||
)
|
||||
.values(ended_at=_now(), end_reason=reason, last_seen_at=_now())
|
||||
)
|
||||
await db.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
|
||||
async def get_login_summaries(
|
||||
db: AsyncSession,
|
||||
user_ids: Iterable[uuid.UUID],
|
||||
) -> dict[uuid.UUID, UserLoginSummary]:
|
||||
ids = list(user_ids)
|
||||
if not ids:
|
||||
return {}
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(UserLoginSession)
|
||||
.where(UserLoginSession.user_id.in_(ids))
|
||||
.order_by(UserLoginSession.user_id, UserLoginSession.login_at.desc())
|
||||
)
|
||||
).scalars().all()
|
||||
cutoff = _now() - timedelta(seconds=settings.USER_SESSION_ONLINE_SECONDS)
|
||||
summaries: dict[uuid.UUID, UserLoginSummary] = {}
|
||||
mutable: dict[uuid.UUID, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
current = mutable.setdefault(
|
||||
row.user_id,
|
||||
{
|
||||
"last_login_at": row.login_at,
|
||||
"last_seen_at": row.last_seen_at,
|
||||
"client_type": row.client_type,
|
||||
"active_session_count": 0,
|
||||
},
|
||||
)
|
||||
row_last_seen_at = _as_utc(row.last_seen_at)
|
||||
if row_last_seen_at and (
|
||||
current["last_seen_at"] is None or row_last_seen_at > _as_utc(current["last_seen_at"])
|
||||
):
|
||||
current["last_seen_at"] = row_last_seen_at
|
||||
if row.ended_at is None and row_last_seen_at >= cutoff:
|
||||
current["active_session_count"] += 1
|
||||
for user_id, item in mutable.items():
|
||||
summaries[user_id] = UserLoginSummary(
|
||||
status="ONLINE" if item["active_session_count"] else "OFFLINE",
|
||||
last_login_at=item["last_login_at"],
|
||||
last_seen_at=item["last_seen_at"],
|
||||
client_type=item["client_type"],
|
||||
active_session_count=item["active_session_count"],
|
||||
)
|
||||
return summaries
|
||||
|
||||
|
||||
async def list_login_activities(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
user_id: uuid.UUID,
|
||||
limit: int,
|
||||
) -> list[UserLoginSession]:
|
||||
result = await db.execute(
|
||||
select(UserLoginSession)
|
||||
.where(UserLoginSession.user_id == user_id)
|
||||
.order_by(UserLoginSession.login_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
@@ -246,7 +246,7 @@ async def test_project_member_can_read_own_effective_permissions(db_session: Asy
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_project_pm_monitoring_scope_is_limited_to_own_projects(db_session: AsyncSession):
|
||||
async def test_project_pm_cannot_access_system_monitoring(db_session: AsyncSession):
|
||||
pm_id = uuid.uuid4()
|
||||
own_study_id = uuid.uuid4()
|
||||
other_study_id = uuid.uuid4()
|
||||
@@ -256,12 +256,10 @@ async def test_project_pm_monitoring_scope_is_limited_to_own_projects(db_session
|
||||
await _seed_member(db_session, own_study_id, pm_id, "PM")
|
||||
await db_session.commit()
|
||||
|
||||
scope = await resolve_monitoring_scope(db_session, UserStub(id=pm_id))
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await resolve_monitoring_scope(db_session, UserStub(id=pm_id))
|
||||
|
||||
assert scope.is_admin is False
|
||||
assert scope.study_ids == {own_study_id}
|
||||
assert scope.can_access_study(own_study_id)
|
||||
assert not scope.can_access_study(other_study_id)
|
||||
assert exc_info.value.status_code == 403
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -54,6 +54,9 @@ class _DummyTask:
|
||||
def cancel(self):
|
||||
self.cancelled = True
|
||||
|
||||
def done(self):
|
||||
return False
|
||||
|
||||
def __await__(self):
|
||||
async def _wait():
|
||||
self.awaited = True
|
||||
@@ -106,6 +109,7 @@ async def test_development_startup_does_not_seed_business_data(monkeypatch):
|
||||
monkeypatch.setattr(main.settings, "ENV", "development")
|
||||
monkeypatch.setattr(main, "ensure_admin_exists", ensure_admin_exists)
|
||||
monkeypatch.setattr(main, "run_daily_lost_visit_job", fake_scheduler)
|
||||
monkeypatch.setattr(main, "resolve_monitoring_server_location", AsyncMock(return_value=None))
|
||||
monkeypatch.setattr(main.asyncio, "create_task", fake_create_task)
|
||||
monkeypatch.setattr(main, "engine", _DummyEngine())
|
||||
monkeypatch.setattr(main, "SessionLocal", fake_session_local)
|
||||
|
||||
@@ -4,6 +4,7 @@ import pytest
|
||||
|
||||
from app.core.request_context import (
|
||||
RequestAuditContext,
|
||||
build_request_audit_context,
|
||||
reset_request_audit_context,
|
||||
set_request_audit_context,
|
||||
)
|
||||
@@ -22,6 +23,48 @@ class FakeAsyncSession:
|
||||
self.flushed = True
|
||||
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, host: str = "10.0.0.8") -> None:
|
||||
self.host = host
|
||||
self.port = 54321
|
||||
|
||||
|
||||
class FakeUrl:
|
||||
scheme = "https"
|
||||
path = "/api/v1/audit"
|
||||
query = "token=secret-token&view=all&subject_name=Alice"
|
||||
|
||||
|
||||
class FakeQueryParams:
|
||||
def __init__(self) -> None:
|
||||
self._items = [
|
||||
("token", "secret-token"),
|
||||
("view", "all"),
|
||||
("subject_name", "Alice"),
|
||||
]
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
return True
|
||||
|
||||
def multi_items(self):
|
||||
return list(self._items)
|
||||
|
||||
|
||||
class FakeRequest:
|
||||
def __init__(self, headers: dict[str, str], *, client_host: str = "10.0.0.8") -> None:
|
||||
self.headers = headers
|
||||
self.client = FakeClient(client_host)
|
||||
self.method = "GET"
|
||||
self.url = FakeUrl()
|
||||
self.query_params = FakeQueryParams()
|
||||
self.scope = {
|
||||
"http_version": "1.1",
|
||||
"scheme": "https",
|
||||
"path": "/api/v1/audit",
|
||||
"method": "GET",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audit_log_action_captures_request_access_context():
|
||||
context_token = set_request_audit_context(
|
||||
@@ -60,3 +103,105 @@ async def test_audit_log_action_captures_request_access_context():
|
||||
assert log.client_platform == "macos"
|
||||
assert log.build_channel == "release"
|
||||
assert log.build_commit == "abcdef1"
|
||||
|
||||
|
||||
def test_request_audit_context_captures_sanitized_headers():
|
||||
context = build_request_audit_context(
|
||||
FakeRequest(
|
||||
{
|
||||
"user-agent": "Chrome",
|
||||
"referer": "https://ctms.local/audit?token=secret-token",
|
||||
"authorization": "Bearer secret-token",
|
||||
"cookie": "sid=secret",
|
||||
"x-ctms-client-source": "ctms-web",
|
||||
"x-ctms-client-type": "web",
|
||||
"x-ctms-client-version": "0.1.0",
|
||||
"x-request-id": "req-123",
|
||||
"x-custom-secret": "hidden",
|
||||
"x-random-debug": "skip-me",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert context.user_agent == "Chrome"
|
||||
assert context.client_type == "web"
|
||||
assert context.client_ip == "10.0.0.8"
|
||||
assert uuid.UUID(context.request_id)
|
||||
assert context.request_headers == {
|
||||
"authorization": "[redacted]",
|
||||
"cookie": "[redacted]",
|
||||
"user-agent": "Chrome",
|
||||
"x-ctms-client-source": "ctms-web",
|
||||
"x-ctms-client-type": "web",
|
||||
"x-ctms-client-version": "0.1.0",
|
||||
"x-custom-secret": "[redacted]",
|
||||
"x-request-id": "req-123",
|
||||
}
|
||||
assert context.request_snapshot is not None
|
||||
assert context.request_snapshot["method"] == "GET"
|
||||
assert context.request_snapshot["request_id"] == context.request_id
|
||||
assert context.request_snapshot["path"] == "/api/v1/audit"
|
||||
assert context.request_snapshot["query_string"] == (
|
||||
"token=[redacted]&view=all&subject_name=[redacted]"
|
||||
)
|
||||
assert context.request_snapshot["query_params"] == [
|
||||
{"name": "token", "value": "[redacted]"},
|
||||
{"name": "view", "value": "all"},
|
||||
{"name": "subject_name", "value": "[redacted]"},
|
||||
]
|
||||
assert context.request_snapshot["headers"]["authorization"] == "[redacted]"
|
||||
assert context.request_snapshot["headers"]["cookie"] == "[redacted]"
|
||||
assert context.request_snapshot["headers"]["x-ctms-client-source"] == "ctms-web"
|
||||
assert "referer" not in context.request_snapshot["headers"]
|
||||
assert "x-random-debug" not in context.request_snapshot["headers"]
|
||||
assert context.request_snapshot["client"] == {"host": "10.0.0.8", "port": 54321}
|
||||
assert context.request_snapshot["body"] == {
|
||||
"captured": False,
|
||||
"reason": "body_not_captured_by_audit_policy",
|
||||
}
|
||||
|
||||
|
||||
def test_request_audit_context_prefers_explicit_ctms_client_source():
|
||||
context = build_request_audit_context(
|
||||
FakeRequest(
|
||||
{
|
||||
"x-ctms-client-source": "ctms-desktop",
|
||||
"x-ctms-client-type": "web",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert context.client_type == "desktop"
|
||||
assert context.request_headers == {
|
||||
"x-ctms-client-source": "ctms-desktop",
|
||||
"x-ctms-client-type": "web",
|
||||
}
|
||||
|
||||
|
||||
def test_request_audit_context_uses_proxy_observed_ip_instead_of_spoofed_forwarded_prefix(monkeypatch):
|
||||
monkeypatch.setattr("app.core.request_context.settings.TRUSTED_PROXY_CIDRS", "10.0.0.0/8")
|
||||
context = build_request_audit_context(
|
||||
FakeRequest(
|
||||
{
|
||||
"x-real-ip": "203.0.113.20",
|
||||
"x-forwarded-for": "1.2.3.4, 203.0.113.20",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert context.client_ip == "203.0.113.20"
|
||||
|
||||
|
||||
def test_request_audit_context_ignores_forwarded_headers_from_untrusted_peer(monkeypatch):
|
||||
monkeypatch.setattr("app.core.request_context.settings.TRUSTED_PROXY_CIDRS", "10.0.0.0/8")
|
||||
context = build_request_audit_context(
|
||||
FakeRequest(
|
||||
{
|
||||
"x-real-ip": "203.0.113.20",
|
||||
"x-forwarded-for": "1.2.3.4",
|
||||
},
|
||||
client_host="198.51.100.8",
|
||||
)
|
||||
)
|
||||
|
||||
assert context.client_ip == "198.51.100.8"
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from app.models.permission_access_log import PermissionAccessLog
|
||||
from app.models.permission_metric_snapshot import PermissionMetricSnapshot
|
||||
from app.models.security_access_log import SecurityAccessLog
|
||||
from app.models.study import Study
|
||||
from app.models.user import User, UserStatus
|
||||
from app.services import monitoring_retention
|
||||
from app.services import permission_log_writer, security_access_log_writer
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
def __init__(self) -> None:
|
||||
self.added: list[object] = []
|
||||
self.committed = False
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def add(self, value: object) -> None:
|
||||
self.added.append(value)
|
||||
|
||||
async def commit(self) -> None:
|
||||
self.committed = True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_permission_writer_gracefully_flushes_and_reports_stats(monkeypatch):
|
||||
session = _FakeSession()
|
||||
monkeypatch.setattr(permission_log_writer, "SessionLocal", lambda: session)
|
||||
writer = permission_log_writer.PermissionLogWriter()
|
||||
await writer.start()
|
||||
writer.enqueue(
|
||||
{
|
||||
"study_id": uuid.uuid4(),
|
||||
"user_id": uuid.uuid4(),
|
||||
"endpoint_key": "subjects.read",
|
||||
"role": "PM",
|
||||
"allowed": True,
|
||||
"elapsed_ms": 2.5,
|
||||
}
|
||||
)
|
||||
|
||||
await writer.stop()
|
||||
|
||||
stats = writer.stats()
|
||||
assert session.committed is True
|
||||
assert len(session.added) == 1
|
||||
assert stats["running"] is False
|
||||
assert stats["accepted_entries"] == 1
|
||||
assert stats["written_entries"] == 1
|
||||
assert stats["dropped_entries"] == 0
|
||||
assert stats["queue_size"] == 0
|
||||
assert stats["last_success_at"] is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_permission_writer_retries_and_exposes_permanent_failure(monkeypatch):
|
||||
class _FailingSession:
|
||||
async def __aenter__(self):
|
||||
raise RuntimeError("database unavailable")
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(permission_log_writer, "SessionLocal", _FailingSession)
|
||||
writer = permission_log_writer.PermissionLogWriter()
|
||||
batch = [
|
||||
{
|
||||
"study_id": uuid.uuid4(),
|
||||
"user_id": uuid.uuid4(),
|
||||
"endpoint_key": "subjects.read",
|
||||
"role": "PM",
|
||||
"allowed": True,
|
||||
"elapsed_ms": 2.5,
|
||||
}
|
||||
]
|
||||
|
||||
assert await writer._write_batch(batch) is False
|
||||
|
||||
stats = writer.stats()
|
||||
assert stats["retry_count"] == 1
|
||||
assert stats["failed_batches"] == 1
|
||||
assert stats["failed_entries"] == 1
|
||||
assert stats["last_error_type"] == "RuntimeError"
|
||||
|
||||
|
||||
def test_writer_queue_overflow_is_counted():
|
||||
writer = security_access_log_writer.SecurityAccessLogWriter()
|
||||
writer._queue = asyncio.Queue(maxsize=1)
|
||||
|
||||
writer.enqueue({"path": "/first"})
|
||||
writer.enqueue({"path": "/dropped"})
|
||||
|
||||
stats = writer.stats()
|
||||
assert stats["accepted_entries"] == 1
|
||||
assert stats["dropped_entries"] == 1
|
||||
assert stats["queue_size"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retention_deletes_expired_rows_and_preserves_recent_rows(
|
||||
db_session, test_engine, monkeypatch
|
||||
):
|
||||
now = datetime(2026, 7, 10, tzinfo=timezone.utc)
|
||||
study_id = (await db_session.execute(select(Study.id).limit(1))).scalar_one()
|
||||
user = User(
|
||||
email=f"retention-{uuid.uuid4()}@example.com",
|
||||
password_hash="hash",
|
||||
full_name="Retention Test",
|
||||
clinical_department="QA",
|
||||
status=UserStatus.ACTIVE,
|
||||
)
|
||||
db_session.add(user)
|
||||
await db_session.flush()
|
||||
|
||||
old_permission = PermissionAccessLog(
|
||||
study_id=study_id,
|
||||
user_id=user.id,
|
||||
endpoint_key="old",
|
||||
role="PM",
|
||||
allowed=True,
|
||||
elapsed_ms=1,
|
||||
created_at=now - timedelta(days=91),
|
||||
)
|
||||
recent_permission = PermissionAccessLog(
|
||||
study_id=study_id,
|
||||
user_id=user.id,
|
||||
endpoint_key="recent",
|
||||
role="PM",
|
||||
allowed=True,
|
||||
elapsed_ms=1,
|
||||
created_at=now - timedelta(days=89),
|
||||
)
|
||||
old_security = SecurityAccessLog(
|
||||
method="GET",
|
||||
path="/old",
|
||||
status_code=404,
|
||||
elapsed_ms=1,
|
||||
auth_status="ANONYMOUS",
|
||||
created_at=now - timedelta(days=91),
|
||||
)
|
||||
recent_security = SecurityAccessLog(
|
||||
method="GET",
|
||||
path="/recent",
|
||||
status_code=200,
|
||||
elapsed_ms=1,
|
||||
auth_status="AUTHENTICATED",
|
||||
created_at=now - timedelta(days=89),
|
||||
)
|
||||
old_metric = PermissionMetricSnapshot(
|
||||
bucket_time=now - timedelta(days=401)
|
||||
)
|
||||
recent_metric = PermissionMetricSnapshot(
|
||||
bucket_time=now - timedelta(days=399)
|
||||
)
|
||||
db_session.add_all(
|
||||
[
|
||||
old_permission,
|
||||
recent_permission,
|
||||
old_security,
|
||||
recent_security,
|
||||
old_metric,
|
||||
recent_metric,
|
||||
]
|
||||
)
|
||||
await db_session.commit()
|
||||
old_permission_id = old_permission.id
|
||||
recent_permission_id = recent_permission.id
|
||||
old_security_id = old_security.id
|
||||
recent_security_id = recent_security.id
|
||||
old_metric_id = old_metric.id
|
||||
recent_metric_id = recent_metric.id
|
||||
|
||||
session_factory = async_sessionmaker(
|
||||
bind=test_engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False,
|
||||
)
|
||||
monkeypatch.setattr(monitoring_retention, "SessionLocal", session_factory)
|
||||
monkeypatch.setattr(
|
||||
monitoring_retention.settings, "MONITORING_ACCESS_LOG_RETENTION_DAYS", 90
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
monitoring_retention.settings, "MONITORING_METRIC_RETENTION_DAYS", 400
|
||||
)
|
||||
|
||||
deleted = await monitoring_retention.purge_expired_monitoring_data(now=now)
|
||||
|
||||
db_session.expire_all()
|
||||
assert deleted["permission_access_logs"] >= 1
|
||||
assert deleted["security_access_logs"] >= 1
|
||||
assert deleted["permission_metric_snapshots"] >= 1
|
||||
assert await db_session.get(PermissionAccessLog, old_permission_id) is None
|
||||
assert await db_session.get(SecurityAccessLog, old_security_id) is None
|
||||
assert await db_session.get(PermissionMetricSnapshot, old_metric_id) is None
|
||||
assert await db_session.get(PermissionAccessLog, recent_permission_id) is not None
|
||||
assert await db_session.get(SecurityAccessLog, recent_security_id) is not None
|
||||
assert await db_session.get(PermissionMetricSnapshot, recent_metric_id) is not None
|
||||
@@ -0,0 +1,53 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services import monitoring_server_location
|
||||
|
||||
|
||||
def test_public_ip_normalization_rejects_private_addresses():
|
||||
assert monitoring_server_location._normalize_public_ip("8.8.8.8\n") == "8.8.8.8"
|
||||
assert monitoring_server_location._normalize_public_ip("10.0.0.8") is None
|
||||
assert monitoring_server_location._normalize_public_ip("127.0.0.1") is None
|
||||
assert monitoring_server_location._normalize_public_ip("not-an-ip") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_location_is_derived_from_configured_public_ip(monkeypatch):
|
||||
monkeypatch.setattr(settings, "MONITORING_SERVER_PUBLIC_IP", "8.8.8.8")
|
||||
monkeypatch.setattr(
|
||||
monitoring_server_location,
|
||||
"resolve_ip_location",
|
||||
lambda ip: SimpleNamespace(
|
||||
location="United States / California / San Jose",
|
||||
country="United States",
|
||||
province="California",
|
||||
city="San Jose",
|
||||
isp="Google",
|
||||
),
|
||||
)
|
||||
monitoring_server_location.reset_monitoring_server_location_cache()
|
||||
|
||||
location = await monitoring_server_location.resolve_monitoring_server_location()
|
||||
|
||||
assert location is not None
|
||||
assert location.name == "United States / California / San Jose"
|
||||
assert location.longitude == -121.8863
|
||||
assert location.latitude == 37.3382
|
||||
assert location.resolution_source == "configured_public_ip"
|
||||
monitoring_server_location.reset_monitoring_server_location_cache()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_location_returns_none_instead_of_a_hardcoded_fallback(monkeypatch):
|
||||
async def no_public_ip():
|
||||
return None, "unavailable"
|
||||
|
||||
monkeypatch.setattr(monitoring_server_location, "_discover_public_ip", no_public_ip)
|
||||
monitoring_server_location.reset_monitoring_server_location_cache()
|
||||
|
||||
location = await monitoring_server_location.resolve_monitoring_server_location()
|
||||
|
||||
assert location is None
|
||||
monitoring_server_location.reset_monitoring_server_location_cache()
|
||||
@@ -1,7 +1,9 @@
|
||||
"""监控API测试:权限系统监控API端点验证。"""
|
||||
|
||||
import inspect
|
||||
import json
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
@@ -10,6 +12,7 @@ from sqlalchemy import text
|
||||
from app.core.permission_monitor import set_permission_monitor, PermissionMonitor
|
||||
from app.api.v1 import permission_monitoring
|
||||
from app.api.v1.system_permissions import list_system_permissions
|
||||
from app.services.monitoring_server_location import MonitoringServerLocation
|
||||
|
||||
|
||||
class FakeIpInfo:
|
||||
@@ -30,6 +33,18 @@ class ProjectPmUserStub:
|
||||
is_admin = False
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def disable_server_public_ip_discovery(monkeypatch):
|
||||
async def no_server_location():
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(
|
||||
permission_monitoring,
|
||||
"resolve_monitoring_server_location",
|
||||
no_server_location,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_monitoring_scope_rejects_project_pm(db_session):
|
||||
"""系统监测模块仅允许系统管理员访问,项目 PM 不再具备监测范围。"""
|
||||
@@ -103,9 +118,11 @@ async def _seed_permission_log(db_session, study_id: uuid.UUID, user_id: uuid.UU
|
||||
text(
|
||||
"""
|
||||
INSERT INTO permission_access_logs
|
||||
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, created_at)
|
||||
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address,
|
||||
request_snapshot, created_at)
|
||||
VALUES
|
||||
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address, CURRENT_TIMESTAMP)
|
||||
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address,
|
||||
:request_snapshot, CURRENT_TIMESTAMP)
|
||||
"""
|
||||
),
|
||||
{
|
||||
@@ -117,6 +134,7 @@ async def _seed_permission_log(db_session, study_id: uuid.UUID, user_id: uuid.UU
|
||||
"allowed": allowed,
|
||||
"elapsed_ms": elapsed_ms,
|
||||
"ip_address": "127.0.0.1",
|
||||
"request_snapshot": None,
|
||||
},
|
||||
)
|
||||
await db_session.commit()
|
||||
@@ -262,8 +280,8 @@ async def test_permission_system_health_degraded(db_session):
|
||||
assert any("权限拒绝率偏高" in issue for issue in data["issues"])
|
||||
assert any("缓存命中率偏低" in issue for issue in data["issues"])
|
||||
assert data["status"] == "unhealthy"
|
||||
assert data["health_score"] == 30
|
||||
assert data["score_breakdown"]["performance"]["deduction"] == 25
|
||||
assert data["health_score"] == 35
|
||||
assert data["score_breakdown"]["performance"]["deduction"] == 20
|
||||
assert data["score_breakdown"]["deny_rate"]["deduction"] == 20
|
||||
assert data["score_breakdown"]["cache"]["deduction"] == 25
|
||||
|
||||
@@ -279,6 +297,10 @@ async def test_permission_system_health_includes_metrics(db_session):
|
||||
assert "cache_stats" in data
|
||||
assert "score_breakdown" in data
|
||||
assert "sample_sufficient" in data
|
||||
assert "components" in data
|
||||
assert "storage" in data
|
||||
assert "runtime" in data["score_breakdown"]
|
||||
assert data["components"]["database"]["status"] == "healthy"
|
||||
assert "total_checks" in data["last_hour"]
|
||||
assert "cache_metrics" in data["cache_stats"]
|
||||
|
||||
@@ -304,6 +326,41 @@ async def test_permission_system_health_ignores_small_cache_sample(db_session):
|
||||
assert data["health_score"] == 100 - non_cache_deductions
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_permission_system_health_reports_writer_degradation(db_session, monkeypatch):
|
||||
monitor = PermissionMonitor()
|
||||
set_permission_monitor(monitor)
|
||||
writer = SimpleNamespace(
|
||||
stats=lambda: {
|
||||
"running": True,
|
||||
"queue_size": 3,
|
||||
"queue_capacity": 100,
|
||||
"failed_batches": 0,
|
||||
"dropped_entries": 2,
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(permission_monitoring, "get_log_writer", lambda: writer)
|
||||
monkeypatch.setattr(permission_monitoring, "get_security_log_writer", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
permission_monitoring,
|
||||
"get_monitoring_retention_status",
|
||||
lambda: {
|
||||
"running": False,
|
||||
"last_run_started_at": None,
|
||||
"last_success_at": None,
|
||||
"last_error_at": None,
|
||||
},
|
||||
)
|
||||
|
||||
data = await permission_monitoring.permission_system_health(
|
||||
db=db_session, _=AdminUserStub()
|
||||
)
|
||||
|
||||
assert data["components"]["permission_log_writer"]["status"] == "degraded"
|
||||
assert data["score_breakdown"]["runtime"]["deduction"] == 10
|
||||
assert any("权限日志写入器" in issue for issue in data["issues"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ip_locations_counts_unique_users_per_location(db_session, monkeypatch):
|
||||
"""IP 属地统计应按省市聚合访问次数、来源 IP 数和访问用户数。"""
|
||||
@@ -381,10 +438,26 @@ async def test_ip_locations_counts_unique_users_per_location(db_session, monkeyp
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
resolved_ips = []
|
||||
|
||||
def fake_resolve_ip_location(ip):
|
||||
resolved_ips.append(ip)
|
||||
return FakeIpInfo("广东省", "深圳市")
|
||||
|
||||
async def fake_resolve_monitoring_server_location():
|
||||
return MonitoringServerLocation(
|
||||
name="China / 重庆 / 重庆市",
|
||||
longitude=106.5516,
|
||||
latitude=29.563,
|
||||
accuracy_level="city",
|
||||
resolution_source="public_ip_discovery",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(permission_monitoring, "resolve_ip_location", fake_resolve_ip_location)
|
||||
monkeypatch.setattr(
|
||||
permission_monitoring,
|
||||
"resolve_ip_location",
|
||||
lambda ip: FakeIpInfo("广东省", "深圳市"),
|
||||
"resolve_monitoring_server_location",
|
||||
fake_resolve_monitoring_server_location,
|
||||
)
|
||||
|
||||
result = await permission_monitoring.get_ip_locations(db=db_session, _=AdminUserStub(), days=7, limit=10)
|
||||
@@ -396,6 +469,56 @@ async def test_ip_locations_counts_unique_users_per_location(db_session, monkeyp
|
||||
assert result["summary"]["total_count"] == 3
|
||||
assert result["summary"]["unique_ip_count"] == 2
|
||||
assert result["summary"]["unique_user_count"] == 2
|
||||
assert result["items"][0]["country_code"] == "CN"
|
||||
assert result["items"][0]["region_code"] == "CN-GD"
|
||||
assert result["items"][0]["longitude"] is not None
|
||||
assert result["items"][0]["risk_level"] == "attention"
|
||||
assert result["period"]["mode"] == "rolling"
|
||||
assert result["period"]["retention_days"] >= 7
|
||||
assert result["data_quality"]["located_ip_count"] == 2
|
||||
assert result["server_location"]["longitude"] == 106.5516
|
||||
assert result["server_location"]["resolution_source"] == "public_ip_discovery"
|
||||
assert sorted(resolved_ips) == ["10.1.1.1", "10.1.1.2"]
|
||||
|
||||
await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO permission_access_logs
|
||||
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, created_at)
|
||||
VALUES
|
||||
(lower(hex(randomblob(4))) || '-' || lower(hex(randomblob(2))) || '-4' ||
|
||||
substr(lower(hex(randomblob(2))),2) || '-' ||
|
||||
substr('89ab', abs(random()) % 4 + 1, 1) ||
|
||||
substr(lower(hex(randomblob(2))),2) || '-' || lower(hex(randomblob(6))),
|
||||
:study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address,
|
||||
datetime('now', '-120 days'))
|
||||
"""
|
||||
),
|
||||
{
|
||||
"study_id": study_id,
|
||||
"user_id": user_a,
|
||||
"endpoint_key": "admin.permissions.read",
|
||||
"role": "PM",
|
||||
"allowed": True,
|
||||
"elapsed_ms": 2.4,
|
||||
"ip_address": "10.1.1.3",
|
||||
},
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
resolved_ips.clear()
|
||||
all_result = await permission_monitoring.get_ip_locations(
|
||||
db=db_session,
|
||||
_=AdminUserStub(),
|
||||
days=7,
|
||||
limit=10,
|
||||
all_time=True,
|
||||
)
|
||||
|
||||
assert all_result["days"] is None
|
||||
assert all_result["summary"]["total_count"] == 4
|
||||
assert all_result["summary"]["unique_ip_count"] == 3
|
||||
assert sorted(resolved_ips) == ["10.1.1.1", "10.1.1.2", "10.1.1.3"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -530,9 +653,11 @@ async def test_ip_locations_keeps_private_network_location_label(db_session, mon
|
||||
text(
|
||||
"""
|
||||
INSERT INTO permission_access_logs
|
||||
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, created_at)
|
||||
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address,
|
||||
request_snapshot, created_at)
|
||||
VALUES
|
||||
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address, CURRENT_TIMESTAMP)
|
||||
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address,
|
||||
:request_snapshot, CURRENT_TIMESTAMP)
|
||||
"""
|
||||
),
|
||||
{
|
||||
@@ -544,6 +669,7 @@ async def test_ip_locations_keeps_private_network_location_label(db_session, mon
|
||||
"allowed": True,
|
||||
"elapsed_ms": 3.2,
|
||||
"ip_address": "192.168.1.10",
|
||||
"request_snapshot": None,
|
||||
},
|
||||
)
|
||||
await db_session.commit()
|
||||
@@ -556,8 +682,8 @@ async def test_ip_locations_keeps_private_network_location_label(db_session, mon
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ip_locations_includes_abnormal_security_ips(db_session, monkeypatch):
|
||||
"""来源分析应同时统计安全事件中的异常来源 IP。"""
|
||||
async def test_ip_locations_scores_security_sources_by_behavior(db_session, monkeypatch):
|
||||
"""来源分析应按安全行为而非地域评估来源风险。"""
|
||||
await db_session.execute(text("DELETE FROM permission_access_logs"))
|
||||
await db_session.execute(text("DELETE FROM security_access_logs"))
|
||||
await db_session.commit()
|
||||
@@ -603,9 +729,11 @@ async def test_ip_locations_includes_abnormal_security_ips(db_session, monkeypat
|
||||
text(
|
||||
"""
|
||||
INSERT INTO permission_access_logs
|
||||
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, created_at)
|
||||
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address,
|
||||
request_snapshot, created_at)
|
||||
VALUES
|
||||
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address, CURRENT_TIMESTAMP)
|
||||
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address,
|
||||
:request_snapshot, CURRENT_TIMESTAMP)
|
||||
"""
|
||||
),
|
||||
{
|
||||
@@ -617,9 +745,10 @@ async def test_ip_locations_includes_abnormal_security_ips(db_session, monkeypat
|
||||
"allowed": True,
|
||||
"elapsed_ms": 3.2,
|
||||
"ip_address": "10.3.1.1",
|
||||
"request_snapshot": None,
|
||||
},
|
||||
)
|
||||
for ip_address in ["8.8.8.8", "1.1.1.1"]:
|
||||
for ip_address, status_code in [("8.8.8.8", 403), ("1.1.1.1", 200)]:
|
||||
await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
@@ -633,7 +762,7 @@ async def test_ip_locations_includes_abnormal_security_ips(db_session, monkeypat
|
||||
"id": str(uuid.uuid4()),
|
||||
"method": "GET",
|
||||
"path": "/api/v1/admin",
|
||||
"status_code": 403,
|
||||
"status_code": status_code,
|
||||
"elapsed_ms": 8.5,
|
||||
"client_ip": ip_address,
|
||||
"user_agent": "pytest",
|
||||
@@ -658,18 +787,23 @@ async def test_ip_locations_includes_abnormal_security_ips(db_session, monkeypat
|
||||
locations = {item["country"]: item for item in result["items"]}
|
||||
|
||||
assert result["summary"]["total_count"] == 3
|
||||
assert result["summary"]["denied_count"] == 2
|
||||
assert result["summary"]["denied_count"] == 1
|
||||
assert result["summary"]["unique_ip_count"] == 3
|
||||
assert result["summary"]["unique_user_count"] == 1
|
||||
assert locations["United States"]["total_count"] == 1
|
||||
assert locations["United States"]["denied_count"] == 1
|
||||
assert locations["United States"]["risk_level"] == "high"
|
||||
assert locations["United States"]["country_code"] == "US"
|
||||
assert locations["Australia"]["total_count"] == 1
|
||||
assert locations["Australia"]["denied_count"] == 0
|
||||
assert locations["Australia"]["risk_level"] == "normal"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_access_logs_include_user_behavior_summary(db_session):
|
||||
"""访问日志应返回用户行为审计汇总和用户排行。"""
|
||||
await db_session.execute(text("DELETE FROM permission_access_logs"))
|
||||
await db_session.execute(text("DELETE FROM security_access_logs"))
|
||||
await db_session.commit()
|
||||
|
||||
study_id = "00000000-0000-0000-0000-000000000201"
|
||||
@@ -784,10 +918,300 @@ def test_access_logs_user_stats_query_avoids_postgresql_uuid_max():
|
||||
assert "func.max(PermissionAccessLog.user_id)" not in source
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_access_logs_filter_by_account_location_and_client_source(db_session, monkeypatch):
|
||||
"""访问审计应支持按账号、IP 属地和客户端来源排查。"""
|
||||
await db_session.execute(text("DELETE FROM permission_access_logs"))
|
||||
await db_session.execute(text("DELETE FROM security_access_logs"))
|
||||
await db_session.commit()
|
||||
|
||||
study_id = "00000000-0000-0000-0000-000000001401"
|
||||
user_a = "00000000-0000-0000-0000-000000001501"
|
||||
user_b = "00000000-0000-0000-0000-000000001502"
|
||||
|
||||
await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO studies (id, code, name, status, is_locked, visit_schedule, active_roles)
|
||||
VALUES (:id, :code, :name, :status, :is_locked, :visit_schedule, :active_roles)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": study_id,
|
||||
"code": "ACCESS-CONTEXT-STUDY",
|
||||
"name": "Access Context Study",
|
||||
"status": "ACTIVE",
|
||||
"is_locked": False,
|
||||
"visit_schedule": "[]",
|
||||
"active_roles": "[]",
|
||||
},
|
||||
)
|
||||
for user_id, email, name in [
|
||||
(user_a, "audit-shanghai@example.com", "上海审计用户"),
|
||||
(user_b, "audit-beijing@example.com", "北京审计用户"),
|
||||
]:
|
||||
await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO users (id, email, password_hash, full_name, clinical_department, is_admin, status)
|
||||
VALUES (:id, :email, :password_hash, :full_name, :clinical_department, :is_admin, :status)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": user_id,
|
||||
"email": email,
|
||||
"password_hash": "hash",
|
||||
"full_name": name,
|
||||
"clinical_department": "临床运营",
|
||||
"is_admin": False,
|
||||
"status": "ACTIVE",
|
||||
},
|
||||
)
|
||||
|
||||
rows = [
|
||||
(user_a, "203.0.113.10", "web", "Chrome", "macos", {"user-agent": "Chrome", "x-request-id": "req-web"}),
|
||||
(user_b, "198.51.100.20", "desktop", "CTMS Desktop", "windows", {"user-agent": "CTMS Desktop"}),
|
||||
(user_a, "10.10.10.10", None, "curl/8.0", None, {"user-agent": "curl/8.0"}),
|
||||
]
|
||||
for user, ip_address, client_type, user_agent, platform, request_headers in rows:
|
||||
await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO permission_access_logs
|
||||
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address,
|
||||
user_agent, client_type, client_version, client_platform, build_channel, build_commit, request_headers, created_at)
|
||||
VALUES
|
||||
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address,
|
||||
:user_agent, :client_type, :client_version, :client_platform, :build_channel, :build_commit, :request_headers, CURRENT_TIMESTAMP)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"study_id": study_id,
|
||||
"user_id": user,
|
||||
"endpoint_key": "admin.permissions.read",
|
||||
"role": "PM",
|
||||
"allowed": True,
|
||||
"elapsed_ms": 4.0,
|
||||
"ip_address": ip_address,
|
||||
"user_agent": user_agent,
|
||||
"client_type": client_type,
|
||||
"client_version": "0.1.0" if client_type else None,
|
||||
"client_platform": platform,
|
||||
"build_channel": "local" if client_type else None,
|
||||
"build_commit": "abcdef1" if client_type else None,
|
||||
"request_headers": json.dumps(request_headers),
|
||||
},
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
ip_info_by_address = {
|
||||
"203.0.113.10": FakeIpInfo("上海市", "上海市", "电信"),
|
||||
"198.51.100.20": FakeIpInfo("北京市", "北京市", "联通"),
|
||||
"10.10.10.10": FakeIpInfo("", "", "", "局域网"),
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
permission_monitoring,
|
||||
"resolve_ip_location",
|
||||
lambda ip: ip_info_by_address[ip],
|
||||
)
|
||||
|
||||
result = await permission_monitoring.get_access_logs(
|
||||
db=db_session,
|
||||
_=AdminUserStub(),
|
||||
study_id=None,
|
||||
user_id=None,
|
||||
endpoint_key=None,
|
||||
role=None,
|
||||
allowed=None,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
client_ip=None,
|
||||
client_type="web",
|
||||
keyword="上海",
|
||||
page=1,
|
||||
page_size=50,
|
||||
)
|
||||
|
||||
assert result["total"] == 1
|
||||
assert result["items"][0]["user_email"] == "audit-shanghai@example.com"
|
||||
assert result["items"][0]["client_type"] == "web"
|
||||
assert result["items"][0]["client_platform"] == "macos"
|
||||
assert result["items"][0]["user_agent"] == "Chrome"
|
||||
assert result["items"][0]["request_headers"]["x-request-id"] == "req-web"
|
||||
assert result["items"][0]["ip_city"] == "上海市"
|
||||
|
||||
unknown_result = await permission_monitoring.get_access_logs(
|
||||
db=db_session,
|
||||
_=AdminUserStub(),
|
||||
study_id=None,
|
||||
user_id=None,
|
||||
endpoint_key=None,
|
||||
role=None,
|
||||
allowed=None,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
client_ip=None,
|
||||
client_type="unknown",
|
||||
keyword="curl",
|
||||
page=1,
|
||||
page_size=50,
|
||||
)
|
||||
assert unknown_result["total"] == 1
|
||||
assert unknown_result["items"][0]["ip_address"] == "10.10.10.10"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_access_logs_include_security_events_for_admin(db_session, monkeypatch):
|
||||
"""访问日志应合并业务权限日志和安全访问日志,避免匿名/探测请求审计盲区。"""
|
||||
await db_session.execute(text("DELETE FROM permission_access_logs"))
|
||||
await db_session.execute(text("DELETE FROM security_access_logs"))
|
||||
await db_session.commit()
|
||||
|
||||
study_id = "00000000-0000-0000-0000-000000001601"
|
||||
user_id = "00000000-0000-0000-0000-000000001701"
|
||||
await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO studies (id, code, name, status, is_locked, visit_schedule, active_roles)
|
||||
VALUES (:id, :code, :name, :status, :is_locked, :visit_schedule, :active_roles)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": study_id,
|
||||
"code": "ACCESS-SECURITY-FEED",
|
||||
"name": "Access Security Feed",
|
||||
"status": "ACTIVE",
|
||||
"is_locked": False,
|
||||
"visit_schedule": "[]",
|
||||
"active_roles": "[]",
|
||||
},
|
||||
)
|
||||
await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO users (id, email, password_hash, full_name, clinical_department, is_admin, status)
|
||||
VALUES (:id, :email, :password_hash, :full_name, :clinical_department, :is_admin, :status)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": user_id,
|
||||
"email": "access-feed@example.com",
|
||||
"password_hash": "hash",
|
||||
"full_name": "统一访问用户",
|
||||
"clinical_department": "临床运营",
|
||||
"is_admin": False,
|
||||
"status": "ACTIVE",
|
||||
},
|
||||
)
|
||||
await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO permission_access_logs
|
||||
(id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address,
|
||||
request_snapshot, created_at)
|
||||
VALUES
|
||||
(:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address,
|
||||
:request_snapshot, CURRENT_TIMESTAMP)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"study_id": study_id,
|
||||
"user_id": user_id,
|
||||
"endpoint_key": "project_overview:read",
|
||||
"role": "PM",
|
||||
"allowed": True,
|
||||
"elapsed_ms": 3.0,
|
||||
"ip_address": "192.168.10.2",
|
||||
"request_snapshot": json.dumps(
|
||||
{
|
||||
"method": "GET",
|
||||
"path": "/api/v1/projects/overview",
|
||||
"query_string": "tab=summary",
|
||||
"headers": {"user-agent": "Chrome", "x-request-id": "perm-1"},
|
||||
"body": {"captured": False, "reason": "body_not_captured_by_audit_policy"},
|
||||
}
|
||||
),
|
||||
},
|
||||
)
|
||||
await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO security_access_logs
|
||||
(id, method, path, status_code, elapsed_ms, client_ip, user_agent, auth_status,
|
||||
user_identifier, request_headers, request_snapshot, created_at)
|
||||
VALUES
|
||||
(:id, :method, :path, :status_code, :elapsed_ms, :client_ip, :user_agent, :auth_status,
|
||||
:user_identifier, :request_headers, :request_snapshot, CURRENT_TIMESTAMP)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"method": "GET",
|
||||
"path": "/api/.git/config",
|
||||
"status_code": 404,
|
||||
"elapsed_ms": 1.5,
|
||||
"client_ip": "45.148.10.95",
|
||||
"user_agent": "probe-bot/1.0",
|
||||
"auth_status": "ANONYMOUS",
|
||||
"user_identifier": None,
|
||||
"request_headers": json.dumps({"user-agent": "probe-bot/1.0", "x-request-id": "sec-1"}),
|
||||
"request_snapshot": json.dumps(
|
||||
{
|
||||
"method": "GET",
|
||||
"path": "/api/.git/config",
|
||||
"query_string": "token=[redacted]",
|
||||
"headers": {"user-agent": "probe-bot/1.0", "authorization": "[redacted]"},
|
||||
"body": {"captured": False, "reason": "body_not_captured_by_audit_policy"},
|
||||
}
|
||||
),
|
||||
},
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
ip_info_by_address = {
|
||||
"192.168.10.2": FakeIpInfo("", "", "", "局域网"),
|
||||
"45.148.10.95": FakeIpInfo("South Holland", "", "", "Netherlands"),
|
||||
}
|
||||
monkeypatch.setattr(permission_monitoring, "resolve_ip_location", lambda ip: ip_info_by_address[ip])
|
||||
|
||||
result = await permission_monitoring.get_access_logs(
|
||||
db=db_session,
|
||||
_=AdminUserStub(),
|
||||
study_id=None,
|
||||
user_id=None,
|
||||
endpoint_key=None,
|
||||
role=None,
|
||||
allowed=None,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
client_ip=None,
|
||||
client_type=None,
|
||||
keyword=None,
|
||||
page=1,
|
||||
page_size=50,
|
||||
)
|
||||
|
||||
assert result["total"] == 2
|
||||
assert result["summary"]["total_count"] == 2
|
||||
assert result["summary"]["denied_count"] == 1
|
||||
items_by_type = {item["event_type"]: item for item in result["items"]}
|
||||
assert items_by_type["permission"]["endpoint_key"] == "project_overview:read"
|
||||
assert items_by_type["security"]["status_code"] == 404
|
||||
assert items_by_type["security"]["allowed"] is False
|
||||
assert items_by_type["security"]["category"] == "PROBE"
|
||||
assert items_by_type["security"]["severity"] == "CRITICAL"
|
||||
assert items_by_type["security"]["request_headers"]["x-request-id"] == "sec-1"
|
||||
assert items_by_type["permission"]["request_snapshot"]["path"] == "/api/v1/projects/overview"
|
||||
assert items_by_type["security"]["request_snapshot"]["headers"]["authorization"] == "[redacted]"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_access_logs_ip_ranking_aggregates_by_ip_total(db_session):
|
||||
"""IP 访问排行应按同一 IP 的整体访问量聚合排序,而不是按用户/IP/角色拆分。"""
|
||||
await db_session.execute(text("DELETE FROM permission_access_logs"))
|
||||
await db_session.execute(text("DELETE FROM security_access_logs"))
|
||||
await db_session.commit()
|
||||
|
||||
study_id = "00000000-0000-0000-0000-000000001201"
|
||||
@@ -896,10 +1320,13 @@ async def test_security_access_logs_include_anonymous_ip_attempts(db_session, mo
|
||||
text(
|
||||
"""
|
||||
INSERT INTO security_access_logs
|
||||
(id, method, path, status_code, elapsed_ms, client_ip, user_agent, auth_status, user_identifier, created_at)
|
||||
(id, method, path, status_code, elapsed_ms, client_ip, user_agent, auth_status,
|
||||
user_identifier, request_snapshot, created_at)
|
||||
VALUES
|
||||
('00000000-0000-4000-8000-000000000501', 'POST', '/api/v1/auth/login', 401, 12.5,
|
||||
'203.0.113.10', 'attack-bot/1.0', 'ANONYMOUS', NULL, CURRENT_TIMESTAMP)
|
||||
'203.0.113.10', 'attack-bot/1.0', 'ANONYMOUS', NULL,
|
||||
'{"method":"POST","path":"/api/v1/auth/login","headers":{"authorization":"[redacted]"}}',
|
||||
CURRENT_TIMESTAMP)
|
||||
"""
|
||||
)
|
||||
)
|
||||
@@ -925,6 +1352,7 @@ async def test_security_access_logs_include_anonymous_ip_attempts(db_session, mo
|
||||
assert result["items"][0]["ip_isp"] == "联通"
|
||||
assert result["items"][0]["account_label"] == "未知账号"
|
||||
assert result["items"][0]["status_code"] == 401
|
||||
assert result["items"][0]["request_snapshot"]["headers"]["authorization"] == "[redacted]"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -993,8 +1421,8 @@ async def test_security_access_logs_prioritize_sensitive_probe_over_foreign_ip(d
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_security_access_logs_classify_non_china_ip_as_abnormal(db_session, monkeypatch):
|
||||
"""安全事件明细应把非中国公网 IP 归类为异常 IP。"""
|
||||
async def test_security_access_logs_classify_foreign_invalid_token_by_behavior(db_session, monkeypatch):
|
||||
"""海外来源不应单凭地域升级风险,仍按认证行为分类。"""
|
||||
await db_session.execute(text("DELETE FROM security_access_logs"))
|
||||
monkeypatch.setattr(
|
||||
permission_monitoring,
|
||||
@@ -1024,8 +1452,8 @@ async def test_security_access_logs_classify_non_china_ip_as_abnormal(db_session
|
||||
)
|
||||
|
||||
item = result["items"][0]
|
||||
assert item["category"] == "ABNORMAL_IP"
|
||||
assert item["severity"] == "HIGH"
|
||||
assert item["category"] == "INVALID_TOKEN"
|
||||
assert item["severity"] == "MEDIUM"
|
||||
assert item["ip_location"] == "United States / California / San Jose / Google"
|
||||
assert item["ip_country"] == "United States"
|
||||
|
||||
@@ -1062,4 +1490,108 @@ async def test_security_access_logs_resolve_public_ip_with_packaged_ip2region(db
|
||||
assert item["ip_country"] == "United States"
|
||||
assert item["ip_province"] == "California"
|
||||
assert item["ip_city"] == "San Jose"
|
||||
assert item["category"] == "ABNORMAL_IP"
|
||||
assert item["category"] == "NOT_FOUND_NOISE"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_access_logs_deduplicate_permission_and_security_rows_by_request_id(db_session):
|
||||
await db_session.execute(text("DELETE FROM permission_access_logs"))
|
||||
await db_session.execute(text("DELETE FROM security_access_logs"))
|
||||
request_id = str(uuid.uuid4())
|
||||
study_id = str(uuid.uuid4())
|
||||
user_id = str(uuid.uuid4())
|
||||
await _seed_permission_log(db_session, uuid.UUID(study_id), uuid.UUID(user_id), allowed=False, elapsed_ms=4.0)
|
||||
await db_session.execute(
|
||||
text("UPDATE permission_access_logs SET request_id = :request_id"),
|
||||
{"request_id": request_id},
|
||||
)
|
||||
await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO security_access_logs
|
||||
(id, method, path, status_code, elapsed_ms, client_ip, auth_status, user_identifier,
|
||||
request_id, category, severity, created_at)
|
||||
VALUES
|
||||
(:id, 'GET', '/api/v1/studies/denied', 403, 5.0, '203.0.113.20', 'AUTHENTICATED', :user_id,
|
||||
:request_id, 'OTHER', 'LOW', CURRENT_TIMESTAMP)
|
||||
"""
|
||||
),
|
||||
{"id": str(uuid.uuid4()), "user_id": user_id, "request_id": request_id},
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
result = await permission_monitoring.get_access_logs(
|
||||
db=db_session,
|
||||
_=AdminUserStub(),
|
||||
study_id=None,
|
||||
user_id=None,
|
||||
endpoint_key=None,
|
||||
role=None,
|
||||
allowed=None,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
client_ip=None,
|
||||
client_type=None,
|
||||
keyword=None,
|
||||
page=1,
|
||||
page_size=50,
|
||||
)
|
||||
|
||||
assert result["total"] == 1
|
||||
assert result["items"][0]["event_type"] == "security"
|
||||
assert result["items"][0]["request_id"] == request_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_security_events_only_includes_persisted_probe(db_session):
|
||||
await db_session.execute(text("DELETE FROM security_access_logs"))
|
||||
for category, ip in [("PROBE", "52.53.218.145"), ("OTHER", "203.0.113.20")]:
|
||||
await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO security_access_logs
|
||||
(id, method, path, status_code, elapsed_ms, client_ip, auth_status,
|
||||
category, severity, created_at)
|
||||
VALUES
|
||||
(:id, 'GET', '/api/v1/studies/', 200, 3.0, :ip, 'AUTHENTICATED',
|
||||
:category, :severity, CURRENT_TIMESTAMP)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"ip": ip,
|
||||
"category": category,
|
||||
"severity": "CRITICAL" if category == "PROBE" else "LOW",
|
||||
},
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
result = await permission_monitoring.get_security_access_logs(
|
||||
db=db_session,
|
||||
_=AdminUserStub(),
|
||||
status_min=None,
|
||||
auth_status=None,
|
||||
events_only=True,
|
||||
page=1,
|
||||
page_size=20,
|
||||
)
|
||||
|
||||
assert result["total"] == 1
|
||||
assert result["items"][0]["category"] == "PROBE"
|
||||
assert result["summary"]["high_risk_count"] == 1
|
||||
|
||||
|
||||
def test_server_error_classification_takes_priority_over_foreign_ip():
|
||||
log = SimpleNamespace(
|
||||
path="/api/v1/studies/",
|
||||
status_code=503,
|
||||
auth_status="AUTHENTICATED",
|
||||
category=None,
|
||||
severity=None,
|
||||
)
|
||||
location = FakeIpInfo("California", "San Jose", "Google", "United States")
|
||||
|
||||
assert permission_monitoring._classify_security_access_log(log, location) == {
|
||||
"category": "SERVER_ERROR",
|
||||
"severity": "HIGH",
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ from datetime import datetime, timedelta, timezone
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
|
||||
from app.main import create_app
|
||||
from app.core.config import settings
|
||||
@@ -397,6 +398,7 @@ async def test_root_returns_service_metadata(client_and_db):
|
||||
client, _ = client_and_db
|
||||
resp = await client.get("/")
|
||||
assert resp.status_code == 200
|
||||
assert uuid.UUID(resp.headers["x-request-id"])
|
||||
assert resp.json() == {
|
||||
"service": "ctms-backend",
|
||||
"status": "ok",
|
||||
@@ -406,6 +408,17 @@ async def test_root_returns_service_metadata(client_and_db):
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_readiness_checks_database(client_and_db):
|
||||
client, _ = client_and_db
|
||||
resp = await client.get("/readyz")
|
||||
assert resp.status_code == 200
|
||||
payload = resp.json()
|
||||
assert payload["status"] == "ready"
|
||||
assert payload["database"]["status"] == "healthy"
|
||||
assert payload["database"]["latency_ms"] >= 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registered_user_can_login_after_email_verification(client_and_db):
|
||||
client, SessionLocal = client_and_db
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from app.models.source_location_snapshot import SourceLocationSnapshot
|
||||
from app.services.source_location_aggregator import _identity_hash, get_source_location_timeline
|
||||
|
||||
|
||||
def _snapshot(bucket_time: datetime, *, allowed: int, denied: int) -> SourceLocationSnapshot:
|
||||
return SourceLocationSnapshot(
|
||||
bucket_time=bucket_time,
|
||||
ip_hash=_identity_hash("ip", "203.0.113.10"),
|
||||
user_hash=_identity_hash("user", "user-1"),
|
||||
country="United States",
|
||||
country_code="US",
|
||||
province="California",
|
||||
region_code="",
|
||||
city="San Jose",
|
||||
isp="Example ISP",
|
||||
location="United States / California / San Jose",
|
||||
longitude=-121.8863,
|
||||
latitude=37.3382,
|
||||
accuracy_level="city",
|
||||
allowed_count=allowed,
|
||||
denied_count=denied,
|
||||
security_event_count=denied,
|
||||
high_risk_count=0,
|
||||
auth_failure_count=denied,
|
||||
first_seen_at=bucket_time,
|
||||
last_seen_at=bucket_time + timedelta(minutes=30),
|
||||
)
|
||||
|
||||
|
||||
def test_source_location_identity_hash_is_stable_and_does_not_expose_ip():
|
||||
first = _identity_hash("ip", "203.0.113.10")
|
||||
second = _identity_hash("ip", "203.0.113.10")
|
||||
|
||||
assert first == second
|
||||
assert len(first) == 64
|
||||
assert "203.0.113.10" not in first
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_source_location_timeline_supports_hour_and_day_rollups(db_session):
|
||||
start = datetime(2026, 7, 10, 8, tzinfo=timezone.utc)
|
||||
db_session.add(_snapshot(start, allowed=4, denied=1))
|
||||
db_session.add(_snapshot(start + timedelta(hours=1), allowed=3, denied=2))
|
||||
await db_session.commit()
|
||||
|
||||
hourly = await get_source_location_timeline(
|
||||
db_session,
|
||||
start_at=start,
|
||||
end_at=start + timedelta(hours=2),
|
||||
granularity="hour",
|
||||
)
|
||||
daily = await get_source_location_timeline(
|
||||
db_session,
|
||||
start_at=start,
|
||||
end_at=start + timedelta(hours=2),
|
||||
granularity="day",
|
||||
)
|
||||
|
||||
assert [item["total_count"] for item in hourly] == [5, 5]
|
||||
assert len(daily) == 1
|
||||
assert daily[0]["total_count"] == 10
|
||||
assert daily[0]["denied_count"] == 3
|
||||
assert daily[0]["unique_ip_count"] == 1
|
||||
assert daily[0]["unique_user_count"] == 1
|
||||
@@ -0,0 +1,61 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.config import settings
|
||||
from app.models.user import User, UserStatus
|
||||
from app.models.user_login_session import UserLoginSession
|
||||
from app.services.user_login_sessions import get_login_summaries, session_id_from_payload
|
||||
|
||||
|
||||
def test_legacy_session_id_is_stable_without_storing_a_token():
|
||||
payload = {"sub": "00000000-0000-0000-0000-000000000101", "orig_iat": 1_700_000_000, "client_type": "web"}
|
||||
|
||||
first = session_id_from_payload(payload)
|
||||
second = session_id_from_payload(payload)
|
||||
|
||||
assert first == second
|
||||
assert isinstance(first, uuid.UUID)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_login_summary_marks_recent_unended_sessions_online(db_session, monkeypatch):
|
||||
now = datetime.now(timezone.utc)
|
||||
monkeypatch.setattr(settings, "USER_SESSION_ONLINE_SECONDS", 300)
|
||||
user = User(
|
||||
id=uuid.uuid4(),
|
||||
email="login-summary@example.com",
|
||||
password_hash="hash",
|
||||
full_name="Login Summary",
|
||||
clinical_department="IT",
|
||||
status=UserStatus.ACTIVE,
|
||||
)
|
||||
db_session.add(user)
|
||||
db_session.add_all(
|
||||
[
|
||||
UserLoginSession(
|
||||
id=uuid.uuid4(),
|
||||
user_id=user.id,
|
||||
client_type="desktop",
|
||||
login_at=now - timedelta(minutes=10),
|
||||
last_seen_at=now - timedelta(seconds=20),
|
||||
),
|
||||
UserLoginSession(
|
||||
id=uuid.uuid4(),
|
||||
user_id=user.id,
|
||||
client_type="web",
|
||||
login_at=now - timedelta(days=1),
|
||||
last_seen_at=now - timedelta(days=1),
|
||||
ended_at=now - timedelta(days=1),
|
||||
end_reason="logout",
|
||||
),
|
||||
]
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
summaries = await get_login_summaries(db_session, [user.id])
|
||||
|
||||
assert summaries[user.id].status == "ONLINE"
|
||||
assert summaries[user.id].active_session_count == 1
|
||||
assert summaries[user.id].client_type == "desktop"
|
||||
Reference in New Issue
Block a user