From f11a5c84d9783003966fe12243d46995de4c7d2e Mon Sep 17 00:00:00 2001 From: Cheng Zhou Date: Fri, 10 Jul 2026 17:11:24 +0800 Subject: [PATCH] =?UTF-8?q?feat(=E7=9B=91=E6=8E=A7):=20=E5=AE=8C=E5=96=84?= =?UTF-8?q?=E7=B3=BB=E7=BB=9F=E7=9B=91=E6=8E=A7=E3=80=81=E7=99=BB=E5=BD=95?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E4=B8=8E=E8=AE=BF=E9=97=AE=E5=AE=A1=E8=AE=A1?= =?UTF-8?q?=E8=83=BD=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 +- ...260709_02_add_permission_access_context.py | 42 + ...3_add_permission_access_request_headers.py | 35 + ..._04_add_security_access_request_headers.py | 35 + ...05_add_request_snapshots_to_access_logs.py | 40 + ...260710_01_add_monitoring_event_identity.py | 67 + ...260710_02_add_source_location_snapshots.py | 93 + ...ove_private_source_snapshot_coordinates.py | 30 + .../20260710_04_add_user_login_sessions.py | 40 + backend/app/api/v1/auth.py | 62 +- backend/app/api/v1/permission_monitoring.py | 1029 +++++++-- backend/app/api/v1/users.py | 33 +- backend/app/core/config.py | 12 + backend/app/core/deps.py | 16 +- backend/app/core/request_context.py | 193 +- backend/app/core/security.py | 3 + backend/app/db/base.py | 2 + backend/app/main.py | 62 +- backend/app/models/permission_access_log.py | 18 +- backend/app/models/security_access_log.py | 14 +- .../app/models/source_location_snapshot.py | 48 + backend/app/models/user_login_session.py | 32 + backend/app/schemas/user.py | 19 + backend/app/services/geo_location_metadata.py | 175 ++ backend/app/services/ip_location.py | 2 + backend/app/services/monitoring_retention.py | 151 ++ .../services/monitoring_server_location.py | 164 ++ backend/app/services/permission_log_writer.py | 146 +- .../services/security_access_log_writer.py | 157 +- backend/app/services/security_events.py | 37 + .../services/source_location_aggregator.py | 272 +++ backend/app/services/user_login_sessions.py | 198 ++ backend/tests/test_admin_pm_permissions.py | 10 +- backend/tests/test_app_startup.py | 4 + backend/tests/test_audit_access_context.py | 145 ++ backend/tests/test_monitoring_runtime.py | 210 ++ .../tests/test_monitoring_server_location.py | 53 + .../tests/test_permission_monitoring_api.py | 576 ++++- backend/tests/test_registration.py | 13 + .../tests/test_source_location_aggregator.py | 68 + backend/tests/test_user_login_sessions.py | 61 + docker-compose.dev.yaml | 2 +- docker-compose.yaml | 20 +- docs/guides/system-monitoring-operations.md | 82 + frontend/src/api/authClient.ts | 19 + frontend/src/api/projectPermissions.ts | 12 +- frontend/src/api/users.ts | 7 +- .../AccountConnectionStatus.test.ts | 25 + .../components/AccountConnectionStatus.vue | 267 +++ frontend/src/components/DesktopLayout.vue | 10 +- .../src/components/Layout.desktop.test.ts | 9 + .../components/PermissionAccessLogs.test.ts | 343 ++- .../src/components/PermissionAccessLogs.vue | 1973 ++++++++++++----- .../components/PermissionIpLocations.test.ts | 82 + .../src/components/PermissionIpLocations.vue | 604 +++-- .../components/PermissionMonitoring.test.ts | 50 +- .../src/components/PermissionMonitoring.vue | 1270 +++++++++-- .../components/PermissionTrendCharts.test.ts | 43 + .../src/components/PermissionTrendCharts.vue | 233 +- .../src/components/SecurityCenter.test.ts | 121 +- frontend/src/components/SecurityCenter.vue | 1388 ++++++++++-- frontend/src/components/TimeRangeSelector.vue | 46 + frontend/src/components/WebLayout.vue | 4 + frontend/src/components/worldMapSource.ts | 110 - .../composables/useConnectionStatus.test.ts | 40 + .../src/composables/useConnectionStatus.ts | 99 + frontend/src/runtime/appMetadata.test.ts | 19 +- frontend/src/runtime/appMetadata.ts | 4 + frontend/src/session/sessionManager.ts | 6 +- frontend/src/styles/main.css | 144 +- frontend/src/types/api.ts | 172 +- frontend/src/views/Login.vue | 2 +- .../SystemMonitoringPage.desktop.test.ts | 19 + .../src/views/admin/SystemMonitoringPage.vue | 50 +- frontend/src/views/admin/UserForm.vue | 2 +- .../views/admin/UserLoginActivitiesDrawer.vue | 248 +++ .../src/views/admin/UserResetPassword.vue | 2 +- frontend/src/views/admin/Users.vue | 102 +- .../src/views/admin/UsersLoginStatus.test.ts | 47 + nginx/nginx.conf | 8 + nginx/nginx.dev.conf | 8 + scripts/install.sh | 1 + 82 files changed, 10283 insertions(+), 1781 deletions(-) create mode 100644 backend/alembic/versions/20260709_02_add_permission_access_context.py create mode 100644 backend/alembic/versions/20260709_03_add_permission_access_request_headers.py create mode 100644 backend/alembic/versions/20260709_04_add_security_access_request_headers.py create mode 100644 backend/alembic/versions/20260709_05_add_request_snapshots_to_access_logs.py create mode 100644 backend/alembic/versions/20260710_01_add_monitoring_event_identity.py create mode 100644 backend/alembic/versions/20260710_02_add_source_location_snapshots.py create mode 100644 backend/alembic/versions/20260710_03_remove_private_source_snapshot_coordinates.py create mode 100644 backend/alembic/versions/20260710_04_add_user_login_sessions.py create mode 100644 backend/app/models/source_location_snapshot.py create mode 100644 backend/app/models/user_login_session.py create mode 100644 backend/app/services/geo_location_metadata.py create mode 100644 backend/app/services/monitoring_retention.py create mode 100644 backend/app/services/monitoring_server_location.py create mode 100644 backend/app/services/security_events.py create mode 100644 backend/app/services/source_location_aggregator.py create mode 100644 backend/app/services/user_login_sessions.py create mode 100644 backend/tests/test_monitoring_runtime.py create mode 100644 backend/tests/test_monitoring_server_location.py create mode 100644 backend/tests/test_source_location_aggregator.py create mode 100644 backend/tests/test_user_login_sessions.py create mode 100644 docs/guides/system-monitoring-operations.md create mode 100644 frontend/src/components/AccountConnectionStatus.test.ts create mode 100644 frontend/src/components/AccountConnectionStatus.vue create mode 100644 frontend/src/components/PermissionIpLocations.test.ts create mode 100644 frontend/src/components/PermissionTrendCharts.test.ts create mode 100644 frontend/src/components/TimeRangeSelector.vue delete mode 100644 frontend/src/components/worldMapSource.ts create mode 100644 frontend/src/composables/useConnectionStatus.test.ts create mode 100644 frontend/src/composables/useConnectionStatus.ts create mode 100644 frontend/src/views/admin/SystemMonitoringPage.desktop.test.ts create mode 100644 frontend/src/views/admin/UserLoginActivitiesDrawer.vue create mode 100644 frontend/src/views/admin/UsersLoginStatus.test.ts diff --git a/README.md b/README.md index 4ec818b3..91bb2968 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ - `docker compose config` - `curl -i http://127.0.0.1:8888/` - `curl -i http://127.0.0.1:8888/health` + - `curl -i http://127.0.0.1:8888/readyz` ## 账号与注册 - 初始化管理员：`admin@huapont.cn / admin123`（通过生产初始化命令显式创建） @@ -38,7 +39,7 @@ ## 访问方式 - 前端：`http://localhost:8888` - 后端 API：同域 `/api/v1/*` -- `nginx` 负责托管前端静态资源，并将 `/api` 与 `/health` 转发到 `backend` +- `nginx` 负责托管前端静态资源，并将 `/api`、`/health` 与 `/readyz` 转发到 `backend` ## macOS 桌面端开发 - 桌面端约束集中在 `AGENTS.md`：Tauri 在线客户端，不内嵌后端、不保存本地业务权威数据、不做离线同步；在线辅助缓存必须遵守 `docs/desktop-local-cache-plan.md`。 @@ -54,6 +55,7 @@ - 分支维护中文 SOP：`docs/guides/branch-maintenance-sop-zh.md` - 分支环境安装配置：`docs/guides/branch-environment-installation.md` - 发布检查清单：`docs/guides/release-checklist.md` +- 系统监测简易运维：`docs/guides/system-monitoring-operations.md` ## 本地配置 - 本地编辑器配置（如 `.vscode/`）不纳入版本库。 diff --git a/backend/alembic/versions/20260709_02_add_permission_access_context.py b/backend/alembic/versions/20260709_02_add_permission_access_context.py new file mode 100644 index 00000000..4f1ff2ce --- /dev/null +++ b/backend/alembic/versions/20260709_02_add_permission_access_context.py @@ -0,0 +1,42 @@ +"""add access context to permission access logs + +Revision ID: 20260709_02 +Revises: 20260709_01 +Create Date: 2026-07-09 15:45:00.000000 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + + +revision: str = "20260709_02" +down_revision: Union[str, None] = "20260709_01" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("permission_access_logs", sa.Column("user_agent", sa.String(500), nullable=True)) + op.add_column("permission_access_logs", sa.Column("client_type", sa.String(16), nullable=True)) + op.add_column("permission_access_logs", sa.Column("client_version", sa.String(32), nullable=True)) + op.add_column("permission_access_logs", sa.Column("client_platform", sa.String(16), nullable=True)) + op.add_column("permission_access_logs", sa.Column("build_channel", sa.String(16), nullable=True)) + op.add_column("permission_access_logs", sa.Column("build_commit", sa.String(64), nullable=True)) + op.create_index("ix_perm_log_ip_created", "permission_access_logs", ["ip_address", "created_at"]) + op.create_index("ix_perm_log_client_source_created", "permission_access_logs", ["client_type", "created_at"]) + + +def downgrade() -> None: + op.drop_index("ix_perm_log_client_source_created", table_name="permission_access_logs") + op.drop_index("ix_perm_log_ip_created", table_name="permission_access_logs") + for column in ( + "build_commit", + "build_channel", + "client_platform", + "client_version", + "client_type", + "user_agent", + ): + op.drop_column("permission_access_logs", column) diff --git a/backend/alembic/versions/20260709_03_add_permission_access_request_headers.py b/backend/alembic/versions/20260709_03_add_permission_access_request_headers.py new file mode 100644 index 00000000..f97f4af0 --- /dev/null +++ b/backend/alembic/versions/20260709_03_add_permission_access_request_headers.py @@ -0,0 +1,35 @@ +"""add request headers to permission access logs + +Revision ID: 20260709_03 +Revises: 20260709_02 +Create Date: 2026-07-09 16:58:00.000000 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + + +revision: str = "20260709_03" +down_revision: Union[str, None] = "20260709_02" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +REQUEST_HEADERS_TYPE = sa.JSON().with_variant( + postgresql.JSONB(astext_type=sa.Text()), + "postgresql", +) + + +def upgrade() -> None: + op.add_column( + "permission_access_logs", + sa.Column("request_headers", REQUEST_HEADERS_TYPE, nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("permission_access_logs", "request_headers") diff --git a/backend/alembic/versions/20260709_04_add_security_access_request_headers.py b/backend/alembic/versions/20260709_04_add_security_access_request_headers.py new file mode 100644 index 00000000..61acc353 --- /dev/null +++ b/backend/alembic/versions/20260709_04_add_security_access_request_headers.py @@ -0,0 +1,35 @@ +"""add request headers to security access logs + +Revision ID: 20260709_04 +Revises: 20260709_03 +Create Date: 2026-07-09 17:30:00.000000 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + + +revision: str = "20260709_04" +down_revision: Union[str, None] = "20260709_03" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +REQUEST_HEADERS_TYPE = sa.JSON().with_variant( + postgresql.JSONB(astext_type=sa.Text()), + "postgresql", +) + + +def upgrade() -> None: + op.add_column( + "security_access_logs", + sa.Column("request_headers", REQUEST_HEADERS_TYPE, nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("security_access_logs", "request_headers") diff --git a/backend/alembic/versions/20260709_05_add_request_snapshots_to_access_logs.py b/backend/alembic/versions/20260709_05_add_request_snapshots_to_access_logs.py new file mode 100644 index 00000000..73524f11 --- /dev/null +++ b/backend/alembic/versions/20260709_05_add_request_snapshots_to_access_logs.py @@ -0,0 +1,40 @@ +"""add request snapshots to access logs + +Revision ID: 20260709_05 +Revises: 20260709_04 +Create Date: 2026-07-09 18:00:00.000000 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + + +revision: str = "20260709_05" +down_revision: Union[str, None] = "20260709_04" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +REQUEST_SNAPSHOT_TYPE = sa.JSON().with_variant( + postgresql.JSONB(astext_type=sa.Text()), + "postgresql", +) + + +def upgrade() -> None: + op.add_column( + "permission_access_logs", + sa.Column("request_snapshot", REQUEST_SNAPSHOT_TYPE, nullable=True), + ) + op.add_column( + "security_access_logs", + sa.Column("request_snapshot", REQUEST_SNAPSHOT_TYPE, nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("security_access_logs", "request_snapshot") + op.drop_column("permission_access_logs", "request_snapshot") diff --git a/backend/alembic/versions/20260710_01_add_monitoring_event_identity.py b/backend/alembic/versions/20260710_01_add_monitoring_event_identity.py new file mode 100644 index 00000000..14d9ee19 --- /dev/null +++ b/backend/alembic/versions/20260710_01_add_monitoring_event_identity.py @@ -0,0 +1,67 @@ +"""add monitoring event identity and persisted security classification + +Revision ID: 20260710_01 +Revises: 20260709_05 +Create Date: 2026-07-10 09:30:00.000000 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + + +revision: str = "20260710_01" +down_revision: Union[str, None] = "20260709_05" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("permission_access_logs", sa.Column("request_id", sa.String(36), nullable=True)) + op.add_column("security_access_logs", sa.Column("request_id", sa.String(36), nullable=True)) + op.add_column("security_access_logs", sa.Column("category", sa.String(30), nullable=True)) + op.add_column("security_access_logs", sa.Column("severity", sa.String(16), nullable=True)) + + op.create_index("ix_perm_log_request_id", "permission_access_logs", ["request_id"]) + op.create_index("ix_security_log_request_id", "security_access_logs", ["request_id"]) + op.create_index("ix_security_log_category_created", "security_access_logs", ["category", "created_at"]) + op.create_index("ix_security_log_severity_created", "security_access_logs", ["severity", "created_at"]) + + op.execute( + """ + UPDATE security_access_logs + SET category = CASE + WHEN lower(path) LIKE '%/.env%' OR lower(path) LIKE '%.git%' + OR lower(path) LIKE '%backup%' OR lower(path) LIKE '%config.php%' + OR lower(path) LIKE '%wp-config%' OR lower(path) LIKE '%database.yml%' + THEN 'PROBE' + WHEN status_code >= 500 THEN 'SERVER_ERROR' + WHEN auth_status = 'INVALID_TOKEN' THEN 'INVALID_TOKEN' + WHEN auth_status = 'ANONYMOUS' AND status_code IN (401, 403) THEN 'ANONYMOUS_API' + WHEN status_code = 404 THEN 'NOT_FOUND_NOISE' + ELSE 'OTHER' + END, + severity = CASE + WHEN lower(path) LIKE '%/.env%' OR lower(path) LIKE '%.git%' + OR lower(path) LIKE '%backup%' OR lower(path) LIKE '%config.php%' + OR lower(path) LIKE '%wp-config%' OR lower(path) LIKE '%database.yml%' + THEN 'CRITICAL' + WHEN status_code >= 500 THEN 'HIGH' + WHEN auth_status = 'INVALID_TOKEN' THEN 'MEDIUM' + WHEN auth_status = 'ANONYMOUS' AND status_code IN (401, 403) THEN 'MEDIUM' + ELSE 'LOW' + END + """ + ) + + +def downgrade() -> None: + op.drop_index("ix_security_log_severity_created", table_name="security_access_logs") + op.drop_index("ix_security_log_category_created", table_name="security_access_logs") + op.drop_index("ix_security_log_request_id", table_name="security_access_logs") + op.drop_index("ix_perm_log_request_id", table_name="permission_access_logs") + op.drop_column("security_access_logs", "severity") + op.drop_column("security_access_logs", "category") + op.drop_column("security_access_logs", "request_id") + op.drop_column("permission_access_logs", "request_id") diff --git a/backend/alembic/versions/20260710_02_add_source_location_snapshots.py b/backend/alembic/versions/20260710_02_add_source_location_snapshots.py new file mode 100644 index 00000000..bec7fd70 --- /dev/null +++ b/backend/alembic/versions/20260710_02_add_source_location_snapshots.py @@ -0,0 +1,93 @@ +"""add source location hourly snapshots + +Revision ID: 20260710_02 +Revises: 20260710_01 +Create Date: 2026-07-10 15:30:00.000000 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + + +revision: str = "20260710_02" +down_revision: Union[str, None] = "20260710_01" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "source_location_snapshots", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("bucket_time", sa.DateTime(timezone=True), nullable=False), + sa.Column("ip_hash", sa.String(length=64), nullable=False), + sa.Column("user_hash", sa.String(length=64), nullable=False, server_default=""), + sa.Column("country", sa.String(length=100), nullable=False, server_default=""), + sa.Column("country_code", sa.String(length=16), nullable=False, server_default=""), + sa.Column("province", sa.String(length=100), nullable=False, server_default=""), + sa.Column("region_code", sa.String(length=24), nullable=False, server_default=""), + sa.Column("city", sa.String(length=100), nullable=False, server_default=""), + sa.Column("isp", sa.String(length=160), nullable=False, server_default=""), + sa.Column("location", sa.String(length=320), nullable=False, server_default=""), + sa.Column("longitude", sa.Float(), nullable=True), + sa.Column("latitude", sa.Float(), nullable=True), + sa.Column("accuracy_level", sa.String(length=16), nullable=False, server_default="unknown"), + sa.Column("allowed_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("denied_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("security_event_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("high_risk_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("auth_failure_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("bucket_time", "ip_hash", "user_hash", name="uq_source_location_bucket_identity"), + ) + op.create_index("ix_source_location_bucket", "source_location_snapshots", ["bucket_time"]) + op.create_index( + "ix_source_location_country_bucket", + "source_location_snapshots", + ["country_code", "bucket_time"], + ) + op.create_index( + "ix_source_location_risk_bucket", + "source_location_snapshots", + ["high_risk_count", "bucket_time"], + ) + + op.execute( + """ + UPDATE security_access_logs + SET category = CASE + WHEN lower(path) LIKE '%/.env%' OR lower(path) LIKE '%.git%' + OR lower(path) LIKE '%backup%' OR lower(path) LIKE '%config.php%' + OR lower(path) LIKE '%wp-config%' OR lower(path) LIKE '%database.yml%' + THEN 'PROBE' + WHEN status_code >= 500 THEN 'SERVER_ERROR' + WHEN auth_status = 'INVALID_TOKEN' THEN 'INVALID_TOKEN' + WHEN auth_status = 'ANONYMOUS' AND status_code IN (401, 403) THEN 'ANONYMOUS_API' + WHEN status_code = 404 THEN 'NOT_FOUND_NOISE' + ELSE 'OTHER' + END, + severity = CASE + WHEN lower(path) LIKE '%/.env%' OR lower(path) LIKE '%.git%' + OR lower(path) LIKE '%backup%' OR lower(path) LIKE '%config.php%' + OR lower(path) LIKE '%wp-config%' OR lower(path) LIKE '%database.yml%' + THEN 'CRITICAL' + WHEN status_code >= 500 THEN 'HIGH' + WHEN auth_status = 'INVALID_TOKEN' THEN 'MEDIUM' + WHEN auth_status = 'ANONYMOUS' AND status_code IN (401, 403) THEN 'MEDIUM' + ELSE 'LOW' + END + WHERE category = 'ABNORMAL_IP' + """ + ) + + +def downgrade() -> None: + op.drop_index("ix_source_location_risk_bucket", table_name="source_location_snapshots") + op.drop_index("ix_source_location_country_bucket", table_name="source_location_snapshots") + op.drop_index("ix_source_location_bucket", table_name="source_location_snapshots") + op.drop_table("source_location_snapshots") diff --git a/backend/alembic/versions/20260710_03_remove_private_source_snapshot_coordinates.py b/backend/alembic/versions/20260710_03_remove_private_source_snapshot_coordinates.py new file mode 100644 index 00000000..72c2f8fc --- /dev/null +++ b/backend/alembic/versions/20260710_03_remove_private_source_snapshot_coordinates.py @@ -0,0 +1,30 @@ +"""Remove deployment-specific coordinates from private source snapshots. + +Revision ID: 20260710_03 +Revises: 20260710_02 +""" + +from alembic import op + + +revision = "20260710_03" +down_revision = "20260710_02" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute( + """ + UPDATE source_location_snapshots + SET longitude = NULL, + latitude = NULL + WHERE accuracy_level = 'private' + """ + ) + + +def downgrade() -> None: + # The previous coordinates were a hardcoded deployment assumption and + # cannot be restored without reintroducing incorrect data. + pass diff --git a/backend/alembic/versions/20260710_04_add_user_login_sessions.py b/backend/alembic/versions/20260710_04_add_user_login_sessions.py new file mode 100644 index 00000000..0dc819ff --- /dev/null +++ b/backend/alembic/versions/20260710_04_add_user_login_sessions.py @@ -0,0 +1,40 @@ +"""Add user login session activity records. + +Revision ID: 20260710_04 +Revises: 20260710_03 +""" + +import sqlalchemy as sa +from alembic import op + + +revision = "20260710_04" +down_revision = "20260710_03" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "user_login_sessions", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("client_type", sa.String(length=16), nullable=False, server_default="web"), + sa.Column("client_platform", sa.String(length=32), nullable=True), + sa.Column("client_version", sa.String(length=64), nullable=True), + sa.Column("client_source", sa.String(length=32), nullable=True), + sa.Column("login_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("CURRENT_TIMESTAMP")), + sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("CURRENT_TIMESTAMP")), + sa.Column("ended_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("end_reason", sa.String(length=32), nullable=True), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_user_login_sessions_user_login", "user_login_sessions", ["user_id", "login_at"]) + op.create_index("ix_user_login_sessions_last_seen", "user_login_sessions", ["last_seen_at"]) + + +def downgrade() -> None: + op.drop_index("ix_user_login_sessions_last_seen", table_name="user_login_sessions") + op.drop_index("ix_user_login_sessions_user_login", table_name="user_login_sessions") + op.drop_table("user_login_sessions") diff --git a/backend/app/api/v1/auth.py b/backend/app/api/v1/auth.py index 9cd0be45..b6824db5 100644 --- a/backend/app/api/v1/auth.py +++ b/backend/app/api/v1/auth.py @@ -9,7 +9,8 @@ import uuid from app.core.config import settings from app.core.login_crypto import create_login_challenge, decrypt_login_payload, get_public_key_pem -from app.core.security import create_access_token, decode_token_allow_expired, oauth2_scheme, verify_password +from app.core.request_context import resolve_ctms_client_type +from app.core.security import create_access_token, decode_token, decode_token_allow_expired, oauth2_scheme, verify_password from app.core.deps import get_current_user, get_db_session from app.crud import user as user_crud from app.models.user import UserStatus @@ -26,6 +27,12 @@ from app.schemas.email_settings import ( ) from app.schemas.user import Token, UserRead, UserRegisterRequest, UserSelfUpdate, UserUpdate from app.services import email_service +from app.services.user_login_sessions import ( + create_login_session, + end_login_session, + session_id_from_payload, + touch_login_session, +) from fastapi.responses import FileResponse @@ -95,7 +102,7 @@ def get_session_policy_for_client_type(client_type: str) -> SessionPolicy: def get_request_session_client_type(request: Request) -> str: - return normalize_session_client_type(request.headers.get("x-ctms-client-type")) + return normalize_session_client_type(resolve_ctms_client_type(request.headers)) def policy_expires_at(issued_at: datetime, session_start: datetime, policy: SessionPolicy) -> datetime: @@ -104,8 +111,9 @@ def policy_expires_at(issued_at: datetime, session_start: datetime, policy: Sess return min(access_expires_at, session_expires_at) -def issue_user_token(db_user, request: Request) -> Token: +async def issue_user_token(db_user, request: Request, db: AsyncSession) -> Token: session_start = datetime.now(timezone.utc) + session_id = uuid.uuid4() client_type = get_request_session_client_type(request) policy = get_session_policy_for_client_type(client_type) access_token = create_access_token( @@ -115,6 +123,14 @@ def issue_user_token(db_user, request: Request) -> Token: max_age_seconds=policy.absolute_max_seconds, issued_at=session_start, client_type=client_type, + session_id=str(session_id), + ) + await create_login_session( + db, + session_id=session_id, + user_id=db_user.id, + request=request, + login_at=session_start, ) return Token(access_token=access_token, token_type="bearer") @@ -284,7 +300,7 @@ async def login_for_access_token( db_user = await authenticate_encrypted_password(payload, db) ensure_user_active(db_user) - return issue_user_token(db_user, request) + return await issue_user_token(db_user, request, db) @router.post("/dev-login", response_model=Token) @@ -295,7 +311,7 @@ async def dev_login_for_access_token( raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found") db_user = await authenticate_plain_password(payload, db) ensure_user_active(db_user) - return issue_user_token(db_user, request) + return await issue_user_token(db_user, request, db) @router.get("/me", response_model=UserRead) @@ -339,11 +355,47 @@ async def extend_access_token( max_age_seconds=policy.absolute_max_seconds, issued_at=issued_at, client_type=normalize_session_client_type(payload.get("client_type")), + session_id=str(session_id_from_payload(payload)), ) expires_at = policy_expires_at(issued_at, session_start, policy) return ExtendResponse(accessToken=new_token, expiresAt=expires_at) +@router.post("/session/heartbeat") +async def heartbeat_login_session( + request: Request, + token: str = Depends(oauth2_scheme), + current_user=Depends(get_current_user), + db: AsyncSession = Depends(get_db_session), +) -> dict: + session = await touch_login_session( + db, + user_id=current_user.id, + payload=decode_token(token), + request=request, + ) + if session is None: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="登录会话已结束") + return { + "status": "online", + "last_seen_at": session.last_seen_at.isoformat(), + } + + +@router.post("/session/logout", status_code=status.HTTP_204_NO_CONTENT) +async def logout_login_session( + token: str = Depends(oauth2_scheme), + current_user=Depends(get_current_user), + db: AsyncSession = Depends(get_db_session), +) -> Response: + await end_login_session( + db, + user_id=current_user.id, + payload=decode_token(token), + ) + return Response(status_code=status.HTTP_204_NO_CONTENT) + + @router.patch("/me", response_model=UserRead) async def update_me( payload: UserSelfUpdate, diff --git a/backend/app/api/v1/permission_monitoring.py b/backend/app/api/v1/permission_monitoring.py index 6e78242b..8140dce0 100644 --- a/backend/app/api/v1/permission_monitoring.py +++ b/backend/app/api/v1/permission_monitoring.py @@ -6,15 +6,18 @@ from __future__ import annotations import uuid +import json +import time from collections import defaultdict from datetime import datetime, timedelta, timezone from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query, status -from sqlalchemy import func, select, desc +from sqlalchemy import String, case, cast, desc, func, literal, or_, select, union_all from sqlalchemy.ext.asyncio import AsyncSession -from app.core.deps import get_current_user, get_db_session, is_system_admin, list_active_pm_study_ids +from app.core.config import settings +from app.core.deps import get_current_user, get_db_session, is_system_admin from app.core.permission_monitor import ( CACHE_HIT_RATE_HEALTH_MIN_ACCESSES, CACHE_METRICS_WINDOW_SECONDS, @@ -24,7 +27,14 @@ from app.models.permission_access_log import PermissionAccessLog from app.models.permission_metric_snapshot import PermissionMetricSnapshot from app.models.security_access_log import SecurityAccessLog from app.models.user import User +from app.services.geo_location_metadata import GeoLocationMetadata, resolve_geo_location_metadata from app.services.ip_location import IpLocation, resolve_ip_location +from app.services.monitoring_retention import get_monitoring_retention_status +from app.services.permission_log_writer import get_log_writer +from app.services.security_access_log_writer import get_security_log_writer +from app.services.security_events import classify_security_event +from app.services.monitoring_server_location import resolve_monitoring_server_location +from app.services.source_location_aggregator import get_source_location_timeline router = APIRouter(prefix="/permission-monitoring", tags=["permission-monitoring"]) @@ -43,11 +53,6 @@ class MonitoringScope: async def resolve_monitoring_scope(db: AsyncSession, current_user) -> MonitoringScope: if is_system_admin(current_user): return MonitoringScope(is_admin=True, study_ids=set()) - user_id = getattr(current_user, "id", None) - if user_id: - study_ids = await list_active_pm_study_ids(db, user_id) - if study_ids: - return MonitoringScope(is_admin=False, study_ids=study_ids) raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") @@ -57,6 +62,217 @@ def _apply_monitoring_scope_to_log_query(query, scope: MonitoringScope): return query.where(PermissionAccessLog.study_id.in_(scope.study_ids)) +KNOWN_PERMISSION_CLIENT_TYPES = ("web", "desktop") + + +def _permission_client_type_condition(client_type: str): + normalized = client_type.strip().lower() + if normalized == "unknown": + return or_( + PermissionAccessLog.client_type.is_(None), + PermissionAccessLog.client_type == "", + ~PermissionAccessLog.client_type.in_(KNOWN_PERMISSION_CLIENT_TYPES), + ) + return PermissionAccessLog.client_type == normalized + + +def _security_client_type_condition(client_type: str): + normalized = client_type.strip().lower() + if normalized == "unknown": + return or_( + SecurityAccessLog.client_type.is_(None), + SecurityAccessLog.client_type == "", + ~SecurityAccessLog.client_type.in_(KNOWN_PERMISSION_CLIENT_TYPES), + ) + return SecurityAccessLog.client_type == normalized + + +def _coerce_request_headers(value) -> dict[str, str] | None: + if not value: + return None + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError: + return None + if not isinstance(value, dict): + return None + headers = {str(key): str(header_value) for key, header_value in value.items() if header_value is not None} + return headers or None + + +def _coerce_request_snapshot(value) -> dict | None: + if not value: + return None + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError: + return None + return value if isinstance(value, dict) else None + + +async def _resolve_permission_location_keyword_ips( + db: AsyncSession, + conditions: list, + keyword: str, +) -> list[str]: + token = keyword.strip().lower() + if not token: + return [] + ip_query = ( + select(PermissionAccessLog.ip_address) + .where(PermissionAccessLog.ip_address.is_not(None)) + .distinct() + ) + for condition in conditions: + ip_query = ip_query.where(condition) + result = await db.execute(ip_query) + matched_ips: list[str] = [] + for ip_address in result.scalars().all(): + ip_location = resolve_ip_location(ip_address) + haystack = " ".join( + [ + ip_location.location or "", + ip_location.country or "", + ip_location.province or "", + ip_location.city or "", + ip_location.isp or "", + ] + ).lower() + if token in haystack: + matched_ips.append(ip_address) + return matched_ips + + +async def _resolve_security_location_keyword_ips( + db: AsyncSession, + conditions: list, + keyword: str, +) -> list[str]: + token = keyword.strip().lower() + if not token: + return [] + ip_query = ( + select(SecurityAccessLog.client_ip) + .where(SecurityAccessLog.client_ip.is_not(None)) + .distinct() + ) + for condition in conditions: + ip_query = ip_query.where(condition) + result = await db.execute(ip_query) + matched_ips: list[str] = [] + for ip_address in result.scalars().all(): + ip_location = resolve_ip_location(ip_address) + haystack = " ".join( + [ + ip_location.location or "", + ip_location.country or "", + ip_location.province or "", + ip_location.city or "", + ip_location.isp or "", + ] + ).lower() + if token in haystack: + matched_ips.append(ip_address) + return matched_ips + + +def _permission_access_item(log: PermissionAccessLog, full_name: str | None, email: str | None) -> dict: + ip_location = resolve_ip_location(log.ip_address) + return { + "id": str(log.id), + "event_type": "permission", + "study_id": str(log.study_id) if log.study_id else "", + "user_id": str(log.user_id) if log.user_id else "", + "user_name": full_name or "未知用户", + "user_email": email, + "endpoint_key": log.endpoint_key, + "role": log.role, + "allowed": log.allowed, + "elapsed_ms": round(log.elapsed_ms, 2), + "ip_address": log.ip_address, + "ip_location": ip_location.location, + "ip_country": ip_location.country, + "ip_province": ip_location.province, + "ip_city": ip_location.city, + "ip_isp": ip_location.isp, + "user_agent": log.user_agent, + "client_type": log.client_type, + "client_version": log.client_version, + "client_platform": log.client_platform, + "build_channel": log.build_channel, + "build_commit": log.build_commit, + "request_headers": _coerce_request_headers(log.request_headers), + "request_snapshot": _coerce_request_snapshot(getattr(log, "request_snapshot", None)), + "request_id": getattr(log, "request_id", None), + "method": None, + "path": None, + "status_code": None, + "auth_status": "AUTHENTICATED", + "account_label": full_name or email, + "category": None, + "severity": None, + "created_at": log.created_at.isoformat(), + } + + +def _security_access_item(log: SecurityAccessLog, user_names: dict[str, str] | None = None) -> dict: + ip_location = resolve_ip_location(log.client_ip) + classification = _classify_security_access_log(log, ip_location) + account_label = _security_account_label(log.auth_status, log.user_identifier, user_names) + return { + "id": f"security:{log.id}", + "event_type": "security", + "study_id": "", + "user_id": log.user_identifier or "", + "user_name": account_label, + "user_email": None, + "endpoint_key": f"{log.method} {log.path}", + "role": classification["category"], + "allowed": log.status_code < 400, + "elapsed_ms": round(log.elapsed_ms, 2), + "ip_address": log.client_ip, + "ip_location": ip_location.location, + "ip_country": ip_location.country, + "ip_province": ip_location.province, + "ip_city": ip_location.city, + "ip_isp": ip_location.isp, + "user_agent": log.user_agent, + "client_type": log.client_type, + "client_version": log.client_version, + "client_platform": log.client_platform, + "build_channel": log.build_channel, + "build_commit": log.build_commit, + "request_headers": _coerce_request_headers(getattr(log, "request_headers", None)), + "request_snapshot": _coerce_request_snapshot(getattr(log, "request_snapshot", None)), + "request_id": getattr(log, "request_id", None), + "method": log.method, + "path": log.path, + "status_code": log.status_code, + "auth_status": log.auth_status, + "account_label": account_label, + "category": classification["category"], + "severity": classification["severity"], + "created_at": log.created_at.isoformat(), + } + + +async def _security_user_names(db: AsyncSession, logs: list[SecurityAccessLog]) -> dict[str, str]: + user_ids: list[uuid.UUID] = [] + for log in logs: + if log.auth_status != "AUTHENTICATED" or not log.user_identifier: + continue + try: + user_ids.append(uuid.UUID(log.user_identifier)) + except ValueError: + continue + if not user_ids: + return {} + users_result = await db.execute(select(User.id, User.full_name).where(User.id.in_(user_ids))) + return {str(row.id): row.full_name for row in users_result.all()} + + # ═══════════════════════════════════════════ # 原有端点（保持兼容） # ═══════════════════════════════════════════ @@ -173,6 +389,9 @@ async def permission_system_health( if not scope.is_admin and not scope.study_ids: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") start_time = datetime.now(timezone.utc) - timedelta(hours=1) + database_started_at = time.perf_counter() + await db.execute(select(literal(1))) + database_latency_ms = round((time.perf_counter() - database_started_at) * 1000, 2) health_query = select( func.count().label("total"), func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied"), @@ -195,18 +414,66 @@ async def permission_system_health( if alert["timestamp"] >= now_timestamp - 3600 ] + permission_writer = get_log_writer() + security_writer = get_security_log_writer() + permission_writer_stats = permission_writer.stats() if permission_writer else None + security_writer_stats = security_writer.stats() if security_writer else None + retention_stats = get_monitoring_retention_status() + + def writer_component(stats: dict | None) -> dict: + if stats is None: + return {"status": "unknown", "reason": "writer_not_initialized"} + queue_capacity = max(1, int(stats.get("queue_capacity") or 1)) + queue_usage = round(int(stats.get("queue_size") or 0) / queue_capacity * 100, 2) + if not stats.get("running"): + component_status = "unhealthy" + elif queue_usage >= 80 or stats.get("failed_batches") or stats.get("dropped_entries"): + component_status = "degraded" + else: + component_status = "healthy" + return {**stats, "status": component_status, "queue_usage_percent": queue_usage} + + permission_writer_component = writer_component(permission_writer_stats) + security_writer_component = writer_component(security_writer_stats) + if retention_stats["running"]: + retention_has_unrecovered_error = bool( + retention_stats["last_error_at"] + and ( + not retention_stats["last_success_at"] + or retention_stats["last_error_at"] > retention_stats["last_success_at"] + ) + ) + retention_status = "degraded" if retention_has_unrecovered_error else "healthy" + elif retention_stats["last_run_started_at"]: + retention_status = "unhealthy" + else: + retention_status = "unknown" + retention_component = {**retention_stats, "status": retention_status} + + storage_result = await db.execute( + select( + select(func.count()).select_from(PermissionAccessLog).scalar_subquery().label("permission_count"), + select(func.min(PermissionAccessLog.created_at)).scalar_subquery().label("permission_oldest"), + select(func.count()).select_from(SecurityAccessLog).scalar_subquery().label("security_count"), + select(func.min(SecurityAccessLog.created_at)).scalar_subquery().label("security_oldest"), + select(func.count()).select_from(PermissionMetricSnapshot).scalar_subquery().label("metric_count"), + ) + ) + storage_row = storage_result.one() + deductions: dict[str, int] = { "performance": 0, "deny_rate": 0, "cache": 0, "alerts": 0, + "runtime": 0, } issues: list[str] = [] if avg_ms > 200: - deductions["performance"] = 40 - elif avg_ms > 50: deductions["performance"] = 25 + elif avg_ms > 50: + deductions["performance"] = 20 elif avg_ms > 10: deductions["performance"] = 10 if deductions["performance"]: @@ -235,12 +502,44 @@ async def permission_system_health( error_alert_count = sum(alert["level"] == "error" for alert in recent_alerts) warning_alert_count = sum(alert["level"] == "warning" for alert in recent_alerts) if error_alert_count: - deductions["alerts"] = 15 + deductions["alerts"] = 10 issues.append(f"近 1 小时发生 {error_alert_count} 条错误告警") elif warning_alert_count: deductions["alerts"] = 5 issues.append(f"近 1 小时发生 {warning_alert_count} 条警告") + runtime_components = { + "database": {"status": "healthy", "latency_ms": database_latency_ms}, + "permission_log_writer": permission_writer_component, + "security_log_writer": security_writer_component, + "retention": retention_component, + } + component_labels = { + "database": "数据库", + "permission_log_writer": "权限日志写入器", + "security_log_writer": "安全日志写入器", + "retention": "留存清理任务", + } + unhealthy_components = [ + component_labels[name] + for name, component in runtime_components.items() + if component["status"] == "unhealthy" + ] + degraded_components = [ + component_labels[name] + for name, component in runtime_components.items() + if component["status"] == "degraded" + ] + if unhealthy_components: + deductions["runtime"] = 20 + issues.append(f"运行组件异常：{', '.join(unhealthy_components)}") + elif degraded_components: + deductions["runtime"] = 10 + issues.append(f"运行组件降级：{', '.join(degraded_components)}") + elif database_latency_ms > 500: + deductions["runtime"] = 5 + issues.append(f"数据库健康检查延迟偏高（{database_latency_ms:.2f}ms）") + health_score = max(0, 100 - sum(deductions.values())) sample_sufficient = total >= 10 @@ -250,7 +549,7 @@ async def permission_system_health( "issues": issues, "sample_sufficient": sample_sufficient, "score_breakdown": { - "performance": {"weight": 40, "deduction": deductions["performance"]}, + "performance": {"weight": 25, "deduction": deductions["performance"]}, "deny_rate": {"weight": 20, "deduction": deductions["deny_rate"]}, "cache": { "weight": 25, @@ -259,7 +558,8 @@ async def permission_system_health( "scope": "sliding_window", "window_seconds": CACHE_METRICS_WINDOW_SECONDS, }, - "alerts": {"weight": 15, "deduction": deductions["alerts"]}, + "alerts": {"weight": 10, "deduction": deductions["alerts"]}, + "runtime": {"weight": 20, "deduction": deductions["runtime"]}, }, "last_hour": { "total_checks": total, @@ -270,6 +570,18 @@ async def permission_system_health( "warning_alerts": warning_alert_count, }, "cache_stats": monitor.get_cache_stats(), + "components": runtime_components, + "storage": { + "permission_access_logs": storage_row.permission_count or 0, + "security_access_logs": storage_row.security_count or 0, + "permission_metric_snapshots": storage_row.metric_count or 0, + "oldest_permission_access_log_at": ( + storage_row.permission_oldest.isoformat() if storage_row.permission_oldest else None + ), + "oldest_security_access_log_at": ( + storage_row.security_oldest.isoformat() if storage_row.security_oldest else None + ), + }, } @@ -288,6 +600,9 @@ async def get_access_logs( allowed: Optional[bool] = Query(None), start_time: Optional[datetime] = Query(None), end_time: Optional[datetime] = Query(None), + client_ip: Optional[str] = Query(None), + client_type: Optional[str] = Query(None), + keyword: Optional[str] = Query(None), page: int = Query(1, ge=1), page_size: int = Query(50, ge=1, le=200), ) -> dict: @@ -297,6 +612,11 @@ async def get_access_logs( raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") if study_id and not scope.can_access_study(study_id): raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") + client_ip = client_ip if isinstance(client_ip, str) else None + client_type = client_type if isinstance(client_type, str) else None + keyword = keyword if isinstance(keyword, str) else None + start_time = start_time if isinstance(start_time, datetime) else None + end_time = end_time if isinstance(end_time, datetime) else None conditions = [] if study_id: conditions.append(PermissionAccessLog.study_id == study_id) @@ -314,19 +634,54 @@ async def get_access_logs( conditions.append(PermissionAccessLog.created_at >= start_time) if end_time: conditions.append(PermissionAccessLog.created_at <= end_time) + if client_ip and client_ip.strip(): + conditions.append(PermissionAccessLog.ip_address.ilike(f"%{client_ip.strip()}%")) + if client_type and client_type.strip(): + conditions.append(_permission_client_type_condition(client_type)) + if keyword and keyword.strip(): + token = keyword.strip() + like_token = f"%{token}%" + location_ips = await _resolve_permission_location_keyword_ips(db, conditions, token) + keyword_conditions = [ + PermissionAccessLog.endpoint_key.ilike(like_token), + PermissionAccessLog.role.ilike(like_token), + PermissionAccessLog.ip_address.ilike(like_token), + PermissionAccessLog.client_type.ilike(like_token), + PermissionAccessLog.client_version.ilike(like_token), + PermissionAccessLog.client_platform.ilike(like_token), + PermissionAccessLog.user_agent.ilike(like_token), + cast(PermissionAccessLog.request_headers, String).ilike(like_token), + cast(PermissionAccessLog.request_snapshot, String).ilike(like_token), + User.full_name.ilike(like_token), + User.email.ilike(like_token), + cast(PermissionAccessLog.user_id, String).ilike(like_token), + ] + if location_ips: + keyword_conditions.append(PermissionAccessLog.ip_address.in_(location_ips)) + conditions.append(or_(*keyword_conditions)) - query = ( - select(PermissionAccessLog, User.full_name) + include_security_logs = scope.is_admin and study_id is None and role is None and endpoint_key is None and allowed is not True + event_conditions = list(conditions) + if include_security_logs: + matching_security_event = select(SecurityAccessLog.id).where( + PermissionAccessLog.request_id.is_not(None), + SecurityAccessLog.request_id == PermissionAccessLog.request_id, + SecurityAccessLog.status_code >= 400, + ).exists() + event_conditions.append(~matching_security_event) + + permission_event_query = ( + select( + literal("permission").label("event_type"), + cast(PermissionAccessLog.id, String).label("event_id"), + PermissionAccessLog.created_at.label("created_at"), + cast(PermissionAccessLog.user_id, String).label("user_id"), + PermissionAccessLog.ip_address.label("ip_address"), + PermissionAccessLog.allowed.label("allowed"), + PermissionAccessLog.elapsed_ms.label("elapsed_ms"), + ) .outerjoin(User, PermissionAccessLog.user_id == User.id) ) - count_query = select(func.count()).select_from(PermissionAccessLog) - summary_query = select( - func.count().label("total_count"), - func.count(func.distinct(PermissionAccessLog.user_id)).label("unique_user_count"), - func.count(func.distinct(PermissionAccessLog.ip_address)).label("unique_ip_count"), - func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied_count"), - func.coalesce(func.avg(PermissionAccessLog.elapsed_ms), 0).label("avg_elapsed_ms"), - ).select_from(PermissionAccessLog) user_stats_query = ( select( PermissionAccessLog.ip_address.label("sample_ip_address"), @@ -335,20 +690,16 @@ async def get_access_logs( func.count(func.distinct(PermissionAccessLog.ip_address)).label("unique_ip_count"), func.max(PermissionAccessLog.created_at).label("last_seen_at"), ) + .outerjoin(User, PermissionAccessLog.user_id == User.id) .group_by(PermissionAccessLog.ip_address) .order_by(desc("total_count"), desc("last_seen_at")) .limit(10) ) + for condition in event_conditions: + permission_event_query = permission_event_query.where(condition) for condition in conditions: - query = query.where(condition) - count_query = count_query.where(condition) - summary_query = summary_query.where(condition) user_stats_query = user_stats_query.where(condition) - total_result = await db.execute(count_query) - total = total_result.scalar() or 0 - summary_result = await db.execute(summary_query) - summary_row = summary_result.one() user_stats_result = await db.execute(user_stats_query) user_stats_rows = user_stats_result.all() latest_user_by_ip = {} @@ -359,6 +710,7 @@ async def get_access_logs( PermissionAccessLog.ip_address, PermissionAccessLog.user_id, User.full_name, + User.email, PermissionAccessLog.role, PermissionAccessLog.created_at, ) @@ -372,33 +724,120 @@ async def get_access_logs( for row in latest_user_result.all(): latest_user_by_ip.setdefault(row.ip_address, row) - query = query.order_by(desc(PermissionAccessLog.created_at)) - query = query.offset((page - 1) * page_size).limit(page_size) - result = await db.execute(query) - rows = result.all() + event_queries = [permission_event_query] + if include_security_logs: + security_conditions = [SecurityAccessLog.status_code >= 400] + if user_id: + security_conditions.append(SecurityAccessLog.user_identifier == str(user_id)) + if start_time: + security_conditions.append(SecurityAccessLog.created_at >= start_time) + if end_time: + security_conditions.append(SecurityAccessLog.created_at <= end_time) + if client_ip and client_ip.strip(): + security_conditions.append(SecurityAccessLog.client_ip.ilike(f"%{client_ip.strip()}%")) + if client_type and client_type.strip(): + security_conditions.append(_security_client_type_condition(client_type)) + if keyword and keyword.strip(): + token = keyword.strip() + like_token = f"%{token}%" + location_ips = await _resolve_security_location_keyword_ips(db, security_conditions, token) + matching_users_result = await db.execute( + select(cast(User.id, String)).where( + or_(User.full_name.ilike(like_token), User.email.ilike(like_token)) + ) + ) + matching_user_ids = list(matching_users_result.scalars().all()) + keyword_conditions = [ + SecurityAccessLog.method.ilike(like_token), + SecurityAccessLog.path.ilike(like_token), + SecurityAccessLog.client_ip.ilike(like_token), + SecurityAccessLog.client_type.ilike(like_token), + SecurityAccessLog.client_version.ilike(like_token), + SecurityAccessLog.client_platform.ilike(like_token), + SecurityAccessLog.user_agent.ilike(like_token), + SecurityAccessLog.auth_status.ilike(like_token), + SecurityAccessLog.user_identifier.ilike(like_token), + cast(SecurityAccessLog.status_code, String).ilike(like_token), + cast(SecurityAccessLog.request_headers, String).ilike(like_token), + cast(SecurityAccessLog.request_snapshot, String).ilike(like_token), + ] + if location_ips: + keyword_conditions.append(SecurityAccessLog.client_ip.in_(location_ips)) + if matching_user_ids: + keyword_conditions.append(SecurityAccessLog.user_identifier.in_(matching_user_ids)) + security_conditions.append(or_(*keyword_conditions)) - items = [] - for log, full_name in rows: - ip_location = resolve_ip_location(log.ip_address) - items.append( - { - "id": str(log.id), - "study_id": str(log.study_id), - "user_id": str(log.user_id), - "user_name": full_name or "未知用户", - "endpoint_key": log.endpoint_key, - "role": log.role, - "allowed": log.allowed, - "elapsed_ms": round(log.elapsed_ms, 2), - "ip_address": log.ip_address, - "ip_location": ip_location.location, - "ip_country": ip_location.country, - "ip_province": ip_location.province, - "ip_city": ip_location.city, - "ip_isp": ip_location.isp, - "created_at": log.created_at.isoformat(), - } + security_event_query = select( + literal("security").label("event_type"), + cast(SecurityAccessLog.id, String).label("event_id"), + SecurityAccessLog.created_at.label("created_at"), + SecurityAccessLog.user_identifier.label("user_id"), + SecurityAccessLog.client_ip.label("ip_address"), + (SecurityAccessLog.status_code < 400).label("allowed"), + SecurityAccessLog.elapsed_ms.label("elapsed_ms"), ) + for condition in security_conditions: + security_event_query = security_event_query.where(condition) + event_queries.append(security_event_query) + + events = ( + union_all(*event_queries).subquery("monitoring_events") + if len(event_queries) > 1 + else event_queries[0].subquery("monitoring_events") + ) + summary_result = await db.execute( + select( + func.count().label("total_count"), + func.count(func.distinct(events.c.user_id)).label("unique_user_count"), + func.count(func.distinct(events.c.ip_address)).label("unique_ip_count"), + func.count().filter(events.c.allowed.is_(False)).label("denied_count"), + func.coalesce(func.avg(events.c.elapsed_ms), 0).label("avg_elapsed_ms"), + ).select_from(events) + ) + summary_row = summary_result.one() + total = summary_row.total_count or 0 + + page_result = await db.execute( + select(events.c.event_type, events.c.event_id, events.c.created_at) + .order_by(desc(events.c.created_at), desc(events.c.event_id)) + .offset((page - 1) * page_size) + .limit(page_size) + ) + page_events = page_result.all() + permission_ids = [uuid.UUID(row.event_id) for row in page_events if row.event_type == "permission"] + security_ids = [uuid.UUID(row.event_id) for row in page_events if row.event_type == "security"] + + items_by_key: dict[tuple[str, str], dict] = {} + if permission_ids: + permission_result = await db.execute( + select(PermissionAccessLog, User.full_name, User.email) + .outerjoin(User, PermissionAccessLog.user_id == User.id) + .where(PermissionAccessLog.id.in_(permission_ids)) + ) + for log, full_name, email in permission_result.all(): + items_by_key[("permission", str(log.id))] = _permission_access_item(log, full_name, email) + if security_ids: + security_result = await db.execute( + select(SecurityAccessLog).where(SecurityAccessLog.id.in_(security_ids)) + ) + security_logs = list(security_result.scalars().all()) + security_user_names = await _security_user_names(db, security_logs) + for log in security_logs: + items_by_key[("security", str(log.id))] = _security_access_item(log, security_user_names) + + page_items = [ + items_by_key[(row.event_type, row.event_id)] + for row in page_events + if (row.event_type, row.event_id) in items_by_key + ] + summary = { + "total_count": total, + "unique_user_count": summary_row.unique_user_count or 0, + "unique_ip_count": summary_row.unique_ip_count or 0, + "denied_count": summary_row.denied_count or 0, + "avg_elapsed_ms": round(float(summary_row.avg_elapsed_ms), 2), + } + user_stats = [] for stat in user_stats_rows: sample_location = resolve_ip_location(stat.sample_ip_address) @@ -407,6 +846,7 @@ async def get_access_logs( { "user_id": str(latest_user.user_id) if latest_user else "", "user_name": latest_user.full_name if latest_user and latest_user.full_name else "未知用户", + "user_email": latest_user.email if latest_user else None, "role": latest_user.role if latest_user else "", "total_count": stat.total_count, "denied_count": stat.denied_count, @@ -421,15 +861,9 @@ async def get_access_logs( "total": total, "page": page, "page_size": page_size, - "summary": { - "total_count": summary_row.total_count, - "unique_user_count": summary_row.unique_user_count, - "unique_ip_count": summary_row.unique_ip_count, - "denied_count": summary_row.denied_count, - "avg_elapsed_ms": round(float(summary_row.avg_elapsed_ms), 2), - }, + "summary": summary, "user_stats": user_stats, - "items": items, + "items": page_items, } @@ -441,41 +875,28 @@ def _security_account_label(auth_status: str, user_identifier: str | None, user_ return "未知账号" -SENSITIVE_PROBE_MARKERS = ( - "/.env", - ".env", - "/.git", - ".git/config", - "backup", - "config.php", - "wp-config", - "database.yml", -) - - -CHINA_IP_COUNTRY_LABELS = {"中国", "China", "Mainland China", "中国香港", "中国澳门", "中国台湾"} - - -def _is_non_china_ip(ip_location: IpLocation) -> bool: - country = (ip_location.country or "").strip() - return bool(country) and country not in CHINA_IP_COUNTRY_LABELS and not country.startswith("中国") - - def _classify_security_access_log(log: SecurityAccessLog, ip_location: IpLocation | None = None) -> dict[str, str]: - path = (log.path or "").lower() - if any(marker in path for marker in SENSITIVE_PROBE_MARKERS): - return {"category": "PROBE", "severity": "CRITICAL"} - if ip_location and _is_non_china_ip(ip_location): - return {"category": "ABNORMAL_IP", "severity": "HIGH"} - if log.status_code >= 500: - return {"category": "SERVER_ERROR", "severity": "HIGH"} - if log.auth_status == "INVALID_TOKEN": - return {"category": "INVALID_TOKEN", "severity": "MEDIUM"} - if log.auth_status == "ANONYMOUS" and log.status_code in {401, 403}: - return {"category": "ANONYMOUS_API", "severity": "MEDIUM"} - if log.status_code == 404: - return {"category": "NOT_FOUND_NOISE", "severity": "LOW"} - return {"category": "OTHER", "severity": "LOW"} + if getattr(log, "category", None) and getattr(log, "severity", None): + return {"category": log.category, "severity": log.severity} + return classify_security_event( + path=log.path, + status_code=log.status_code, + auth_status=log.auth_status, + ip_location=ip_location, + ) + + +def _security_endpoint_type(path: str) -> str: + normalized_path = (path or "").lower() + if "/auth" in normalized_path or "/login" in normalized_path: + return "认证接口" + if normalized_path.startswith("/api/"): + return "业务 API" + if normalized_path.endswith((".js", ".css", ".png", ".jpg", ".jpeg", ".svg", ".ico", ".map")): + return "静态资源" + if any(marker in normalized_path for marker in (".env", "wp-", "php", "admin", "shell", "config")): + return "探测路径" + return "其他请求" @router.get("/security-logs", status_code=status.HTTP_200_OK) @@ -486,6 +907,13 @@ async def get_security_access_logs( auth_status: Optional[str] = Query(None), client_type: Optional[str] = Query(None), client_version: Optional[str] = Query(None), + client_ip: Optional[str] = Query(None), + category: Optional[str] = Query(None), + severity: Optional[str] = Query(None), + keyword: Optional[str] = Query(None), + events_only: bool = Query(False), + start_time: Optional[datetime] = Query(None), + end_time: Optional[datetime] = Query(None), page: int = Query(1, ge=1), page_size: int = Query(50, ge=1, le=200), ) -> dict: @@ -495,6 +923,13 @@ async def get_security_access_logs( raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") client_type = client_type if isinstance(client_type, str) else None client_version = client_version if isinstance(client_version, str) else None + client_ip = client_ip if isinstance(client_ip, str) else None + category = category if isinstance(category, str) else None + severity = severity if isinstance(severity, str) else None + keyword = keyword if isinstance(keyword, str) else None + events_only = events_only if isinstance(events_only, bool) else False + start_time = start_time if isinstance(start_time, datetime) else None + end_time = end_time if isinstance(end_time, datetime) else None conditions = [] if status_min is not None: conditions.append(SecurityAccessLog.status_code >= status_min) @@ -504,6 +939,47 @@ async def get_security_access_logs( conditions.append(SecurityAccessLog.client_type == client_type) if client_version: conditions.append(SecurityAccessLog.client_version == client_version) + if client_ip: + conditions.append(SecurityAccessLog.client_ip.ilike(f"%{client_ip.strip()}%")) + if category: + conditions.append(SecurityAccessLog.category == category.strip().upper()) + if severity: + conditions.append(SecurityAccessLog.severity == severity.strip().upper()) + if events_only: + conditions.append(SecurityAccessLog.category != "OTHER") + if start_time: + conditions.append(SecurityAccessLog.created_at >= start_time) + if end_time: + conditions.append(SecurityAccessLog.created_at <= end_time) + if keyword and keyword.strip(): + token = keyword.strip() + like_token = f"%{token}%" + location_ips = await _resolve_security_location_keyword_ips(db, conditions, token) + matching_users_result = await db.execute( + select(cast(User.id, String)).where( + or_(User.full_name.ilike(like_token), User.email.ilike(like_token)) + ) + ) + matching_user_ids = list(matching_users_result.scalars().all()) + keyword_conditions = [ + SecurityAccessLog.method.ilike(like_token), + SecurityAccessLog.path.ilike(like_token), + SecurityAccessLog.client_ip.ilike(like_token), + SecurityAccessLog.client_type.ilike(like_token), + SecurityAccessLog.client_version.ilike(like_token), + SecurityAccessLog.client_platform.ilike(like_token), + SecurityAccessLog.user_agent.ilike(like_token), + SecurityAccessLog.auth_status.ilike(like_token), + SecurityAccessLog.user_identifier.ilike(like_token), + SecurityAccessLog.category.ilike(like_token), + SecurityAccessLog.severity.ilike(like_token), + cast(SecurityAccessLog.status_code, String).ilike(like_token), + ] + if location_ips: + keyword_conditions.append(SecurityAccessLog.client_ip.in_(location_ips)) + if matching_user_ids: + keyword_conditions.append(SecurityAccessLog.user_identifier.in_(matching_user_ids)) + conditions.append(or_(*keyword_conditions)) query = select(SecurityAccessLog) count_query = select(func.count()).select_from(SecurityAccessLog) @@ -512,6 +988,9 @@ async def get_security_access_logs( func.count().filter(SecurityAccessLog.auth_status == "ANONYMOUS").label("anonymous_count"), func.count().filter(SecurityAccessLog.auth_status == "INVALID_TOKEN").label("invalid_token_count"), func.count().filter(SecurityAccessLog.status_code >= 400).label("error_count"), + func.count(func.distinct(SecurityAccessLog.client_ip)).label("unique_ip_count"), + func.count().filter(SecurityAccessLog.severity.in_(("CRITICAL", "HIGH"))).label("high_risk_count"), + func.count().filter(SecurityAccessLog.category == "SERVER_ERROR").label("server_error_count"), ).select_from(SecurityAccessLog) for condition in conditions: query = query.where(condition) @@ -526,20 +1005,44 @@ async def get_security_access_logs( .limit(page_size) ) summary_row = summary_result.one() + category_query = select(SecurityAccessLog.category, func.count().label("count")).group_by(SecurityAccessLog.category) + severity_query = select(SecurityAccessLog.severity, func.count().label("count")).group_by(SecurityAccessLog.severity) + top_ip_query = ( + select( + SecurityAccessLog.client_ip, + func.count().label("total_count"), + func.count().filter(SecurityAccessLog.severity.in_(("CRITICAL", "HIGH"))).label("high_risk_count"), + func.max(SecurityAccessLog.created_at).label("last_seen_at"), + func.max(SecurityAccessLog.category).label("category"), + ) + .where(SecurityAccessLog.client_ip.is_not(None)) + .group_by(SecurityAccessLog.client_ip) + .order_by(desc("high_risk_count"), desc("total_count"), desc("last_seen_at")) + .limit(10) + ) + endpoint_query = ( + select(SecurityAccessLog.path, func.count().label("count")) + .group_by(SecurityAccessLog.path) + .order_by(desc("count")) + .limit(100) + ) + for condition in conditions: + category_query = category_query.where(condition) + severity_query = severity_query.where(condition) + top_ip_query = top_ip_query.where(condition) + endpoint_query = endpoint_query.where(condition) + category_result = await db.execute(category_query) + severity_result = await db.execute(severity_query) + top_ip_result = await db.execute(top_ip_query) + endpoint_result = await db.execute(endpoint_query) + endpoint_type_counts: dict[str, dict[str, str | int]] = {} + for row in endpoint_result.all(): + endpoint_type = _security_endpoint_type(row.path) + current = endpoint_type_counts.setdefault(endpoint_type, {"count": 0, "sample_path": row.path}) + current["count"] = int(current["count"]) + row.count logs = result.scalars().all() - user_ids: list[uuid.UUID] = [] - for log in logs: - if log.auth_status != "AUTHENTICATED" or not log.user_identifier: - continue - try: - user_ids.append(uuid.UUID(log.user_identifier)) - except ValueError: - continue - user_names: dict[str, str] = {} - if user_ids: - users_result = await db.execute(select(User.id, User.full_name).where(User.id.in_(user_ids))) - user_names = {str(row.id): row.full_name for row in users_result.all()} + user_names = await _security_user_names(db, list(logs)) items = [] for log in logs: @@ -563,6 +1066,9 @@ async def get_security_access_logs( "client_platform": log.client_platform, "build_channel": log.build_channel, "build_commit": log.build_commit, + "request_headers": _coerce_request_headers(getattr(log, "request_headers", None)), + "request_snapshot": _coerce_request_snapshot(getattr(log, "request_snapshot", None)), + "request_id": getattr(log, "request_id", None), "auth_status": log.auth_status, "user_identifier": log.user_identifier, "account_label": _security_account_label(log.auth_status, log.user_identifier, user_names), @@ -580,6 +1086,23 @@ async def get_security_access_logs( "anonymous_count": summary_row.anonymous_count, "invalid_token_count": summary_row.invalid_token_count, "error_count": summary_row.error_count, + "unique_ip_count": summary_row.unique_ip_count, + "high_risk_count": summary_row.high_risk_count, + "server_error_count": summary_row.server_error_count, + "category_counts": {str(row.category or "OTHER"): row.count for row in category_result.all()}, + "severity_counts": {str(row.severity or "LOW"): row.count for row in severity_result.all()}, + "endpoint_type_counts": endpoint_type_counts, + "top_ips": [ + { + "ip": row.client_ip, + "location": resolve_ip_location(row.client_ip).location, + "category": row.category or "OTHER", + "total_count": row.total_count, + "high_risk_count": row.high_risk_count, + "last_seen_at": row.last_seen_at.isoformat() if row.last_seen_at else None, + } + for row in top_ip_result.all() + ], }, "items": items, } @@ -742,113 +1265,319 @@ async def get_ip_locations( _=Depends(get_current_user), days: int = Query(7, ge=1, le=90), limit: int = Query(20, ge=1, le=100), + all_time: bool = False, ) -> dict: """获取 IP 省市属地统计。""" scope = await resolve_monitoring_scope(db, _) if not scope.is_admin and not scope.study_ids: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="权限不足") - start_time = datetime.now(timezone.utc) - timedelta(days=days) - query = ( + generated_at = datetime.now(timezone.utc) + server_location = await resolve_monitoring_server_location() + start_time = None if all_time else generated_at - timedelta(days=days) + matching_security_request = select(SecurityAccessLog.id).where( + PermissionAccessLog.request_id.is_not(None), + SecurityAccessLog.request_id == PermissionAccessLog.request_id, + ).exists() + permission_conditions = [ + PermissionAccessLog.ip_address.is_not(None), + ~matching_security_request, + ] + security_conditions = [SecurityAccessLog.client_ip.is_not(None)] + if start_time is not None: + permission_conditions.insert(0, PermissionAccessLog.created_at >= start_time) + security_conditions.insert(0, SecurityAccessLog.created_at >= start_time) + security_allowed = case((SecurityAccessLog.status_code < 400, True), else_=False) + permission_result = await db.execute( select( PermissionAccessLog.ip_address, PermissionAccessLog.user_id, PermissionAccessLog.allowed, + func.count().label("event_count"), + func.min(PermissionAccessLog.created_at).label("first_seen_at"), + func.max(PermissionAccessLog.created_at).label("last_seen_at"), ) - .where( - PermissionAccessLog.created_at >= start_time, - PermissionAccessLog.ip_address.is_not(None), + .where(*permission_conditions) + .group_by( + PermissionAccessLog.ip_address, + PermissionAccessLog.user_id, + PermissionAccessLog.allowed, ) ) - result = await db.execute(_apply_monitoring_scope_to_log_query(query, scope)) security_result = await db.execute( select( SecurityAccessLog.client_ip, SecurityAccessLog.user_identifier, SecurityAccessLog.auth_status, - SecurityAccessLog.status_code, - ).where( - SecurityAccessLog.created_at >= start_time, - SecurityAccessLog.client_ip.is_not(None), + security_allowed.label("allowed"), + SecurityAccessLog.severity, + SecurityAccessLog.category, + func.count().label("event_count"), + func.min(SecurityAccessLog.created_at).label("first_seen_at"), + func.max(SecurityAccessLog.created_at).label("last_seen_at"), + ) + .where(*security_conditions) + .group_by( + SecurityAccessLog.client_ip, + SecurityAccessLog.user_identifier, + SecurityAccessLog.auth_status, + security_allowed, + SecurityAccessLog.severity, + SecurityAccessLog.category, ) ) buckets: dict[tuple[str, str, str], dict] = {} all_ip_addresses: set[str] = set() all_user_ids: set[str] = set() + located_ip_addresses: set[str] = set() + private_ip_addresses: set[str] = set() + unknown_ip_addresses: set[str] = set() total_count = 0 allowed_count = 0 denied_count = 0 + observed_start_at: datetime | None = None + observed_end_at: datetime | None = None - def add_ip_location_row(ip_address: str, user_id: str | uuid.UUID | None, allowed: bool) -> None: - nonlocal total_count, allowed_count, denied_count - ip_info = resolve_ip_location(ip_address) - key = (ip_info.country, ip_info.province, ip_info.city) - location = " / ".join(part for part in [ip_info.country, ip_info.province, ip_info.city] if part) or ip_info.location or "未知" + resolved_locations: dict[str, IpLocation] = {} + resolved_metadata: dict[str, GeoLocationMetadata] = {} + + def normalize_observed_at(value: datetime | None) -> datetime | None: + if value is None: + return None + return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc) + + def add_ip_location_row( + ip_address: str, + user_id: str | uuid.UUID | None, + allowed: bool, + event_count: int, + first_seen_at: datetime | None, + last_seen_at: datetime | None, + *, + source_kind: str, + severity: str | None = None, + category: str | None = None, + auth_status: str | None = None, + ) -> None: + nonlocal total_count, allowed_count, denied_count, observed_start_at, observed_end_at + ip_info = resolved_locations.get(ip_address) + if ip_info is None: + ip_info = resolve_ip_location(ip_address) + resolved_locations[ip_address] = ip_info + metadata = resolved_metadata.get(ip_address) + if metadata is None: + metadata = resolve_geo_location_metadata(ip_info) + resolved_metadata[ip_address] = metadata + + longitude = metadata.longitude + latitude = metadata.latitude + if metadata.accuracy_level == "private": + longitude = server_location.longitude if server_location else None + latitude = server_location.latitude if server_location else None + private_ip_addresses.add(ip_address) + elif longitude is not None and latitude is not None: + located_ip_addresses.add(ip_address) + else: + unknown_ip_addresses.add(ip_address) + + country = metadata.country or ip_info.country + key = (metadata.country_code or country, metadata.region_code or ip_info.province, ip_info.city) + location = " / ".join(part for part in [country, ip_info.province, ip_info.city] if part) or ip_info.location or "未知" bucket = buckets.setdefault( key, { - "country": ip_info.country, + "country": country, + "country_code": metadata.country_code, "province": ip_info.province, + "region_code": metadata.region_code, "city": ip_info.city, - "isp": "", + "isp": ip_info.isp, "location": location, + "longitude": longitude, + "latitude": latitude, + "accuracy_level": metadata.accuracy_level, "total_count": 0, "allowed_count": 0, "denied_count": 0, + "security_event_count": 0, + "high_risk_count": 0, + "auth_failure_count": 0, + "categories": set(), "ip_addresses": set(), "user_ids": set(), + "first_seen_at": None, + "last_seen_at": None, }, ) - bucket["total_count"] += 1 - bucket["allowed_count" if allowed else "denied_count"] += 1 + bucket["total_count"] += event_count + bucket["allowed_count" if allowed else "denied_count"] += event_count + if ip_info.isp and bucket["isp"] and bucket["isp"] != ip_info.isp: + bucket["isp"] = "多运营商" + elif ip_info.isp: + bucket["isp"] = ip_info.isp bucket["ip_addresses"].add(ip_address) if user_id: bucket["user_ids"].add(str(user_id)) - total_count += 1 + normalized_first = normalize_observed_at(first_seen_at) + normalized_last = normalize_observed_at(last_seen_at) + if normalized_first and (bucket["first_seen_at"] is None or normalized_first < bucket["first_seen_at"]): + bucket["first_seen_at"] = normalized_first + if normalized_last and (bucket["last_seen_at"] is None or normalized_last > bucket["last_seen_at"]): + bucket["last_seen_at"] = normalized_last + if normalized_first and (observed_start_at is None or normalized_first < observed_start_at): + observed_start_at = normalized_first + if normalized_last and (observed_end_at is None or normalized_last > observed_end_at): + observed_end_at = normalized_last + + normalized_category = (category or "").upper() + normalized_severity = (severity or "").upper() + if source_kind == "security" and normalized_category not in {"", "OTHER", "NOT_FOUND_NOISE"}: + bucket["security_event_count"] += event_count + bucket["categories"].add(normalized_category) + if source_kind == "security" and normalized_severity in {"HIGH", "CRITICAL"}: + bucket["high_risk_count"] += event_count + if source_kind == "security" and not allowed and auth_status in {"INVALID_TOKEN", "ANONYMOUS"}: + bucket["auth_failure_count"] += event_count + total_count += event_count if allowed: - allowed_count += 1 + allowed_count += event_count else: - denied_count += 1 + denied_count += event_count all_ip_addresses.add(ip_address) if user_id: all_user_ids.add(str(user_id)) - for ip_address, user_id, allowed in result.all(): - add_ip_location_row(ip_address, user_id, allowed) + for ip_address, user_id, allowed, event_count, first_seen_at, last_seen_at in permission_result.all(): + add_ip_location_row( + ip_address, + user_id, + allowed, + int(event_count or 0), + first_seen_at, + last_seen_at, + source_kind="permission", + ) - for client_ip, user_identifier, auth_status, status_code in security_result.all(): + for client_ip, user_identifier, auth_status, allowed, severity, category, event_count, first_seen_at, last_seen_at in security_result.all(): user_id = None if auth_status == "AUTHENTICATED" and user_identifier: try: user_id = uuid.UUID(user_identifier) except ValueError: user_id = user_identifier - add_ip_location_row(client_ip, user_id, status_code < 400) + add_ip_location_row( + client_ip, + user_id, + bool(allowed), + int(event_count or 0), + first_seen_at, + last_seen_at, + source_kind="security", + severity=severity, + category=category, + auth_status=auth_status, + ) items = sorted(buckets.values(), key=lambda item: item["total_count"], reverse=True)[:limit] + + def serialize_item(item: dict) -> dict: + total = max(1, int(item["total_count"])) + denied_rate = item["denied_count"] / total + risk_score = min( + 100, + round( + denied_rate * 60 + + min(item["high_risk_count"] * 25, 40) + + min(item["auth_failure_count"] * 5, 20) + ), + ) + if item["high_risk_count"] > 0 or risk_score >= 60: + risk_level = "high" + elif item["denied_count"] > 0 or item["security_event_count"] > 0 or item["auth_failure_count"] > 0: + risk_level = "attention" + else: + risk_level = "normal" + risk_reasons = [] + if item["high_risk_count"]: + risk_reasons.append(f"{item['high_risk_count']} 次高风险安全事件") + if item["auth_failure_count"]: + risk_reasons.append(f"{item['auth_failure_count']} 次认证失败") + if item["denied_count"]: + risk_reasons.append(f"拒绝率 {round(denied_rate * 100, 1)}%") + + return { + "country": item["country"], + "country_code": item["country_code"], + "province": item["province"], + "region_code": item["region_code"], + "city": item["city"], + "isp": item["isp"], + "location": item["location"], + "longitude": item["longitude"], + "latitude": item["latitude"], + "accuracy_level": item["accuracy_level"], + "total_count": item["total_count"], + "allowed_count": item["allowed_count"], + "denied_count": item["denied_count"], + "denied_rate": round(denied_rate * 100, 1), + "unique_ip_count": len(item["ip_addresses"]), + "unique_user_count": len(item["user_ids"]), + "security_event_count": item["security_event_count"], + "high_risk_count": item["high_risk_count"], + "risk_score": risk_score, + "risk_level": risk_level, + "risk_reasons": risk_reasons, + "categories": sorted(item["categories"]), + "first_seen_at": item["first_seen_at"].isoformat() if item["first_seen_at"] else None, + "last_seen_at": item["last_seen_at"].isoformat() if item["last_seen_at"] else None, + } + + effective_start_at = start_time or observed_start_at or generated_at - timedelta( + days=settings.MONITORING_ACCESS_LOG_RETENTION_DAYS + ) + located_count = len(located_ip_addresses) + private_count = len(private_ip_addresses) + unknown_count = len(unknown_ip_addresses) + public_ip_count = located_count + unknown_count + timeline_granularity = "hour" if not all_time and days <= 1 else "day" + timeline = await get_source_location_timeline( + db, + start_at=effective_start_at, + end_at=generated_at, + granularity=timeline_granularity, + ) return { - "days": days, + "days": None if all_time else days, + "generated_at": generated_at.isoformat(), + "period": { + "mode": "retained_all" if all_time else "rolling", + "requested_days": None if all_time else days, + "start_at": effective_start_at.isoformat(), + "end_at": generated_at.isoformat(), + "observed_end_at": observed_end_at.isoformat() if observed_end_at else None, + "retention_days": settings.MONITORING_ACCESS_LOG_RETENTION_DAYS, + }, + "server_location": server_location.to_public_dict() if server_location else None, + "data_quality": { + "resolver": "ip2region", + "located_ip_count": located_count, + "private_ip_count": private_count, + "unknown_ip_count": unknown_count, + "location_coverage_rate": round((located_count / public_ip_count) * 100, 1) if public_ip_count else 100.0, + "returned_region_count": len(items), + }, + "timeline_granularity": timeline_granularity, + "timeline_complete_through": generated_at.replace(minute=0, second=0, microsecond=0).isoformat(), + "timeline": timeline, "summary": { "total_count": total_count, "allowed_count": allowed_count, "denied_count": denied_count, "unique_ip_count": len(all_ip_addresses), "unique_user_count": len(all_user_ids), + "security_event_count": sum(item["security_event_count"] for item in buckets.values()), + "high_risk_count": sum(item["high_risk_count"] for item in buckets.values()), }, - "items": [ - { - "country": item["country"], - "province": item["province"], - "city": item["city"], - "isp": item["isp"], - "location": item["location"], - "total_count": item["total_count"], - "allowed_count": item["allowed_count"], - "denied_count": item["denied_count"], - "unique_ip_count": len(item["ip_addresses"]), - "unique_user_count": len(item["user_ids"]), - } - for item in items - ], + "items": [serialize_item(item) for item in items], } diff --git a/backend/app/api/v1/users.py b/backend/app/api/v1/users.py index dec364aa..f9f80787 100644 --- a/backend/app/api/v1/users.py +++ b/backend/app/api/v1/users.py @@ -9,7 +9,8 @@ from app.schemas.common import PaginatedResponse from app.crud import user as user_crud from app.crud import member as member_crud from app.utils.pagination import paginate -from app.schemas.user import UserCreate, UserRead, UserStatus, UserUpdate +from app.schemas.user import UserCreate, UserLoginActivityRead, UserRead, UserStatus, UserUpdate +from app.services.user_login_sessions import get_login_summaries, list_login_activities router = APIRouter() @@ -25,7 +26,35 @@ async def list_users( ) -> PaginatedResponse[UserRead]: users = await user_crud.list_users(db, skip=skip, limit=limit, keyword=keyword, status=user_status) total_users = await user_crud.count_users(db, keyword=keyword, status=user_status) - return paginate(list(users), total=total_users) + summaries = await get_login_summaries(db, [user.id for user in users]) + items = [] + for user in users: + summary = summaries.get(user.id) + item = UserRead.model_validate(user).model_dump() + if summary: + item.update( + { + "login_status": summary.status, + "last_login_at": summary.last_login_at, + "last_seen_at": summary.last_seen_at, + "last_client_type": summary.client_type, + "active_session_count": summary.active_session_count, + } + ) + items.append(item) + return paginate(items, total=total_users) + + +@router.get("/{user_id}/login-activities", response_model=list[UserLoginActivityRead]) +async def read_user_login_activities( + user_id: uuid.UUID, + limit: int = Query(default=30, ge=1, le=100), + db: AsyncSession = Depends(get_db_session), + current_user=Depends(require_roles(["ADMIN"])), +) -> list[UserLoginActivityRead]: + if not await user_crud.get_by_id(db, user_id): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="用户不存在") + return await list_login_activities(db, user_id=user_id, limit=limit) @router.post("/", response_model=UserRead, status_code=status.HTTP_201_CREATED) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 8781e6c6..76f3b8e4 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -33,6 +33,18 @@ class Settings(BaseSettings): ) IP2REGION_XDB_PATH: Optional[str] = None IP2REGION_IPV6_XDB_PATH: Optional[str] = None + TRUSTED_PROXY_CIDRS: str = "127.0.0.1/32,::1/128,172.16.0.0/12" + MONITORING_SERVER_PUBLIC_IP: Optional[str] = None + MONITORING_PUBLIC_IP_DISCOVERY_URLS: str = ( + "https://api64.ipify.org,https://icanhazip.com" + ) + MONITORING_PUBLIC_IP_DISCOVERY_TIMEOUT_SECONDS: float = Field(default=2.5, ge=0.5, le=10) + MONITORING_SERVER_LOCATION_CACHE_SECONDS: int = Field(default=86400, ge=300, le=604800) + MONITORING_ACCESS_LOG_RETENTION_DAYS: int = Field(default=90, ge=7, le=3650) + MONITORING_METRIC_RETENTION_DAYS: int = Field(default=400, ge=30, le=3650) + MONITORING_RETENTION_INTERVAL_SECONDS: int = Field(default=86400, ge=60, le=604800) + USER_LOGIN_ACTIVITY_RETENTION_DAYS: int = Field(default=180, ge=30, le=3650) + USER_SESSION_ONLINE_SECONDS: int = Field(default=300, ge=60, le=3600) @lru_cache diff --git a/backend/app/core/deps.py b/backend/app/core/deps.py index d14a4140..0f63719c 100644 --- a/backend/app/core/deps.py +++ b/backend/app/core/deps.py @@ -286,8 +286,9 @@ def _enqueue_permission_log( writer = get_log_writer() if writer: - forwarded = request.headers.get("x-forwarded-for") - ip = forwarded.split(",")[0].strip() if forwarded else (request.client.host if request.client else None) + from app.core.request_context import build_request_audit_context, get_request_audit_context + + context = get_request_audit_context() or build_request_audit_context(request) writer.enqueue({ "study_id": study_id, "user_id": user_id, @@ -295,7 +296,16 @@ def _enqueue_permission_log( "role": role, "allowed": allowed, "elapsed_ms": elapsed_ms, - "ip_address": ip, + "ip_address": context.client_ip, + "user_agent": context.user_agent, + "client_type": context.client_type, + "client_version": context.client_version, + "client_platform": context.client_platform, + "build_channel": context.build_channel, + "build_commit": context.build_commit, + "request_headers": context.request_headers, + "request_snapshot": context.request_snapshot, + "request_id": context.request_id, }) diff --git a/backend/app/core/request_context.py b/backend/app/core/request_context.py index 1d38a908..a2a92cee 100644 --- a/backend/app/core/request_context.py +++ b/backend/app/core/request_context.py @@ -2,14 +2,19 @@ from __future__ import annotations +import ipaddress import re +import uuid from contextvars import ContextVar, Token from dataclasses import dataclass from typing import Any +from app.core.config import settings + @dataclass(frozen=True) class RequestAuditContext: + request_id: str | None = None client_ip: str | None = None user_agent: str | None = None client_type: str | None = None @@ -17,6 +22,8 @@ class RequestAuditContext: client_platform: str | None = None build_channel: str | None = None build_commit: str | None = None + request_headers: dict[str, str] | None = None + request_snapshot: dict[str, Any] | None = None _request_audit_context: ContextVar[RequestAuditContext | None] = ContextVar( @@ -25,11 +32,37 @@ _request_audit_context: ContextVar[RequestAuditContext | None] = ContextVar( ) _SENSITIVE_TEXT_PATTERN = re.compile( - r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]+|((?:access_)?token|authorization)=([^&\s]+)" + r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]+|((?:access_)?token|authorization|password|passwd|secret|credential|api[-_]?key|session)=([^&\s]+)" ) +_SENSITIVE_HEADER_NAME_PATTERN = re.compile( + r"(?i)(authorization|cookie|set-cookie|token|password|passwd|secret|credential|api[-_]?key|session)" +) +_SAFE_REQUEST_HEADER_NAMES = frozenset( + { + "accept", + "accept-language", + "content-type", + "host", + "origin", + "user-agent", + "x-request-id", + "x-correlation-id", + } +) +_SAFE_REQUEST_HEADER_PREFIXES = ("x-ctms-",) +_SENSITIVE_PARAMETER_NAME_PATTERN = re.compile( + r"(?i)(token|authorization|password|passwd|secret|credential|api[-_]?key|session|" + r"subject|participant|patient|user_?name|full_?name|email|phone|mobile|id_?card|" + r"identity|certificate|contact|address)" +) +_KNOWN_CLIENT_TYPES = frozenset({"web", "desktop"}) +_CTMS_CLIENT_SOURCE_TO_TYPE = { + "ctms-web": "web", + "ctms-desktop": "desktop", +} -def _clean_header_value(value: str | None, max_length: int) -> str | None: +def _clean_audit_value(value: str | None, max_length: int) -> str | None: if value is None: return None cleaned = _SENSITIVE_TEXT_PATTERN.sub(lambda m: f"{m.group(1) or m.group(2) + '='}[redacted]", value.strip()) @@ -38,26 +71,174 @@ def _clean_header_value(value: str | None, max_length: int) -> str | None: return cleaned[:max_length] +def _clean_header_value(value: str | None, max_length: int) -> str | None: + return _clean_audit_value(value, max_length) + + +def resolve_ctms_client_type(headers: Any) -> str | None: + source = _clean_header_value(headers.get("x-ctms-client-source"), 32) + if source: + mapped_type = _CTMS_CLIENT_SOURCE_TO_TYPE.get(source.strip().lower()) + if mapped_type: + return mapped_type + + client_type = _clean_header_value(headers.get("x-ctms-client-type"), 16) + if not client_type: + return None + normalized = client_type.strip().lower() + return normalized if normalized in _KNOWN_CLIENT_TYPES else normalized[:16] + + +def build_sanitized_request_headers(request: Any) -> dict[str, str] | None: + captured: dict[str, str] = {} + for raw_name, raw_value in request.headers.items(): + name = str(raw_name).strip().lower() + if not name: + continue + if _SENSITIVE_HEADER_NAME_PATTERN.search(name): + captured[name] = "[redacted]" + continue + if name not in _SAFE_REQUEST_HEADER_NAMES and not name.startswith(_SAFE_REQUEST_HEADER_PREFIXES): + continue + value = _clean_header_value(str(raw_value), 500) + if value: + captured[name] = value + return captured or None + + +def _sanitize_named_value(raw_name: Any, raw_value: Any, max_length: int = 500) -> dict[str, str] | None: + name = _clean_audit_value(str(raw_name), 120) + if not name: + return None + if _SENSITIVE_PARAMETER_NAME_PATTERN.search(name): + return {"name": name, "value": "[redacted]"} + value = _clean_audit_value(str(raw_value), max_length) + if value is None: + return {"name": name, "value": ""} + return {"name": name, "value": value} + + +def _query_param_items(request: Any) -> list[dict[str, str]] | None: + query_params = getattr(request, "query_params", None) + if not query_params: + return None + if hasattr(query_params, "multi_items"): + raw_items = query_params.multi_items() + else: + raw_items = query_params.items() + items = [ + item + for item in (_sanitize_named_value(name, value) for name, value in raw_items) + if item is not None + ][:50] + return items or None + + +def _sanitized_query_string(items: list[dict[str, str]] | None) -> str | None: + if not items: + return None + value = "&".join(f"{item['name']}={item['value']}" for item in items) + return value[:2000] or None + + +def _request_client_snapshot(request: Any) -> dict[str, str | int | None] | None: + client = getattr(request, "client", None) + if not client: + return None + host = _clean_audit_value(getattr(client, "host", None), 120) + port = getattr(client, "port", None) + if host is None and port is None: + return None + return {"host": host, "port": port} + + +def build_request_snapshot(request: Any, *, request_id: str | None = None) -> dict[str, Any]: + headers = getattr(request, "headers", {}) + scope = getattr(request, "scope", {}) or {} + url = getattr(request, "url", None) + query_params = _query_param_items(request) + path = getattr(url, "path", None) or scope.get("path") + method = getattr(request, "method", None) or scope.get("method") + snapshot: dict[str, Any] = { + "request_id": request_id, + "method": _clean_audit_value(str(method).upper() if method else None, 12), + "path": _clean_audit_value(path, 500), + "query_string": _sanitized_query_string(query_params), + "query_params": query_params, + "headers": build_sanitized_request_headers(request), + "http_version": _clean_audit_value(scope.get("http_version"), 16), + "scheme": _clean_audit_value(getattr(url, "scheme", None) or scope.get("scheme"), 16), + "client": _request_client_snapshot(request), + "content": { + "type": _clean_audit_value(headers.get("content-type"), 200), + "length": _clean_audit_value(headers.get("content-length"), 32), + }, + "body": { + "captured": False, + "reason": "body_not_captured_by_audit_policy", + }, + } + return {key: value for key, value in snapshot.items() if value not in (None, {}, [])} + + +def _parse_ip(value: str | None): + try: + return ipaddress.ip_address((value or "").strip()) + except ValueError: + return None + + +def _trusted_proxy_networks(): + networks = [] + for value in settings.TRUSTED_PROXY_CIDRS.split(","): + candidate = value.strip() + if not candidate: + continue + try: + networks.append(ipaddress.ip_network(candidate, strict=False)) + except ValueError: + continue + return tuple(networks) + + +def _is_trusted_proxy(value: str | None) -> bool: + address = _parse_ip(value) + return bool(address and any(address in network for network in _trusted_proxy_networks())) + + def resolve_client_ip(request: Any) -> str | None: + peer_ip = request.client.host if request.client else None + if not _is_trusted_proxy(peer_ip): + return _clean_header_value(peer_ip, 45) + forwarded = request.headers.get("x-forwarded-for") if forwarded: - return _clean_header_value(forwarded.split(",")[0].strip(), 45) + chain = [part.strip() for part in forwarded.split(",") if _parse_ip(part)] + chain.append(str(peer_ip)) + for candidate in reversed(chain): + if not _is_trusted_proxy(candidate): + return _clean_header_value(candidate, 45) + real_ip = request.headers.get("x-real-ip") - if real_ip: + if _parse_ip(real_ip): return _clean_header_value(real_ip.strip(), 45) - return _clean_header_value(request.client.host if request.client else None, 45) + return _clean_header_value(peer_ip, 45) def build_request_audit_context(request: Any) -> RequestAuditContext: headers = request.headers + request_id = str(uuid.uuid4()) return RequestAuditContext( + request_id=request_id, client_ip=resolve_client_ip(request), user_agent=_clean_header_value(headers.get("user-agent"), 500), - client_type=_clean_header_value(headers.get("x-ctms-client-type"), 16), + client_type=resolve_ctms_client_type(headers), client_version=_clean_header_value(headers.get("x-ctms-client-version"), 32), client_platform=_clean_header_value(headers.get("x-ctms-client-platform"), 16), build_channel=_clean_header_value(headers.get("x-ctms-build-channel"), 16), build_commit=_clean_header_value(headers.get("x-ctms-build-commit"), 64), + request_headers=build_sanitized_request_headers(request), + request_snapshot=build_request_snapshot(request, request_id=request_id), ) diff --git a/backend/app/core/security.py b/backend/app/core/security.py index f0692ee1..3b968cdb 100644 --- a/backend/app/core/security.py +++ b/backend/app/core/security.py @@ -22,6 +22,7 @@ def create_access_token( max_age_seconds: Optional[int] = None, issued_at: Optional[datetime] = None, client_type: Optional[str] = None, + session_id: Optional[str] = None, ) -> str: now = issued_at or datetime.now(timezone.utc) if now.tzinfo is None: @@ -42,6 +43,8 @@ def create_access_token( } if client_type: to_encode["client_type"] = client_type + if session_id: + to_encode["sid"] = session_id return jwt.encode(to_encode, settings.JWT_SECRET_KEY, algorithm=ALGORITHM) diff --git a/backend/app/db/base.py b/backend/app/db/base.py index dac30b5b..b39b03d6 100644 --- a/backend/app/db/base.py +++ b/backend/app/db/base.py @@ -42,6 +42,8 @@ from app.models.permission_access_log import PermissionAccessLog # noqa: F401 from app.models.permission_metric_snapshot import PermissionMetricSnapshot # noqa: F401 from app.models.permission_template import PermissionTemplate, PermissionTemplateVersion # noqa: F401 from app.models.security_access_log import SecurityAccessLog # noqa: F401 +from app.models.source_location_snapshot import SourceLocationSnapshot # noqa: F401 +from app.models.user_login_session import UserLoginSession # noqa: F401 from app.models.desktop_notification import ( # noqa: F401 DesktopNotificationDelivery, DesktopNotificationSubscription, diff --git a/backend/app/main.py b/backend/app/main.py index 39dbcc67..d3c6b701 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -5,8 +5,9 @@ import time from contextlib import asynccontextmanager from collections import defaultdict -from fastapi import FastAPI +from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse from sqlalchemy import text from app.api.v1.router import api_router @@ -19,15 +20,23 @@ from app.db.session import SessionLocal, engine from app.services.visit_scheduler import run_daily_lost_visit_job from app.services.permission_log_writer import start_log_writer, stop_log_writer from app.services.permission_metric_aggregator import run_hourly_metric_aggregation +from app.services.source_location_aggregator import run_hourly_source_location_aggregation +from app.services.monitoring_server_location import resolve_monitoring_server_location +from app.services.monitoring_retention import run_monitoring_retention from app.services.security_access_log_writer import ( get_security_log_writer, start_security_log_writer, stop_security_log_writer, ) from app.core.security import decode_token +from app.core.deps import get_db_session from app.core.request_context import ( build_request_audit_context, + build_request_snapshot, + build_sanitized_request_headers, + get_request_audit_context, reset_request_audit_context, + resolve_ctms_client_type, resolve_client_ip, set_request_audit_context, ) @@ -42,8 +51,12 @@ setup_config_stats: dict[str, int] = defaultdict(int) @asynccontextmanager async def lifespan(_: FastAPI): stop_event = asyncio.Event() + await resolve_monitoring_server_location() scheduler_task = asyncio.create_task(run_daily_lost_visit_job(stop_event)) aggregator_task = asyncio.create_task(run_hourly_metric_aggregation(stop_event)) + source_location_aggregator_task = asyncio.create_task( + run_hourly_source_location_aggregation(stop_event) + ) await start_log_writer() await start_security_log_writer() if settings.ENV == "development": @@ -52,12 +65,15 @@ async def lifespan(_: FastAPI): await conn.run_sync(Base.metadata.create_all) async with SessionLocal() as session: await ensure_admin_exists(session) + retention_task = asyncio.create_task(run_monitoring_retention(stop_event)) yield stop_event.set() await stop_log_writer() await stop_security_log_writer() await scheduler_task await aggregator_task + await source_location_aggregator_task + await retention_task async def _ensure_legacy_primary_keys(conn) -> None: @@ -129,19 +145,23 @@ def create_app() -> FastAPI: "Accept", "Authorization", "Content-Type", + "X-CTMS-Client-Source", "X-CTMS-Client-Type", "X-CTMS-Client-Version", "X-CTMS-Client-Platform", "X-CTMS-Build-Channel", "X-CTMS-Build-Commit", + "X-Request-ID", + "X-Correlation-ID", ], + expose_headers=["X-Request-ID"], ) @app.middleware("http") async def setup_config_monitoring_middleware(request, call_next): path = request.url.path is_setup_config_path = "/api/v1/studies/" in path and "/setup-config" in path - should_security_log = path.startswith("/api/") + should_security_log = path.startswith("/api/") and path != "/api/v1/auth/session/heartbeat" started_at = time.perf_counter() audit_context = build_request_audit_context(request) audit_context_token = set_request_audit_context(audit_context) @@ -167,6 +187,8 @@ def create_app() -> FastAPI: raise status_code = int(response.status_code) + if audit_context.request_id: + response.headers["X-Request-ID"] = audit_context.request_id if is_setup_config_path: normalized_path = UUID_RE.sub("{study_id}", path) duration_ms = int((time.perf_counter() - started_at) * 1000) @@ -229,6 +251,36 @@ def create_app() -> FastAPI: async def health() -> dict[str, str]: return {"status": "ok"} + @app.get( + "/readyz", + tags=["health"], + summary="服务就绪检查", + description="验证应用可访问数据库；失败时返回 503。", + ) + async def readiness(db=Depends(get_db_session)): + started_at = time.perf_counter() + try: + await db.execute(text("SELECT 1")) + except Exception as exc: + logger.exception("Readiness database check failed") + return JSONResponse( + status_code=503, + content={ + "status": "not_ready", + "database": { + "status": "unhealthy", + "error_type": type(exc).__name__, + }, + }, + ) + return { + "status": "ready", + "database": { + "status": "healthy", + "latency_ms": round((time.perf_counter() - started_at) * 1000, 2), + }, + } + @app.get( "/health/setup-config-stats", tags=["health"], @@ -271,6 +323,7 @@ def _enqueue_security_access_log(request, path: str, status_code: int, started_a if not writer: return auth_status, user_identifier = _resolve_auth_context(request) + context = get_request_audit_context() writer.enqueue( { "method": request.method, @@ -279,11 +332,14 @@ def _enqueue_security_access_log(request, path: str, status_code: int, started_a "elapsed_ms": round((time.perf_counter() - started_at) * 1000, 2), "client_ip": resolve_client_ip(request), "user_agent": request.headers.get("user-agent"), - "client_type": request.headers.get("x-ctms-client-type"), + "client_type": resolve_ctms_client_type(request.headers), "client_version": request.headers.get("x-ctms-client-version"), "client_platform": request.headers.get("x-ctms-client-platform"), "build_channel": request.headers.get("x-ctms-build-channel"), "build_commit": request.headers.get("x-ctms-build-commit"), + "request_headers": context.request_headers if context else build_sanitized_request_headers(request), + "request_snapshot": context.request_snapshot if context else build_request_snapshot(request), + "request_id": context.request_id if context else None, "auth_status": auth_status, "user_identifier": user_identifier, } diff --git a/backend/app/models/permission_access_log.py b/backend/app/models/permission_access_log.py index 3ccf0870..402a6751 100644 --- a/backend/app/models/permission_access_log.py +++ b/backend/app/models/permission_access_log.py @@ -6,13 +6,15 @@ import uuid from datetime import datetime from typing import Optional -from sqlalchemy import Boolean, DateTime, Float, ForeignKey, Index, String -from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy import JSON, Boolean, DateTime, Float, ForeignKey, Index, String +from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.sql import func from app.db.base_class import Base +JSONB_TYPE = JSON().with_variant(JSONB, "postgresql") + class PermissionAccessLog(Base): __tablename__ = "permission_access_logs" @@ -22,6 +24,9 @@ class PermissionAccessLog(Base): Index("ix_perm_log_endpoint_created", "endpoint_key", "created_at"), Index("ix_perm_log_created_at", "created_at"), Index("ix_perm_log_allowed", "allowed", "created_at"), + Index("ix_perm_log_ip_created", "ip_address", "created_at"), + Index("ix_perm_log_client_source_created", "client_type", "created_at"), + Index("ix_perm_log_request_id", "request_id"), ) id: Mapped[uuid.UUID] = mapped_column( @@ -38,6 +43,15 @@ class PermissionAccessLog(Base): allowed: Mapped[bool] = mapped_column(Boolean, nullable=False) elapsed_ms: Mapped[float] = mapped_column(Float, nullable=False) ip_address: Mapped[Optional[str]] = mapped_column(String(45), nullable=True) + user_agent: Mapped[Optional[str]] = mapped_column(String(500), nullable=True) + client_type: Mapped[Optional[str]] = mapped_column(String(16), nullable=True) + client_version: Mapped[Optional[str]] = mapped_column(String(32), nullable=True) + client_platform: Mapped[Optional[str]] = mapped_column(String(16), nullable=True) + build_channel: Mapped[Optional[str]] = mapped_column(String(16), nullable=True) + build_commit: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + request_headers: Mapped[Optional[dict]] = mapped_column(JSONB_TYPE, nullable=True) + request_snapshot: Mapped[Optional[dict]] = mapped_column(JSONB_TYPE, nullable=True) + request_id: Mapped[Optional[str]] = mapped_column(String(36), nullable=True) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) diff --git a/backend/app/models/security_access_log.py b/backend/app/models/security_access_log.py index c52217ee..247fabe9 100644 --- a/backend/app/models/security_access_log.py +++ b/backend/app/models/security_access_log.py @@ -6,13 +6,15 @@ import uuid from datetime import datetime from typing import Optional -from sqlalchemy import DateTime, Float, Index, Integer, String -from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy import JSON, DateTime, Float, Index, Integer, String +from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.sql import func from app.db.base_class import Base +JSONB_TYPE = JSON().with_variant(JSONB, "postgresql") + class SecurityAccessLog(Base): __tablename__ = "security_access_logs" @@ -21,6 +23,9 @@ class SecurityAccessLog(Base): Index("ix_security_log_ip_created", "client_ip", "created_at"), Index("ix_security_log_status_created", "status_code", "created_at"), Index("ix_security_log_auth_created", "auth_status", "created_at"), + Index("ix_security_log_request_id", "request_id"), + Index("ix_security_log_category_created", "category", "created_at"), + Index("ix_security_log_severity_created", "severity", "created_at"), ) id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) @@ -35,8 +40,13 @@ class SecurityAccessLog(Base): client_platform: Mapped[Optional[str]] = mapped_column(String(16), nullable=True) build_channel: Mapped[Optional[str]] = mapped_column(String(16), nullable=True) build_commit: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + request_headers: Mapped[Optional[dict]] = mapped_column(JSONB_TYPE, nullable=True) + request_snapshot: Mapped[Optional[dict]] = mapped_column(JSONB_TYPE, nullable=True) auth_status: Mapped[str] = mapped_column(String(30), nullable=False) user_identifier: Mapped[Optional[str]] = mapped_column(String(80), nullable=True) + request_id: Mapped[Optional[str]] = mapped_column(String(36), nullable=True) + category: Mapped[Optional[str]] = mapped_column(String(30), nullable=True) + severity: Mapped[Optional[str]] = mapped_column(String(16), nullable=True) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) diff --git a/backend/app/models/source_location_snapshot.py b/backend/app/models/source_location_snapshot.py new file mode 100644 index 00000000..414c1f16 --- /dev/null +++ b/backend/app/models/source_location_snapshot.py @@ -0,0 +1,48 @@ +"""Hourly source-location rollup for monitoring analytics.""" + +from __future__ import annotations + +import uuid +from datetime import datetime + +from sqlalchemy import DateTime, Float, Index, Integer, String, UniqueConstraint +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.sql import func + +from app.db.base_class import Base + + +class SourceLocationSnapshot(Base): + __tablename__ = "source_location_snapshots" + __table_args__ = ( + UniqueConstraint("bucket_time", "ip_hash", "user_hash", name="uq_source_location_bucket_identity"), + Index("ix_source_location_bucket", "bucket_time"), + Index("ix_source_location_country_bucket", "country_code", "bucket_time"), + Index("ix_source_location_risk_bucket", "high_risk_count", "bucket_time"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + bucket_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + ip_hash: Mapped[str] = mapped_column(String(64), nullable=False) + user_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="") + country: Mapped[str] = mapped_column(String(100), nullable=False, default="") + country_code: Mapped[str] = mapped_column(String(16), nullable=False, default="") + province: Mapped[str] = mapped_column(String(100), nullable=False, default="") + region_code: Mapped[str] = mapped_column(String(24), nullable=False, default="") + city: Mapped[str] = mapped_column(String(100), nullable=False, default="") + isp: Mapped[str] = mapped_column(String(160), nullable=False, default="") + location: Mapped[str] = mapped_column(String(320), nullable=False, default="") + longitude: Mapped[float | None] = mapped_column(Float, nullable=True) + latitude: Mapped[float | None] = mapped_column(Float, nullable=True) + accuracy_level: Mapped[str] = mapped_column(String(16), nullable=False, default="unknown") + allowed_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + denied_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + security_event_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + high_risk_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + auth_failure_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/models/user_login_session.py b/backend/app/models/user_login_session.py new file mode 100644 index 00000000..f0d4b588 --- /dev/null +++ b/backend/app/models/user_login_session.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import uuid +from datetime import datetime + +from sqlalchemy import DateTime, ForeignKey, String, func +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base_class import Base + + +class UserLoginSession(Base): + """Server-side login activity record without storing credentials or raw IPs.""" + + __tablename__ = "user_login_sessions" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True) + user_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + client_type: Mapped[str] = mapped_column(String(16), nullable=False, default="web") + client_platform: Mapped[str | None] = mapped_column(String(32), nullable=True) + client_version: Mapped[str | None] = mapped_column(String(64), nullable=True) + client_source: Mapped[str | None] = mapped_column(String(32), nullable=True) + login_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + end_reason: Mapped[str | None] = mapped_column(String(32), nullable=True) diff --git a/backend/app/schemas/user.py b/backend/app/schemas/user.py index f4b42bc2..9a4c9ef7 100644 --- a/backend/app/schemas/user.py +++ b/backend/app/schemas/user.py @@ -6,6 +6,7 @@ from typing import Literal, Optional from pydantic import BaseModel, ConfigDict, EmailStr, Field, field_validator UserStatus = Literal["PENDING", "ACTIVE", "REJECTED", "DISABLED"] +LoginStatus = Literal["ONLINE", "OFFLINE"] PASSWORD_REGEX = re.compile(r"^(?=.*[A-Za-z])(?=.*\d).{8,}$") @@ -60,6 +61,11 @@ class UserRead(BaseModel): approved_at: Optional[datetime] = None approved_by: Optional[uuid.UUID] = None avatar_url: Optional[str] = None + login_status: LoginStatus = "OFFLINE" + last_login_at: Optional[datetime] = None + last_seen_at: Optional[datetime] = None + last_client_type: Optional[Literal["web", "desktop"]] = None + active_session_count: int = 0 model_config = ConfigDict(from_attributes=True) @@ -78,6 +84,19 @@ class UserResponse(UserRead): pass +class UserLoginActivityRead(BaseModel): + id: uuid.UUID + client_type: Literal["web", "desktop"] + client_platform: Optional[str] = None + client_version: Optional[str] = None + login_at: datetime + last_seen_at: datetime + ended_at: Optional[datetime] = None + end_reason: Optional[str] = None + + model_config = ConfigDict(from_attributes=True) + + class UserSelfUpdate(_PasswordValidator): full_name: Optional[str] = None clinical_department: Optional[str] = None diff --git a/backend/app/services/geo_location_metadata.py b/backend/app/services/geo_location_metadata.py new file mode 100644 index 00000000..8fef7c01 --- /dev/null +++ b/backend/app/services/geo_location_metadata.py @@ -0,0 +1,175 @@ +"""Canonical map metadata for monitoring source locations. + +ip2region provides administrative names but no coordinates. This module keeps +the normalization and centroid contract on the server so web and desktop +clients render the same location semantics. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from app.services.ip_location import IpLocation + + +@dataclass(frozen=True) +class GeoLocationMetadata: + country: str + country_code: str + region_code: str + longitude: float | None + latitude: float | None + accuracy_level: str + + +COUNTRY_ALIASES = { + "中国": "China", + "China": "China", + "Mainland China": "China", + "中国香港": "China", + "中国澳门": "China", + "中国台湾": "China", + "美国": "United States", + "United States": "United States", + "United States of America": "United States", + "USA": "United States", + "土耳其": "Türkiye", + "Turkey": "Türkiye", + "Türkiye": "Türkiye", +} + +COUNTRY_CODES = { + "China": "CN", + "United States": "US", + "Netherlands": "NL", + "Türkiye": "TR", + "Australia": "AU", + "Japan": "JP", + "Singapore": "SG", + "Germany": "DE", + "France": "FR", + "United Kingdom": "GB", + "Canada": "CA", + "India": "IN", + "Russia": "RU", +} + +COUNTRY_CENTROIDS = { + "China": (104.1954, 35.8617), + "United States": (-95.7129, 37.0902), + "Netherlands": (5.2913, 52.1326), + "Türkiye": (35.2433, 38.9637), + "Australia": (133.7751, -25.2744), + "Japan": (138.2529, 36.2048), + "Singapore": (103.8198, 1.3521), + "Germany": (10.4515, 51.1657), + "France": (2.2137, 46.2276), + "United Kingdom": (-3.436, 55.3781), + "Canada": (-106.3468, 56.1304), + "India": (78.9629, 20.5937), + "Russia": (105.3188, 61.524), +} + +CHINA_REGION_METADATA = { + "北京市": ("CN-BJ", 116.4074, 39.9042), + "天津市": ("CN-TJ", 117.2008, 39.0842), + "河北省": ("CN-HE", 114.5025, 38.0455), + "山西省": ("CN-SX", 112.5492, 37.857), + "内蒙古自治区": ("CN-NM", 111.6708, 40.8183), + "辽宁省": ("CN-LN", 123.4315, 41.8057), + "吉林省": ("CN-JL", 125.3245, 43.8868), + "黑龙江省": ("CN-HL", 126.6424, 45.7567), + "上海市": ("CN-SH", 121.4737, 31.2304), + "江苏省": ("CN-JS", 118.7633, 32.0617), + "浙江省": ("CN-ZJ", 120.1551, 30.2741), + "安徽省": ("CN-AH", 117.2272, 31.8206), + "福建省": ("CN-FJ", 119.2965, 26.0745), + "江西省": ("CN-JX", 115.8582, 28.682), + "山东省": ("CN-SD", 117.1201, 36.6512), + "河南省": ("CN-HA", 113.6254, 34.7466), + "湖北省": ("CN-HB", 114.3055, 30.5928), + "湖南省": ("CN-HN", 112.9388, 28.2282), + "广东省": ("CN-GD", 113.2644, 23.1291), + "广西壮族自治区": ("CN-GX", 108.3669, 22.817), + "海南省": ("CN-HI", 110.3312, 20.0311), + "重庆": ("CN-CQ", 106.5516, 29.563), + "重庆市": ("CN-CQ", 106.5516, 29.563), + "四川省": ("CN-SC", 104.0665, 30.5723), + "贵州省": ("CN-GZ", 106.6302, 26.647), + "云南省": ("CN-YN", 102.8329, 24.8801), + "西藏自治区": ("CN-XZ", 91.1322, 29.6604), + "陕西省": ("CN-SN", 108.9398, 34.3416), + "甘肃省": ("CN-GS", 103.8343, 36.0611), + "青海省": ("CN-QH", 101.7782, 36.6171), + "宁夏回族自治区": ("CN-NX", 106.2309, 38.4872), + "新疆维吾尔自治区": ("CN-XJ", 87.6168, 43.8256), + "台湾省": ("CN-TW", 121.5654, 25.033), + "香港特别行政区": ("CN-HK", 114.1694, 22.3193), + "澳门特别行政区": ("CN-MO", 113.5439, 22.1987), +} + +CITY_CENTROIDS = { + "南京": (118.7969, 32.0603), + "南京市": (118.7969, 32.0603), + "San Jose": (-121.8863, 37.3382), + "South Holland": (4.493, 52.0208), + "Istanbul": (28.9784, 41.0082), +} + + +def resolve_geo_location_metadata(location: IpLocation) -> GeoLocationMetadata: + if location.location in {"局域网", "本机"}: + return GeoLocationMetadata( + country="", + country_code="PRIVATE", + region_code="PRIVATE", + longitude=None, + latitude=None, + accuracy_level="private", + ) + + country = COUNTRY_ALIASES.get(location.country, location.country) + country_code = COUNTRY_CODES.get(country, "") + + city_coordinate = CITY_CENTROIDS.get(location.city) + if city_coordinate: + return GeoLocationMetadata( + country=country, + country_code=country_code, + region_code="", + longitude=city_coordinate[0], + latitude=city_coordinate[1], + accuracy_level="city", + ) + + region_metadata = CHINA_REGION_METADATA.get(location.province) + if country in {"China", "中国"} and region_metadata: + region_code, longitude, latitude = region_metadata + return GeoLocationMetadata( + country="China", + country_code="CN", + region_code=region_code, + longitude=longitude, + latitude=latitude, + accuracy_level="region", + ) + + country_coordinate = COUNTRY_CENTROIDS.get(country) + if country_coordinate: + return GeoLocationMetadata( + country=country, + country_code=country_code, + region_code="", + longitude=country_coordinate[0], + latitude=country_coordinate[1], + accuracy_level="country", + ) + + return GeoLocationMetadata( + country=country, + country_code=country_code, + region_code="", + longitude=None, + latitude=None, + accuracy_level="unknown", + ) diff --git a/backend/app/services/ip_location.py b/backend/app/services/ip_location.py index 36a8ed28..7f61e0aa 100644 --- a/backend/app/services/ip_location.py +++ b/backend/app/services/ip_location.py @@ -9,6 +9,7 @@ import ipaddress import logging import sys from dataclasses import dataclass +from functools import lru_cache from pathlib import Path from typing import Optional @@ -125,5 +126,6 @@ class Ip2RegionResolver: _resolver = Ip2RegionResolver() +@lru_cache(maxsize=8192) def resolve_ip_location(ip: str | None) -> IpLocation: return _resolver.lookup(ip) diff --git a/backend/app/services/monitoring_retention.py b/backend/app/services/monitoring_retention.py new file mode 100644 index 00000000..fb13369d --- /dev/null +++ b/backend/app/services/monitoring_retention.py @@ -0,0 +1,151 @@ +"""监测数据留存清理任务。""" + +from __future__ import annotations + +import asyncio +import logging +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Any + +from sqlalchemy import delete + +from app.core.config import settings +from app.db.session import SessionLocal +from app.models.permission_access_log import PermissionAccessLog +from app.models.permission_metric_snapshot import PermissionMetricSnapshot +from app.models.security_access_log import SecurityAccessLog +from app.models.source_location_snapshot import SourceLocationSnapshot +from app.models.user_login_session import UserLoginSession + +logger = logging.getLogger("ctms.monitoring_retention") + + +@dataclass +class MonitoringRetentionState: + running: bool = False + last_run_started_at: datetime | None = None + last_success_at: datetime | None = None + last_error_at: datetime | None = None + last_error_type: str | None = None + error_count: int = 0 + last_deleted: dict[str, int] = field(default_factory=dict) + total_deleted: dict[str, int] = field(default_factory=dict) + + def snapshot(self) -> dict[str, Any]: + return { + "running": self.running, + "access_log_retention_days": settings.MONITORING_ACCESS_LOG_RETENTION_DAYS, + "metric_retention_days": settings.MONITORING_METRIC_RETENTION_DAYS, + "login_activity_retention_days": settings.USER_LOGIN_ACTIVITY_RETENTION_DAYS, + "interval_seconds": settings.MONITORING_RETENTION_INTERVAL_SECONDS, + "last_run_started_at": ( + self.last_run_started_at.isoformat() if self.last_run_started_at else None + ), + "last_success_at": self.last_success_at.isoformat() if self.last_success_at else None, + "last_error_at": self.last_error_at.isoformat() if self.last_error_at else None, + "last_error_type": self.last_error_type, + "error_count": self.error_count, + "last_deleted": dict(self.last_deleted), + "total_deleted": dict(self.total_deleted), + } + + +_state = MonitoringRetentionState() + + +def get_monitoring_retention_status() -> dict[str, Any]: + return _state.snapshot() + + +def _deleted_count(result: Any) -> int: + rowcount = getattr(result, "rowcount", 0) + return max(0, int(rowcount or 0)) + + +async def purge_expired_monitoring_data( + *, now: datetime | None = None +) -> dict[str, int]: + reference_time = now or datetime.now(timezone.utc) + access_cutoff = reference_time - timedelta( + days=settings.MONITORING_ACCESS_LOG_RETENTION_DAYS + ) + metric_cutoff = reference_time - timedelta( + days=settings.MONITORING_METRIC_RETENTION_DAYS + ) + login_activity_cutoff = reference_time - timedelta( + days=settings.USER_LOGIN_ACTIVITY_RETENTION_DAYS + ) + + async with SessionLocal() as session: + permission_result = await session.execute( + delete(PermissionAccessLog).where(PermissionAccessLog.created_at < access_cutoff) + ) + security_result = await session.execute( + delete(SecurityAccessLog).where(SecurityAccessLog.created_at < access_cutoff) + ) + metric_result = await session.execute( + delete(PermissionMetricSnapshot).where( + PermissionMetricSnapshot.bucket_time < metric_cutoff + ) + ) + source_location_result = await session.execute( + delete(SourceLocationSnapshot).where( + SourceLocationSnapshot.bucket_time < access_cutoff + ) + ) + login_session_result = await session.execute( + delete(UserLoginSession).where(UserLoginSession.last_seen_at < login_activity_cutoff) + ) + await session.commit() + + return { + "permission_access_logs": _deleted_count(permission_result), + "security_access_logs": _deleted_count(security_result), + "permission_metric_snapshots": _deleted_count(metric_result), + "source_location_snapshots": _deleted_count(source_location_result), + "user_login_sessions": _deleted_count(login_session_result), + } + + +async def run_monitoring_retention_once( + *, now: datetime | None = None +) -> dict[str, int]: + _state.last_run_started_at = now or datetime.now(timezone.utc) + try: + deleted = await purge_expired_monitoring_data(now=now) + except Exception as exc: + _state.last_error_at = datetime.now(timezone.utc) + _state.last_error_type = type(exc).__name__ + _state.error_count += 1 + raise + + _state.last_success_at = datetime.now(timezone.utc) + _state.last_deleted = deleted + for key, count in deleted.items(): + _state.total_deleted[key] = _state.total_deleted.get(key, 0) + count + return deleted + + +async def run_monitoring_retention(stop_event: asyncio.Event) -> None: + logger.info("Monitoring retention task started") + _state.running = True + try: + while not stop_event.is_set(): + try: + deleted = await run_monitoring_retention_once() + if any(deleted.values()): + logger.info("Purged expired monitoring data: %s", deleted) + except Exception: + logger.exception("Failed to purge expired monitoring data") + + try: + await asyncio.wait_for( + stop_event.wait(), + timeout=settings.MONITORING_RETENTION_INTERVAL_SECONDS, + ) + except asyncio.TimeoutError: + continue + finally: + _state.running = False + logger.info("Monitoring retention task stopped") diff --git a/backend/app/services/monitoring_server_location.py b/backend/app/services/monitoring_server_location.py new file mode 100644 index 00000000..5ffb60b7 --- /dev/null +++ b/backend/app/services/monitoring_server_location.py @@ -0,0 +1,164 @@ +"""Resolve the monitoring deployment's public network location. + +The map server marker is derived from the deployment's public IP instead of a +frontend or configuration coordinate. An explicit public IP is accepted for +restricted networks; otherwise the public frontend hostname and, finally, a +small set of public-IP discovery endpoints are used. Results are cached so the +monitoring API never performs per-request network discovery. +""" + +from __future__ import annotations + +import asyncio +import ipaddress +import logging +import socket +import time +from dataclasses import dataclass +from urllib.parse import urlparse + +import httpx + +from app.core.config import settings +from app.services.geo_location_metadata import resolve_geo_location_metadata +from app.services.ip_location import resolve_ip_location + +logger = logging.getLogger("ctms.monitoring_server_location") + + +@dataclass(frozen=True) +class MonitoringServerLocation: + name: str + longitude: float + latitude: float + accuracy_level: str + resolution_source: str + + def to_public_dict(self) -> dict: + return { + "name": self.name, + "longitude": self.longitude, + "latitude": self.latitude, + "accuracy_level": self.accuracy_level, + "resolution_source": self.resolution_source, + } + + +_cached_location: MonitoringServerLocation | None = None +_cache_initialized = False +_cache_expires_at = 0.0 +_cache_lock = asyncio.Lock() + + +def _normalize_public_ip(value: str | None) -> str | None: + candidate = (value or "").strip().splitlines()[0] if (value or "").strip() else "" + try: + address = ipaddress.ip_address(candidate) + except ValueError: + return None + return str(address) if address.is_global else None + + +async def _resolve_frontend_public_ip() -> str | None: + hostname = urlparse(settings.FRONTEND_PUBLIC_URL).hostname + if not hostname: + return None + literal_ip = _normalize_public_ip(hostname) + if literal_ip: + return literal_ip + try: + loop = asyncio.get_running_loop() + records = await loop.getaddrinfo(hostname, None, type=socket.SOCK_STREAM) + except (OSError, socket.gaierror): + return None + candidates = [] + for _, _, _, _, socket_address in records: + candidate = _normalize_public_ip(str(socket_address[0])) + if candidate and candidate not in candidates: + candidates.append(candidate) + return next((item for item in candidates if ":" not in item), candidates[0] if candidates else None) + + +async def _discover_public_ip() -> tuple[str | None, str]: + configured_ip = _normalize_public_ip(settings.MONITORING_SERVER_PUBLIC_IP) + if configured_ip: + return configured_ip, "configured_public_ip" + + frontend_ip = await _resolve_frontend_public_ip() + if frontend_ip: + return frontend_ip, "frontend_dns" + + if settings.ENV == "test": + return None, "unavailable" + + urls = [ + item.strip() + for item in settings.MONITORING_PUBLIC_IP_DISCOVERY_URLS.split(",") + if item.strip() + ] + timeout = httpx.Timeout(settings.MONITORING_PUBLIC_IP_DISCOVERY_TIMEOUT_SECONDS) + async with httpx.AsyncClient(timeout=timeout, follow_redirects=False) as client: + for url in urls: + try: + response = await client.get(url, headers={"Accept": "text/plain"}) + response.raise_for_status() + except httpx.HTTPError: + logger.warning("Public IP discovery endpoint unavailable", extra={"endpoint": url}) + continue + discovered_ip = _normalize_public_ip(response.text[:128]) + if discovered_ip: + return discovered_ip, "public_ip_discovery" + return None, "unavailable" + + +async def resolve_monitoring_server_location(*, force: bool = False) -> MonitoringServerLocation | None: + global _cached_location, _cache_initialized, _cache_expires_at + + now = time.monotonic() + if not force and _cache_initialized and now < _cache_expires_at: + return _cached_location + + async with _cache_lock: + now = time.monotonic() + if not force and _cache_initialized and now < _cache_expires_at: + return _cached_location + + public_ip, resolution_source = await _discover_public_ip() + location = None + if public_ip: + ip_location = resolve_ip_location(public_ip) + metadata = resolve_geo_location_metadata(ip_location) + if metadata.longitude is not None and metadata.latitude is not None: + country = metadata.country or ip_location.country + name = " / ".join( + part for part in [country, ip_location.province, ip_location.city] if part + ) or "公网服务器" + location = MonitoringServerLocation( + name=name, + longitude=metadata.longitude, + latitude=metadata.latitude, + accuracy_level=metadata.accuracy_level, + resolution_source=resolution_source, + ) + logger.info( + "Monitoring server location resolved", + extra={"resolution_source": resolution_source, "accuracy_level": metadata.accuracy_level}, + ) + else: + logger.warning("Deployment public IP has no usable geo coordinates") + else: + logger.warning("Unable to discover the deployment public IP") + + _cached_location = location + _cache_initialized = True + cache_seconds = settings.MONITORING_SERVER_LOCATION_CACHE_SECONDS if location else 300 + _cache_expires_at = now + cache_seconds + return location + + +def reset_monitoring_server_location_cache() -> None: + """Reset process-local state for configuration reloads and tests.""" + global _cached_location, _cache_initialized, _cache_expires_at + _cached_location = None + _cache_initialized = False + _cache_expires_at = 0.0 diff --git a/backend/app/services/permission_log_writer.py b/backend/app/services/permission_log_writer.py index 114afa93..0903d44e 100644 --- a/backend/app/services/permission_log_writer.py +++ b/backend/app/services/permission_log_writer.py @@ -13,6 +13,7 @@ import asyncio import logging import uuid from datetime import datetime, timezone +from typing import Any from app.db.session import SessionLocal from app.models.permission_access_log import PermissionAccessLog @@ -22,85 +23,150 @@ logger = logging.getLogger("ctms.permission_log_writer") BATCH_SIZE = 50 FLUSH_INTERVAL = 5.0 QUEUE_MAX_SIZE = 10000 +WRITE_ATTEMPTS = 2 +_QUEUE_STOP = object() class PermissionLogWriter: def __init__(self): - self._queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=QUEUE_MAX_SIZE) + self._queue: asyncio.Queue[dict | object] = asyncio.Queue(maxsize=QUEUE_MAX_SIZE) self._task: asyncio.Task | None = None + self._stopping = False + self._started_at: datetime | None = None + self._last_success_at: datetime | None = None + self._last_error_at: datetime | None = None + self._last_error_type: str | None = None + self._accepted_entries = 0 + self._written_entries = 0 + self._dropped_entries = 0 + self._failed_batches = 0 + self._failed_entries = 0 + self._retry_count = 0 def enqueue(self, entry: dict) -> None: + if self._stopping: + self._dropped_entries += 1 + logger.warning("Permission log writer is stopping, dropping entry") + return try: self._queue.put_nowait(entry) + self._accepted_entries += 1 except asyncio.QueueFull: + self._dropped_entries += 1 logger.warning("Permission log queue full, dropping entry") async def start(self) -> None: + if self._task and not self._task.done(): + return + self._stopping = False + self._started_at = datetime.now(timezone.utc) self._task = asyncio.create_task(self._flush_loop()) logger.info("PermissionLogWriter started") async def stop(self) -> None: - if self._task: - self._task.cancel() - try: - await self._task - except asyncio.CancelledError: - pass - await self._drain() + if self._task and not self._task.done(): + self._stopping = True + await self._queue.put(_QUEUE_STOP) + await self._task + self._task = None logger.info("PermissionLogWriter stopped") async def _flush_loop(self) -> None: while True: - batch = await self._collect_batch() + batch, should_stop = await self._collect_batch() if batch: await self._write_batch(batch) + if should_stop: + break - async def _collect_batch(self) -> list[dict]: + async def _collect_batch(self) -> tuple[list[dict], bool]: batch: list[dict] = [] try: first = await asyncio.wait_for(self._queue.get(), timeout=FLUSH_INTERVAL) + if first is _QUEUE_STOP: + return batch, True batch.append(first) except asyncio.TimeoutError: - return batch + return batch, False while len(batch) < BATCH_SIZE: try: item = self._queue.get_nowait() + if item is _QUEUE_STOP: + return batch, True batch.append(item) except asyncio.QueueEmpty: break - return batch + return batch, False - async def _write_batch(self, batch: list[dict]) -> None: - try: - async with SessionLocal() as session: - for entry in batch: - log = PermissionAccessLog( - id=uuid.uuid4(), - study_id=entry["study_id"], - user_id=entry["user_id"], - endpoint_key=entry["endpoint_key"], - role=entry["role"], - allowed=entry["allowed"], - elapsed_ms=entry["elapsed_ms"], - ip_address=entry.get("ip_address"), - created_at=entry.get("created_at", datetime.now(timezone.utc)), - ) - session.add(log) - await session.commit() - except Exception: - logger.exception("Failed to write permission access log batch (%d entries)", len(batch)) - - async def _drain(self) -> None: - batch: list[dict] = [] - while not self._queue.empty(): + async def _write_batch(self, batch: list[dict]) -> bool: + for attempt in range(WRITE_ATTEMPTS): try: - batch.append(self._queue.get_nowait()) - except asyncio.QueueEmpty: - break - if batch: - await self._write_batch(batch) + async with SessionLocal() as session: + for entry in batch: + log = PermissionAccessLog( + id=uuid.uuid4(), + study_id=entry["study_id"], + user_id=entry["user_id"], + endpoint_key=entry["endpoint_key"], + role=entry["role"], + allowed=entry["allowed"], + elapsed_ms=entry["elapsed_ms"], + ip_address=entry.get("ip_address"), + user_agent=entry.get("user_agent"), + client_type=entry.get("client_type"), + client_version=entry.get("client_version"), + client_platform=entry.get("client_platform"), + build_channel=entry.get("build_channel"), + build_commit=entry.get("build_commit"), + request_headers=entry.get("request_headers"), + request_snapshot=entry.get("request_snapshot"), + request_id=entry.get("request_id"), + created_at=entry.get("created_at", datetime.now(timezone.utc)), + ) + session.add(log) + await session.commit() + except Exception as exc: + self._last_error_at = datetime.now(timezone.utc) + self._last_error_type = type(exc).__name__ + if attempt + 1 < WRITE_ATTEMPTS: + self._retry_count += 1 + logger.warning( + "Permission access log batch write failed; retrying (%d entries)", + len(batch), + ) + await asyncio.sleep(0) + continue + self._failed_batches += 1 + self._failed_entries += len(batch) + logger.exception( + "Failed to write permission access log batch after retries (%d entries)", + len(batch), + ) + return False + self._written_entries += len(batch) + self._last_success_at = datetime.now(timezone.utc) + return True + return False + + def stats(self) -> dict[str, Any]: + return { + "running": bool(self._task and not self._task.done()), + "stopping": self._stopping, + "queue_size": self._queue.qsize(), + "queue_capacity": self._queue.maxsize, + "accepted_entries": self._accepted_entries, + "written_entries": self._written_entries, + "dropped_entries": self._dropped_entries, + "failed_batches": self._failed_batches, + "failed_entries": self._failed_entries, + "retry_count": self._retry_count, + "started_at": self._started_at.isoformat() if self._started_at else None, + "last_success_at": self._last_success_at.isoformat() if self._last_success_at else None, + "last_error_at": self._last_error_at.isoformat() if self._last_error_at else None, + "last_error_type": self._last_error_type, + } _writer: PermissionLogWriter | None = None diff --git a/backend/app/services/security_access_log_writer.py b/backend/app/services/security_access_log_writer.py index da25c487..b455b0a7 100644 --- a/backend/app/services/security_access_log_writer.py +++ b/backend/app/services/security_access_log_writer.py @@ -6,99 +6,170 @@ import asyncio import logging import uuid from datetime import datetime, timezone +from typing import Any from app.db.session import SessionLocal from app.models.security_access_log import SecurityAccessLog +from app.services.ip_location import resolve_ip_location +from app.services.security_events import classify_security_event logger = logging.getLogger("ctms.security_access_log_writer") BATCH_SIZE = 100 FLUSH_INTERVAL = 3.0 QUEUE_MAX_SIZE = 20000 +WRITE_ATTEMPTS = 2 +_QUEUE_STOP = object() class SecurityAccessLogWriter: def __init__(self) -> None: - self._queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=QUEUE_MAX_SIZE) + self._queue: asyncio.Queue[dict | object] = asyncio.Queue(maxsize=QUEUE_MAX_SIZE) self._task: asyncio.Task | None = None + self._stopping = False + self._started_at: datetime | None = None + self._last_success_at: datetime | None = None + self._last_error_at: datetime | None = None + self._last_error_type: str | None = None + self._accepted_entries = 0 + self._written_entries = 0 + self._dropped_entries = 0 + self._failed_batches = 0 + self._failed_entries = 0 + self._retry_count = 0 def enqueue(self, entry: dict) -> None: + if self._stopping: + self._dropped_entries += 1 + logger.warning("Security access log writer is stopping, dropping entry") + return try: self._queue.put_nowait(entry) + self._accepted_entries += 1 except asyncio.QueueFull: + self._dropped_entries += 1 logger.warning("Security access log queue full, dropping entry") async def start(self) -> None: + if self._task and not self._task.done(): + return + self._stopping = False + self._started_at = datetime.now(timezone.utc) self._task = asyncio.create_task(self._flush_loop()) logger.info("SecurityAccessLogWriter started") async def stop(self) -> None: - if self._task: - self._task.cancel() - try: - await self._task - except asyncio.CancelledError: - pass - await self._drain() + if self._task and not self._task.done(): + self._stopping = True + await self._queue.put(_QUEUE_STOP) + await self._task + self._task = None logger.info("SecurityAccessLogWriter stopped") async def _flush_loop(self) -> None: while True: - batch = await self._collect_batch() + batch, should_stop = await self._collect_batch() if batch: await self._write_batch(batch) + if should_stop: + break - async def _collect_batch(self) -> list[dict]: + async def _collect_batch(self) -> tuple[list[dict], bool]: batch: list[dict] = [] try: first = await asyncio.wait_for(self._queue.get(), timeout=FLUSH_INTERVAL) + if first is _QUEUE_STOP: + return batch, True batch.append(first) except asyncio.TimeoutError: - return batch + return batch, False while len(batch) < BATCH_SIZE: try: - batch.append(self._queue.get_nowait()) + item = self._queue.get_nowait() + if item is _QUEUE_STOP: + return batch, True + batch.append(item) except asyncio.QueueEmpty: break - return batch + return batch, False - async def _write_batch(self, batch: list[dict]) -> None: - try: - async with SessionLocal() as session: - for entry in batch: - session.add( - SecurityAccessLog( - id=uuid.uuid4(), - method=entry["method"], + async def _write_batch(self, batch: list[dict]) -> bool: + for attempt in range(WRITE_ATTEMPTS): + try: + async with SessionLocal() as session: + for entry in batch: + classification = classify_security_event( path=entry["path"], status_code=entry["status_code"], - elapsed_ms=entry["elapsed_ms"], - client_ip=entry.get("client_ip"), - user_agent=entry.get("user_agent"), - client_type=entry.get("client_type"), - client_version=entry.get("client_version"), - client_platform=entry.get("client_platform"), - build_channel=entry.get("build_channel"), - build_commit=entry.get("build_commit"), auth_status=entry["auth_status"], - user_identifier=entry.get("user_identifier"), - created_at=entry.get("created_at", datetime.now(timezone.utc)), + ip_location=resolve_ip_location(entry.get("client_ip")), ) + session.add( + SecurityAccessLog( + id=uuid.uuid4(), + method=entry["method"], + path=entry["path"], + status_code=entry["status_code"], + elapsed_ms=entry["elapsed_ms"], + client_ip=entry.get("client_ip"), + user_agent=entry.get("user_agent"), + client_type=entry.get("client_type"), + client_version=entry.get("client_version"), + client_platform=entry.get("client_platform"), + build_channel=entry.get("build_channel"), + build_commit=entry.get("build_commit"), + request_headers=entry.get("request_headers"), + request_snapshot=entry.get("request_snapshot"), + request_id=entry.get("request_id"), + category=classification["category"], + severity=classification["severity"], + auth_status=entry["auth_status"], + user_identifier=entry.get("user_identifier"), + created_at=entry.get("created_at", datetime.now(timezone.utc)), + ) + ) + await session.commit() + except Exception as exc: + self._last_error_at = datetime.now(timezone.utc) + self._last_error_type = type(exc).__name__ + if attempt + 1 < WRITE_ATTEMPTS: + self._retry_count += 1 + logger.warning( + "Security access log batch write failed; retrying (%d entries)", + len(batch), ) - await session.commit() - except Exception: - logger.exception("Failed to write security access log batch (%d entries)", len(batch)) + await asyncio.sleep(0) + continue + self._failed_batches += 1 + self._failed_entries += len(batch) + logger.exception( + "Failed to write security access log batch after retries (%d entries)", + len(batch), + ) + return False + self._written_entries += len(batch) + self._last_success_at = datetime.now(timezone.utc) + return True + return False - async def _drain(self) -> None: - batch: list[dict] = [] - while not self._queue.empty(): - try: - batch.append(self._queue.get_nowait()) - except asyncio.QueueEmpty: - break - if batch: - await self._write_batch(batch) + def stats(self) -> dict[str, Any]: + return { + "running": bool(self._task and not self._task.done()), + "stopping": self._stopping, + "queue_size": self._queue.qsize(), + "queue_capacity": self._queue.maxsize, + "accepted_entries": self._accepted_entries, + "written_entries": self._written_entries, + "dropped_entries": self._dropped_entries, + "failed_batches": self._failed_batches, + "failed_entries": self._failed_entries, + "retry_count": self._retry_count, + "started_at": self._started_at.isoformat() if self._started_at else None, + "last_success_at": self._last_success_at.isoformat() if self._last_success_at else None, + "last_error_at": self._last_error_at.isoformat() if self._last_error_at else None, + "last_error_type": self._last_error_type, + } _writer: SecurityAccessLogWriter | None = None diff --git a/backend/app/services/security_events.py b/backend/app/services/security_events.py new file mode 100644 index 00000000..ed79fbca --- /dev/null +++ b/backend/app/services/security_events.py @@ -0,0 +1,37 @@ +"""Security event classification shared by writers and monitoring APIs.""" + +from __future__ import annotations + +from app.services.ip_location import IpLocation + + +SENSITIVE_PROBE_MARKERS = ( + "/.env", + ".env", + "/.git", + ".git/config", + "backup", + "config.php", + "wp-config", + "database.yml", +) + +def classify_security_event( + *, + path: str, + status_code: int, + auth_status: str, + ip_location: IpLocation | None = None, +) -> dict[str, str]: + normalized_path = (path or "").lower() + if any(marker in normalized_path for marker in SENSITIVE_PROBE_MARKERS): + return {"category": "PROBE", "severity": "CRITICAL"} + if status_code >= 500: + return {"category": "SERVER_ERROR", "severity": "HIGH"} + if auth_status == "INVALID_TOKEN": + return {"category": "INVALID_TOKEN", "severity": "MEDIUM"} + if auth_status == "ANONYMOUS" and status_code in {401, 403}: + return {"category": "ANONYMOUS_API", "severity": "MEDIUM"} + if status_code == 404: + return {"category": "NOT_FOUND_NOISE", "severity": "LOW"} + return {"category": "OTHER", "severity": "LOW"} diff --git a/backend/app/services/source_location_aggregator.py b/backend/app/services/source_location_aggregator.py new file mode 100644 index 00000000..188581b1 --- /dev/null +++ b/backend/app/services/source_location_aggregator.py @@ -0,0 +1,272 @@ +"""Hourly source-location aggregation and timeline reads.""" + +from __future__ import annotations + +import asyncio +import hashlib +import hmac +import logging +import uuid +from collections import defaultdict +from datetime import datetime, timedelta, timezone +from typing import Literal + +from sqlalchemy import delete, func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import settings +from app.db.session import SessionLocal +from app.models.permission_access_log import PermissionAccessLog +from app.models.security_access_log import SecurityAccessLog +from app.models.source_location_snapshot import SourceLocationSnapshot +from app.services.geo_location_metadata import resolve_geo_location_metadata +from app.services.ip_location import resolve_ip_location + +logger = logging.getLogger("ctms.source_location_aggregator") + + +def _normalize_datetime(value: datetime) -> datetime: + return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc) + + +def _identity_hash(kind: str, value: str | None) -> str: + if not value: + return "" + payload = f"source-location:{kind}:{value}".encode("utf-8") + return hmac.new(settings.JWT_SECRET_KEY.encode("utf-8"), payload, hashlib.sha256).hexdigest() + + +async def aggregate_source_location_hour(bucket_start: datetime, bucket_end: datetime) -> int: + bucket_start = _normalize_datetime(bucket_start) + bucket_end = _normalize_datetime(bucket_end) + async with SessionLocal() as session: + matching_security_request = select(SecurityAccessLog.id).where( + PermissionAccessLog.request_id.is_not(None), + SecurityAccessLog.request_id == PermissionAccessLog.request_id, + ).exists() + permission_rows = ( + await session.execute( + select( + PermissionAccessLog.ip_address, + PermissionAccessLog.user_id, + func.count().filter(PermissionAccessLog.allowed.is_(True)).label("allowed_count"), + func.count().filter(PermissionAccessLog.allowed.is_(False)).label("denied_count"), + func.min(PermissionAccessLog.created_at).label("first_seen_at"), + func.max(PermissionAccessLog.created_at).label("last_seen_at"), + ) + .where( + PermissionAccessLog.created_at >= bucket_start, + PermissionAccessLog.created_at < bucket_end, + PermissionAccessLog.ip_address.is_not(None), + ~matching_security_request, + ) + .group_by(PermissionAccessLog.ip_address, PermissionAccessLog.user_id) + ) + ).all() + security_rows = ( + await session.execute( + select( + SecurityAccessLog.client_ip, + SecurityAccessLog.user_identifier, + func.count().filter(SecurityAccessLog.status_code < 400).label("allowed_count"), + func.count().filter(SecurityAccessLog.status_code >= 400).label("denied_count"), + func.count() + .filter( + SecurityAccessLog.category.is_not(None), + ~SecurityAccessLog.category.in_(("OTHER", "NOT_FOUND_NOISE")), + ) + .label("security_event_count"), + func.count() + .filter(SecurityAccessLog.severity.in_(("HIGH", "CRITICAL"))) + .label("high_risk_count"), + func.count() + .filter( + SecurityAccessLog.status_code >= 400, + SecurityAccessLog.auth_status.in_(("INVALID_TOKEN", "ANONYMOUS")), + ) + .label("auth_failure_count"), + func.min(SecurityAccessLog.created_at).label("first_seen_at"), + func.max(SecurityAccessLog.created_at).label("last_seen_at"), + ) + .where( + SecurityAccessLog.created_at >= bucket_start, + SecurityAccessLog.created_at < bucket_end, + SecurityAccessLog.client_ip.is_not(None), + ) + .group_by(SecurityAccessLog.client_ip, SecurityAccessLog.user_identifier) + ) + ).all() + + aggregates: dict[tuple[str, str], dict] = {} + + def merge_row( + ip_address: str, + user_identity: str, + allowed_count: int, + denied_count: int, + first_seen_at: datetime, + last_seen_at: datetime, + *, + security_event_count: int = 0, + high_risk_count: int = 0, + auth_failure_count: int = 0, + ) -> None: + key = (ip_address, user_identity) + row = aggregates.setdefault( + key, + { + "ip_address": ip_address, + "user_identity": user_identity, + "allowed_count": 0, + "denied_count": 0, + "security_event_count": 0, + "high_risk_count": 0, + "auth_failure_count": 0, + "first_seen_at": _normalize_datetime(first_seen_at), + "last_seen_at": _normalize_datetime(last_seen_at), + }, + ) + row["allowed_count"] += int(allowed_count or 0) + row["denied_count"] += int(denied_count or 0) + row["security_event_count"] += int(security_event_count or 0) + row["high_risk_count"] += int(high_risk_count or 0) + row["auth_failure_count"] += int(auth_failure_count or 0) + row["first_seen_at"] = min(row["first_seen_at"], _normalize_datetime(first_seen_at)) + row["last_seen_at"] = max(row["last_seen_at"], _normalize_datetime(last_seen_at)) + + for ip_address, user_id, allowed, denied, first_seen, last_seen in permission_rows: + merge_row(str(ip_address), str(user_id or ""), allowed, denied, first_seen, last_seen) + for ip_address, user_identifier, allowed, denied, security_events, high_risk, auth_failures, first_seen, last_seen in security_rows: + merge_row( + str(ip_address), + str(user_identifier or ""), + allowed, + denied, + first_seen, + last_seen, + security_event_count=security_events, + high_risk_count=high_risk, + auth_failure_count=auth_failures, + ) + + await session.execute( + delete(SourceLocationSnapshot).where(SourceLocationSnapshot.bucket_time == bucket_start) + ) + for row in aggregates.values(): + ip_info = resolve_ip_location(row["ip_address"]) + metadata = resolve_geo_location_metadata(ip_info) + longitude = metadata.longitude + latitude = metadata.latitude + country = metadata.country or ip_info.country + location = " / ".join( + part for part in [country, ip_info.province, ip_info.city] if part + ) or ip_info.location or "未知" + session.add( + SourceLocationSnapshot( + id=uuid.uuid4(), + bucket_time=bucket_start, + ip_hash=_identity_hash("ip", row["ip_address"]), + user_hash=_identity_hash("user", row["user_identity"]), + country=country, + country_code=metadata.country_code, + province=ip_info.province, + region_code=metadata.region_code, + city=ip_info.city, + isp=ip_info.isp, + location=location, + longitude=longitude, + latitude=latitude, + accuracy_level=metadata.accuracy_level, + allowed_count=row["allowed_count"], + denied_count=row["denied_count"], + security_event_count=row["security_event_count"], + high_risk_count=row["high_risk_count"], + auth_failure_count=row["auth_failure_count"], + first_seen_at=row["first_seen_at"], + last_seen_at=row["last_seen_at"], + ) + ) + await session.commit() + return len(aggregates) + + +async def get_source_location_timeline( + db: AsyncSession, + *, + start_at: datetime, + end_at: datetime, + granularity: Literal["hour", "day"], +) -> list[dict]: + rows = ( + await db.execute( + select(SourceLocationSnapshot) + .where( + SourceLocationSnapshot.bucket_time >= start_at, + SourceLocationSnapshot.bucket_time < end_at, + ) + .order_by(SourceLocationSnapshot.bucket_time) + ) + ).scalars().all() + buckets: dict[datetime, dict] = defaultdict( + lambda: { + "allowed_count": 0, + "denied_count": 0, + "security_event_count": 0, + "high_risk_count": 0, + "ip_hashes": set(), + "user_hashes": set(), + } + ) + for row in rows: + bucket = _normalize_datetime(row.bucket_time) + if granularity == "day": + bucket = bucket.replace(hour=0, minute=0, second=0, microsecond=0) + else: + bucket = bucket.replace(minute=0, second=0, microsecond=0) + item = buckets[bucket] + item["allowed_count"] += row.allowed_count + item["denied_count"] += row.denied_count + item["security_event_count"] += row.security_event_count + item["high_risk_count"] += row.high_risk_count + item["ip_hashes"].add(row.ip_hash) + if row.user_hash: + item["user_hashes"].add(row.user_hash) + + return [ + { + "bucket_time": bucket.isoformat(), + "total_count": item["allowed_count"] + item["denied_count"], + "allowed_count": item["allowed_count"], + "denied_count": item["denied_count"], + "security_event_count": item["security_event_count"], + "high_risk_count": item["high_risk_count"], + "unique_ip_count": len(item["ip_hashes"]), + "unique_user_count": len(item["user_hashes"]), + } + for bucket, item in sorted(buckets.items()) + ] + + +async def run_hourly_source_location_aggregation(stop_event: asyncio.Event) -> None: + logger.info("Source location aggregator started") + now = datetime.now(timezone.utc) + completed_hour = now.replace(minute=0, second=0, microsecond=0) + try: + await aggregate_source_location_hour(completed_hour - timedelta(hours=1), completed_hour) + except Exception: + logger.exception("Failed to backfill source location snapshot for %s", completed_hour) + + while not stop_event.is_set(): + now = datetime.now(timezone.utc) + next_hour = now.replace(minute=0, second=0, microsecond=0) + timedelta(hours=1) + try: + await asyncio.wait_for(stop_event.wait(), timeout=(next_hour - now).total_seconds()) + break + except asyncio.TimeoutError: + pass + try: + await aggregate_source_location_hour(next_hour - timedelta(hours=1), next_hour) + except Exception: + logger.exception("Failed to aggregate source locations for %s", next_hour) + + logger.info("Source location aggregator stopped") diff --git a/backend/app/services/user_login_sessions.py b/backend/app/services/user_login_sessions.py new file mode 100644 index 00000000..924b6d98 --- /dev/null +++ b/backend/app/services/user_login_sessions.py @@ -0,0 +1,198 @@ +"""Server-authoritative login session activity for the admin account list.""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Any, Iterable + +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import settings +from app.models.user_login_session import UserLoginSession + + +@dataclass(frozen=True) +class UserLoginSummary: + status: str = "OFFLINE" + last_login_at: datetime | None = None + last_seen_at: datetime | None = None + client_type: str | None = None + active_session_count: int = 0 + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _as_utc(value: datetime) -> datetime: + return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc) + + +def _text_header(headers: Any, name: str, limit: int) -> str | None: + value = (headers.get(name) or "").strip() + return value[:limit] or None + + +def session_id_from_payload(payload: dict[str, Any]) -> uuid.UUID: + raw_session_id = payload.get("sid") + try: + return uuid.UUID(str(raw_session_id)) + except (TypeError, ValueError): + # Tokens issued before session tracking are mapped to a deterministic + # legacy session without persisting the token or a token-derived value. + legacy_key = ":".join( + [ + str(payload.get("sub") or ""), + str(payload.get("orig_iat") or payload.get("iat") or ""), + str(payload.get("client_type") or "web"), + ] + ) + return uuid.uuid5(uuid.NAMESPACE_URL, f"ctms:legacy-session:{legacy_key}") + + +def session_client_type(payload: dict[str, Any], headers: Any) -> str: + from_payload = (payload.get("client_type") or "").strip().lower() + from_header = (headers.get("x-ctms-client-type") or "").strip().lower() + return "desktop" if "desktop" in {from_payload, from_header} else "web" + + +async def create_login_session( + db: AsyncSession, + *, + session_id: uuid.UUID, + user_id: uuid.UUID, + request: Any, + login_at: datetime | None = None, +) -> UserLoginSession: + occurred_at = login_at or _now() + session = UserLoginSession( + id=session_id, + user_id=user_id, + client_type=session_client_type({}, request.headers), + client_platform=_text_header(request.headers, "x-ctms-client-platform", 32), + client_version=_text_header(request.headers, "x-ctms-client-version", 64), + client_source=_text_header(request.headers, "x-ctms-client-source", 32), + login_at=occurred_at, + last_seen_at=occurred_at, + ) + db.add(session) + await db.commit() + await db.refresh(session) + return session + + +async def touch_login_session( + db: AsyncSession, + *, + user_id: uuid.UUID, + payload: dict[str, Any], + request: Any, +) -> UserLoginSession | None: + session_id = session_id_from_payload(payload) + session = await db.get(UserLoginSession, session_id) + now = _now() + if session is None: + session = UserLoginSession( + id=session_id, + user_id=user_id, + client_type=session_client_type(payload, request.headers), + client_platform=_text_header(request.headers, "x-ctms-client-platform", 32), + client_version=_text_header(request.headers, "x-ctms-client-version", 64), + client_source=_text_header(request.headers, "x-ctms-client-source", 32), + login_at=now, + last_seen_at=now, + ) + db.add(session) + elif session.user_id != user_id or session.ended_at is not None: + return None + else: + session.last_seen_at = now + session.client_platform = _text_header(request.headers, "x-ctms-client-platform", 32) or session.client_platform + session.client_version = _text_header(request.headers, "x-ctms-client-version", 64) or session.client_version + await db.commit() + await db.refresh(session) + return session + + +async def end_login_session( + db: AsyncSession, + *, + user_id: uuid.UUID, + payload: dict[str, Any], + reason: str = "logout", +) -> bool: + session_id = session_id_from_payload(payload) + result = await db.execute( + update(UserLoginSession) + .where( + UserLoginSession.id == session_id, + UserLoginSession.user_id == user_id, + UserLoginSession.ended_at.is_(None), + ) + .values(ended_at=_now(), end_reason=reason, last_seen_at=_now()) + ) + await db.commit() + return bool(result.rowcount) + + +async def get_login_summaries( + db: AsyncSession, + user_ids: Iterable[uuid.UUID], +) -> dict[uuid.UUID, UserLoginSummary]: + ids = list(user_ids) + if not ids: + return {} + rows = ( + await db.execute( + select(UserLoginSession) + .where(UserLoginSession.user_id.in_(ids)) + .order_by(UserLoginSession.user_id, UserLoginSession.login_at.desc()) + ) + ).scalars().all() + cutoff = _now() - timedelta(seconds=settings.USER_SESSION_ONLINE_SECONDS) + summaries: dict[uuid.UUID, UserLoginSummary] = {} + mutable: dict[uuid.UUID, dict[str, Any]] = {} + for row in rows: + current = mutable.setdefault( + row.user_id, + { + "last_login_at": row.login_at, + "last_seen_at": row.last_seen_at, + "client_type": row.client_type, + "active_session_count": 0, + }, + ) + row_last_seen_at = _as_utc(row.last_seen_at) + if row_last_seen_at and ( + current["last_seen_at"] is None or row_last_seen_at > _as_utc(current["last_seen_at"]) + ): + current["last_seen_at"] = row_last_seen_at + if row.ended_at is None and row_last_seen_at >= cutoff: + current["active_session_count"] += 1 + for user_id, item in mutable.items(): + summaries[user_id] = UserLoginSummary( + status="ONLINE" if item["active_session_count"] else "OFFLINE", + last_login_at=item["last_login_at"], + last_seen_at=item["last_seen_at"], + client_type=item["client_type"], + active_session_count=item["active_session_count"], + ) + return summaries + + +async def list_login_activities( + db: AsyncSession, + *, + user_id: uuid.UUID, + limit: int, +) -> list[UserLoginSession]: + result = await db.execute( + select(UserLoginSession) + .where(UserLoginSession.user_id == user_id) + .order_by(UserLoginSession.login_at.desc()) + .limit(limit) + ) + return list(result.scalars().all()) diff --git a/backend/tests/test_admin_pm_permissions.py b/backend/tests/test_admin_pm_permissions.py index dc0e444d..863d50c3 100644 --- a/backend/tests/test_admin_pm_permissions.py +++ b/backend/tests/test_admin_pm_permissions.py @@ -246,7 +246,7 @@ async def test_project_member_can_read_own_effective_permissions(db_session: Asy @pytest.mark.asyncio -async def test_project_pm_monitoring_scope_is_limited_to_own_projects(db_session: AsyncSession): +async def test_project_pm_cannot_access_system_monitoring(db_session: AsyncSession): pm_id = uuid.uuid4() own_study_id = uuid.uuid4() other_study_id = uuid.uuid4() @@ -256,12 +256,10 @@ async def test_project_pm_monitoring_scope_is_limited_to_own_projects(db_session await _seed_member(db_session, own_study_id, pm_id, "PM") await db_session.commit() - scope = await resolve_monitoring_scope(db_session, UserStub(id=pm_id)) + with pytest.raises(HTTPException) as exc_info: + await resolve_monitoring_scope(db_session, UserStub(id=pm_id)) - assert scope.is_admin is False - assert scope.study_ids == {own_study_id} - assert scope.can_access_study(own_study_id) - assert not scope.can_access_study(other_study_id) + assert exc_info.value.status_code == 403 @pytest.mark.asyncio diff --git a/backend/tests/test_app_startup.py b/backend/tests/test_app_startup.py index e43a35e8..c9215353 100644 --- a/backend/tests/test_app_startup.py +++ b/backend/tests/test_app_startup.py @@ -54,6 +54,9 @@ class _DummyTask: def cancel(self): self.cancelled = True + def done(self): + return False + def __await__(self): async def _wait(): self.awaited = True @@ -106,6 +109,7 @@ async def test_development_startup_does_not_seed_business_data(monkeypatch): monkeypatch.setattr(main.settings, "ENV", "development") monkeypatch.setattr(main, "ensure_admin_exists", ensure_admin_exists) monkeypatch.setattr(main, "run_daily_lost_visit_job", fake_scheduler) + monkeypatch.setattr(main, "resolve_monitoring_server_location", AsyncMock(return_value=None)) monkeypatch.setattr(main.asyncio, "create_task", fake_create_task) monkeypatch.setattr(main, "engine", _DummyEngine()) monkeypatch.setattr(main, "SessionLocal", fake_session_local) diff --git a/backend/tests/test_audit_access_context.py b/backend/tests/test_audit_access_context.py index 98a3a751..61c9d09a 100644 --- a/backend/tests/test_audit_access_context.py +++ b/backend/tests/test_audit_access_context.py @@ -4,6 +4,7 @@ import pytest from app.core.request_context import ( RequestAuditContext, + build_request_audit_context, reset_request_audit_context, set_request_audit_context, ) @@ -22,6 +23,48 @@ class FakeAsyncSession: self.flushed = True +class FakeClient: + def __init__(self, host: str = "10.0.0.8") -> None: + self.host = host + self.port = 54321 + + +class FakeUrl: + scheme = "https" + path = "/api/v1/audit" + query = "token=secret-token&view=all&subject_name=Alice" + + +class FakeQueryParams: + def __init__(self) -> None: + self._items = [ + ("token", "secret-token"), + ("view", "all"), + ("subject_name", "Alice"), + ] + + def __bool__(self) -> bool: + return True + + def multi_items(self): + return list(self._items) + + +class FakeRequest: + def __init__(self, headers: dict[str, str], *, client_host: str = "10.0.0.8") -> None: + self.headers = headers + self.client = FakeClient(client_host) + self.method = "GET" + self.url = FakeUrl() + self.query_params = FakeQueryParams() + self.scope = { + "http_version": "1.1", + "scheme": "https", + "path": "/api/v1/audit", + "method": "GET", + } + + @pytest.mark.asyncio async def test_audit_log_action_captures_request_access_context(): context_token = set_request_audit_context( @@ -60,3 +103,105 @@ async def test_audit_log_action_captures_request_access_context(): assert log.client_platform == "macos" assert log.build_channel == "release" assert log.build_commit == "abcdef1" + + +def test_request_audit_context_captures_sanitized_headers(): + context = build_request_audit_context( + FakeRequest( + { + "user-agent": "Chrome", + "referer": "https://ctms.local/audit?token=secret-token", + "authorization": "Bearer secret-token", + "cookie": "sid=secret", + "x-ctms-client-source": "ctms-web", + "x-ctms-client-type": "web", + "x-ctms-client-version": "0.1.0", + "x-request-id": "req-123", + "x-custom-secret": "hidden", + "x-random-debug": "skip-me", + } + ) + ) + + assert context.user_agent == "Chrome" + assert context.client_type == "web" + assert context.client_ip == "10.0.0.8" + assert uuid.UUID(context.request_id) + assert context.request_headers == { + "authorization": "[redacted]", + "cookie": "[redacted]", + "user-agent": "Chrome", + "x-ctms-client-source": "ctms-web", + "x-ctms-client-type": "web", + "x-ctms-client-version": "0.1.0", + "x-custom-secret": "[redacted]", + "x-request-id": "req-123", + } + assert context.request_snapshot is not None + assert context.request_snapshot["method"] == "GET" + assert context.request_snapshot["request_id"] == context.request_id + assert context.request_snapshot["path"] == "/api/v1/audit" + assert context.request_snapshot["query_string"] == ( + "token=[redacted]&view=all&subject_name=[redacted]" + ) + assert context.request_snapshot["query_params"] == [ + {"name": "token", "value": "[redacted]"}, + {"name": "view", "value": "all"}, + {"name": "subject_name", "value": "[redacted]"}, + ] + assert context.request_snapshot["headers"]["authorization"] == "[redacted]" + assert context.request_snapshot["headers"]["cookie"] == "[redacted]" + assert context.request_snapshot["headers"]["x-ctms-client-source"] == "ctms-web" + assert "referer" not in context.request_snapshot["headers"] + assert "x-random-debug" not in context.request_snapshot["headers"] + assert context.request_snapshot["client"] == {"host": "10.0.0.8", "port": 54321} + assert context.request_snapshot["body"] == { + "captured": False, + "reason": "body_not_captured_by_audit_policy", + } + + +def test_request_audit_context_prefers_explicit_ctms_client_source(): + context = build_request_audit_context( + FakeRequest( + { + "x-ctms-client-source": "ctms-desktop", + "x-ctms-client-type": "web", + } + ) + ) + + assert context.client_type == "desktop" + assert context.request_headers == { + "x-ctms-client-source": "ctms-desktop", + "x-ctms-client-type": "web", + } + + +def test_request_audit_context_uses_proxy_observed_ip_instead_of_spoofed_forwarded_prefix(monkeypatch): + monkeypatch.setattr("app.core.request_context.settings.TRUSTED_PROXY_CIDRS", "10.0.0.0/8") + context = build_request_audit_context( + FakeRequest( + { + "x-real-ip": "203.0.113.20", + "x-forwarded-for": "1.2.3.4, 203.0.113.20", + } + ) + ) + + assert context.client_ip == "203.0.113.20" + + +def test_request_audit_context_ignores_forwarded_headers_from_untrusted_peer(monkeypatch): + monkeypatch.setattr("app.core.request_context.settings.TRUSTED_PROXY_CIDRS", "10.0.0.0/8") + context = build_request_audit_context( + FakeRequest( + { + "x-real-ip": "203.0.113.20", + "x-forwarded-for": "1.2.3.4", + }, + client_host="198.51.100.8", + ) + ) + + assert context.client_ip == "198.51.100.8" diff --git a/backend/tests/test_monitoring_runtime.py b/backend/tests/test_monitoring_runtime.py new file mode 100644 index 00000000..3e76afb7 --- /dev/null +++ b/backend/tests/test_monitoring_runtime.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import asyncio +import uuid +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from app.models.permission_access_log import PermissionAccessLog +from app.models.permission_metric_snapshot import PermissionMetricSnapshot +from app.models.security_access_log import SecurityAccessLog +from app.models.study import Study +from app.models.user import User, UserStatus +from app.services import monitoring_retention +from app.services import permission_log_writer, security_access_log_writer + + +class _FakeSession: + def __init__(self) -> None: + self.added: list[object] = [] + self.committed = False + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + def add(self, value: object) -> None: + self.added.append(value) + + async def commit(self) -> None: + self.committed = True + + +@pytest.mark.asyncio +async def test_permission_writer_gracefully_flushes_and_reports_stats(monkeypatch): + session = _FakeSession() + monkeypatch.setattr(permission_log_writer, "SessionLocal", lambda: session) + writer = permission_log_writer.PermissionLogWriter() + await writer.start() + writer.enqueue( + { + "study_id": uuid.uuid4(), + "user_id": uuid.uuid4(), + "endpoint_key": "subjects.read", + "role": "PM", + "allowed": True, + "elapsed_ms": 2.5, + } + ) + + await writer.stop() + + stats = writer.stats() + assert session.committed is True + assert len(session.added) == 1 + assert stats["running"] is False + assert stats["accepted_entries"] == 1 + assert stats["written_entries"] == 1 + assert stats["dropped_entries"] == 0 + assert stats["queue_size"] == 0 + assert stats["last_success_at"] is not None + + +@pytest.mark.asyncio +async def test_permission_writer_retries_and_exposes_permanent_failure(monkeypatch): + class _FailingSession: + async def __aenter__(self): + raise RuntimeError("database unavailable") + + async def __aexit__(self, exc_type, exc, tb): + return False + + monkeypatch.setattr(permission_log_writer, "SessionLocal", _FailingSession) + writer = permission_log_writer.PermissionLogWriter() + batch = [ + { + "study_id": uuid.uuid4(), + "user_id": uuid.uuid4(), + "endpoint_key": "subjects.read", + "role": "PM", + "allowed": True, + "elapsed_ms": 2.5, + } + ] + + assert await writer._write_batch(batch) is False + + stats = writer.stats() + assert stats["retry_count"] == 1 + assert stats["failed_batches"] == 1 + assert stats["failed_entries"] == 1 + assert stats["last_error_type"] == "RuntimeError" + + +def test_writer_queue_overflow_is_counted(): + writer = security_access_log_writer.SecurityAccessLogWriter() + writer._queue = asyncio.Queue(maxsize=1) + + writer.enqueue({"path": "/first"}) + writer.enqueue({"path": "/dropped"}) + + stats = writer.stats() + assert stats["accepted_entries"] == 1 + assert stats["dropped_entries"] == 1 + assert stats["queue_size"] == 1 + + +@pytest.mark.asyncio +async def test_retention_deletes_expired_rows_and_preserves_recent_rows( + db_session, test_engine, monkeypatch +): + now = datetime(2026, 7, 10, tzinfo=timezone.utc) + study_id = (await db_session.execute(select(Study.id).limit(1))).scalar_one() + user = User( + email=f"retention-{uuid.uuid4()}@example.com", + password_hash="hash", + full_name="Retention Test", + clinical_department="QA", + status=UserStatus.ACTIVE, + ) + db_session.add(user) + await db_session.flush() + + old_permission = PermissionAccessLog( + study_id=study_id, + user_id=user.id, + endpoint_key="old", + role="PM", + allowed=True, + elapsed_ms=1, + created_at=now - timedelta(days=91), + ) + recent_permission = PermissionAccessLog( + study_id=study_id, + user_id=user.id, + endpoint_key="recent", + role="PM", + allowed=True, + elapsed_ms=1, + created_at=now - timedelta(days=89), + ) + old_security = SecurityAccessLog( + method="GET", + path="/old", + status_code=404, + elapsed_ms=1, + auth_status="ANONYMOUS", + created_at=now - timedelta(days=91), + ) + recent_security = SecurityAccessLog( + method="GET", + path="/recent", + status_code=200, + elapsed_ms=1, + auth_status="AUTHENTICATED", + created_at=now - timedelta(days=89), + ) + old_metric = PermissionMetricSnapshot( + bucket_time=now - timedelta(days=401) + ) + recent_metric = PermissionMetricSnapshot( + bucket_time=now - timedelta(days=399) + ) + db_session.add_all( + [ + old_permission, + recent_permission, + old_security, + recent_security, + old_metric, + recent_metric, + ] + ) + await db_session.commit() + old_permission_id = old_permission.id + recent_permission_id = recent_permission.id + old_security_id = old_security.id + recent_security_id = recent_security.id + old_metric_id = old_metric.id + recent_metric_id = recent_metric.id + + session_factory = async_sessionmaker( + bind=test_engine, + class_=AsyncSession, + expire_on_commit=False, + ) + monkeypatch.setattr(monitoring_retention, "SessionLocal", session_factory) + monkeypatch.setattr( + monitoring_retention.settings, "MONITORING_ACCESS_LOG_RETENTION_DAYS", 90 + ) + monkeypatch.setattr( + monitoring_retention.settings, "MONITORING_METRIC_RETENTION_DAYS", 400 + ) + + deleted = await monitoring_retention.purge_expired_monitoring_data(now=now) + + db_session.expire_all() + assert deleted["permission_access_logs"] >= 1 + assert deleted["security_access_logs"] >= 1 + assert deleted["permission_metric_snapshots"] >= 1 + assert await db_session.get(PermissionAccessLog, old_permission_id) is None + assert await db_session.get(SecurityAccessLog, old_security_id) is None + assert await db_session.get(PermissionMetricSnapshot, old_metric_id) is None + assert await db_session.get(PermissionAccessLog, recent_permission_id) is not None + assert await db_session.get(SecurityAccessLog, recent_security_id) is not None + assert await db_session.get(PermissionMetricSnapshot, recent_metric_id) is not None diff --git a/backend/tests/test_monitoring_server_location.py b/backend/tests/test_monitoring_server_location.py new file mode 100644 index 00000000..df78c80d --- /dev/null +++ b/backend/tests/test_monitoring_server_location.py @@ -0,0 +1,53 @@ +from types import SimpleNamespace + +import pytest + +from app.core.config import settings +from app.services import monitoring_server_location + + +def test_public_ip_normalization_rejects_private_addresses(): + assert monitoring_server_location._normalize_public_ip("8.8.8.8\n") == "8.8.8.8" + assert monitoring_server_location._normalize_public_ip("10.0.0.8") is None + assert monitoring_server_location._normalize_public_ip("127.0.0.1") is None + assert monitoring_server_location._normalize_public_ip("not-an-ip") is None + + +@pytest.mark.asyncio +async def test_server_location_is_derived_from_configured_public_ip(monkeypatch): + monkeypatch.setattr(settings, "MONITORING_SERVER_PUBLIC_IP", "8.8.8.8") + monkeypatch.setattr( + monitoring_server_location, + "resolve_ip_location", + lambda ip: SimpleNamespace( + location="United States / California / San Jose", + country="United States", + province="California", + city="San Jose", + isp="Google", + ), + ) + monitoring_server_location.reset_monitoring_server_location_cache() + + location = await monitoring_server_location.resolve_monitoring_server_location() + + assert location is not None + assert location.name == "United States / California / San Jose" + assert location.longitude == -121.8863 + assert location.latitude == 37.3382 + assert location.resolution_source == "configured_public_ip" + monitoring_server_location.reset_monitoring_server_location_cache() + + +@pytest.mark.asyncio +async def test_server_location_returns_none_instead_of_a_hardcoded_fallback(monkeypatch): + async def no_public_ip(): + return None, "unavailable" + + monkeypatch.setattr(monitoring_server_location, "_discover_public_ip", no_public_ip) + monitoring_server_location.reset_monitoring_server_location_cache() + + location = await monitoring_server_location.resolve_monitoring_server_location() + + assert location is None + monitoring_server_location.reset_monitoring_server_location_cache() diff --git a/backend/tests/test_permission_monitoring_api.py b/backend/tests/test_permission_monitoring_api.py index ed2fd85d..4481085c 100644 --- a/backend/tests/test_permission_monitoring_api.py +++ b/backend/tests/test_permission_monitoring_api.py @@ -1,7 +1,9 @@ """监控API测试：权限系统监控API端点验证。""" import inspect +import json import uuid +from types import SimpleNamespace import pytest from fastapi import HTTPException @@ -10,6 +12,7 @@ from sqlalchemy import text from app.core.permission_monitor import set_permission_monitor, PermissionMonitor from app.api.v1 import permission_monitoring from app.api.v1.system_permissions import list_system_permissions +from app.services.monitoring_server_location import MonitoringServerLocation class FakeIpInfo: @@ -30,6 +33,18 @@ class ProjectPmUserStub: is_admin = False +@pytest.fixture(autouse=True) +def disable_server_public_ip_discovery(monkeypatch): + async def no_server_location(): + return None + + monkeypatch.setattr( + permission_monitoring, + "resolve_monitoring_server_location", + no_server_location, + ) + + @pytest.mark.asyncio async def test_resolve_monitoring_scope_rejects_project_pm(db_session): """系统监测模块仅允许系统管理员访问，项目 PM 不再具备监测范围。""" @@ -103,9 +118,11 @@ async def _seed_permission_log(db_session, study_id: uuid.UUID, user_id: uuid.UU text( """ INSERT INTO permission_access_logs - (id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, created_at) + (id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, + request_snapshot, created_at) VALUES - (:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address, CURRENT_TIMESTAMP) + (:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address, + :request_snapshot, CURRENT_TIMESTAMP) """ ), { @@ -117,6 +134,7 @@ async def _seed_permission_log(db_session, study_id: uuid.UUID, user_id: uuid.UU "allowed": allowed, "elapsed_ms": elapsed_ms, "ip_address": "127.0.0.1", + "request_snapshot": None, }, ) await db_session.commit() @@ -262,8 +280,8 @@ async def test_permission_system_health_degraded(db_session): assert any("权限拒绝率偏高" in issue for issue in data["issues"]) assert any("缓存命中率偏低" in issue for issue in data["issues"]) assert data["status"] == "unhealthy" - assert data["health_score"] == 30 - assert data["score_breakdown"]["performance"]["deduction"] == 25 + assert data["health_score"] == 35 + assert data["score_breakdown"]["performance"]["deduction"] == 20 assert data["score_breakdown"]["deny_rate"]["deduction"] == 20 assert data["score_breakdown"]["cache"]["deduction"] == 25 @@ -279,6 +297,10 @@ async def test_permission_system_health_includes_metrics(db_session): assert "cache_stats" in data assert "score_breakdown" in data assert "sample_sufficient" in data + assert "components" in data + assert "storage" in data + assert "runtime" in data["score_breakdown"] + assert data["components"]["database"]["status"] == "healthy" assert "total_checks" in data["last_hour"] assert "cache_metrics" in data["cache_stats"] @@ -304,6 +326,41 @@ async def test_permission_system_health_ignores_small_cache_sample(db_session): assert data["health_score"] == 100 - non_cache_deductions +@pytest.mark.asyncio +async def test_permission_system_health_reports_writer_degradation(db_session, monkeypatch): + monitor = PermissionMonitor() + set_permission_monitor(monitor) + writer = SimpleNamespace( + stats=lambda: { + "running": True, + "queue_size": 3, + "queue_capacity": 100, + "failed_batches": 0, + "dropped_entries": 2, + } + ) + monkeypatch.setattr(permission_monitoring, "get_log_writer", lambda: writer) + monkeypatch.setattr(permission_monitoring, "get_security_log_writer", lambda: None) + monkeypatch.setattr( + permission_monitoring, + "get_monitoring_retention_status", + lambda: { + "running": False, + "last_run_started_at": None, + "last_success_at": None, + "last_error_at": None, + }, + ) + + data = await permission_monitoring.permission_system_health( + db=db_session, _=AdminUserStub() + ) + + assert data["components"]["permission_log_writer"]["status"] == "degraded" + assert data["score_breakdown"]["runtime"]["deduction"] == 10 + assert any("权限日志写入器" in issue for issue in data["issues"]) + + @pytest.mark.asyncio async def test_ip_locations_counts_unique_users_per_location(db_session, monkeypatch): """IP 属地统计应按省市聚合访问次数、来源 IP 数和访问用户数。""" @@ -381,10 +438,26 @@ async def test_ip_locations_counts_unique_users_per_location(db_session, monkeyp ) await db_session.commit() + resolved_ips = [] + + def fake_resolve_ip_location(ip): + resolved_ips.append(ip) + return FakeIpInfo("广东省", "深圳市") + + async def fake_resolve_monitoring_server_location(): + return MonitoringServerLocation( + name="China / 重庆 / 重庆市", + longitude=106.5516, + latitude=29.563, + accuracy_level="city", + resolution_source="public_ip_discovery", + ) + + monkeypatch.setattr(permission_monitoring, "resolve_ip_location", fake_resolve_ip_location) monkeypatch.setattr( permission_monitoring, - "resolve_ip_location", - lambda ip: FakeIpInfo("广东省", "深圳市"), + "resolve_monitoring_server_location", + fake_resolve_monitoring_server_location, ) result = await permission_monitoring.get_ip_locations(db=db_session, _=AdminUserStub(), days=7, limit=10) @@ -396,6 +469,56 @@ async def test_ip_locations_counts_unique_users_per_location(db_session, monkeyp assert result["summary"]["total_count"] == 3 assert result["summary"]["unique_ip_count"] == 2 assert result["summary"]["unique_user_count"] == 2 + assert result["items"][0]["country_code"] == "CN" + assert result["items"][0]["region_code"] == "CN-GD" + assert result["items"][0]["longitude"] is not None + assert result["items"][0]["risk_level"] == "attention" + assert result["period"]["mode"] == "rolling" + assert result["period"]["retention_days"] >= 7 + assert result["data_quality"]["located_ip_count"] == 2 + assert result["server_location"]["longitude"] == 106.5516 + assert result["server_location"]["resolution_source"] == "public_ip_discovery" + assert sorted(resolved_ips) == ["10.1.1.1", "10.1.1.2"] + + await db_session.execute( + text( + """ + INSERT INTO permission_access_logs + (id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, created_at) + VALUES + (lower(hex(randomblob(4))) || '-' || lower(hex(randomblob(2))) || '-4' || + substr(lower(hex(randomblob(2))),2) || '-' || + substr('89ab', abs(random()) % 4 + 1, 1) || + substr(lower(hex(randomblob(2))),2) || '-' || lower(hex(randomblob(6))), + :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address, + datetime('now', '-120 days')) + """ + ), + { + "study_id": study_id, + "user_id": user_a, + "endpoint_key": "admin.permissions.read", + "role": "PM", + "allowed": True, + "elapsed_ms": 2.4, + "ip_address": "10.1.1.3", + }, + ) + await db_session.commit() + + resolved_ips.clear() + all_result = await permission_monitoring.get_ip_locations( + db=db_session, + _=AdminUserStub(), + days=7, + limit=10, + all_time=True, + ) + + assert all_result["days"] is None + assert all_result["summary"]["total_count"] == 4 + assert all_result["summary"]["unique_ip_count"] == 3 + assert sorted(resolved_ips) == ["10.1.1.1", "10.1.1.2", "10.1.1.3"] @pytest.mark.asyncio @@ -530,9 +653,11 @@ async def test_ip_locations_keeps_private_network_location_label(db_session, mon text( """ INSERT INTO permission_access_logs - (id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, created_at) + (id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, + request_snapshot, created_at) VALUES - (:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address, CURRENT_TIMESTAMP) + (:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address, + :request_snapshot, CURRENT_TIMESTAMP) """ ), { @@ -544,6 +669,7 @@ async def test_ip_locations_keeps_private_network_location_label(db_session, mon "allowed": True, "elapsed_ms": 3.2, "ip_address": "192.168.1.10", + "request_snapshot": None, }, ) await db_session.commit() @@ -556,8 +682,8 @@ async def test_ip_locations_keeps_private_network_location_label(db_session, mon @pytest.mark.asyncio -async def test_ip_locations_includes_abnormal_security_ips(db_session, monkeypatch): - """来源分析应同时统计安全事件中的异常来源 IP。""" +async def test_ip_locations_scores_security_sources_by_behavior(db_session, monkeypatch): + """来源分析应按安全行为而非地域评估来源风险。""" await db_session.execute(text("DELETE FROM permission_access_logs")) await db_session.execute(text("DELETE FROM security_access_logs")) await db_session.commit() @@ -603,9 +729,11 @@ async def test_ip_locations_includes_abnormal_security_ips(db_session, monkeypat text( """ INSERT INTO permission_access_logs - (id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, created_at) + (id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, + request_snapshot, created_at) VALUES - (:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address, CURRENT_TIMESTAMP) + (:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address, + :request_snapshot, CURRENT_TIMESTAMP) """ ), { @@ -617,9 +745,10 @@ async def test_ip_locations_includes_abnormal_security_ips(db_session, monkeypat "allowed": True, "elapsed_ms": 3.2, "ip_address": "10.3.1.1", + "request_snapshot": None, }, ) - for ip_address in ["8.8.8.8", "1.1.1.1"]: + for ip_address, status_code in [("8.8.8.8", 403), ("1.1.1.1", 200)]: await db_session.execute( text( """ @@ -633,7 +762,7 @@ async def test_ip_locations_includes_abnormal_security_ips(db_session, monkeypat "id": str(uuid.uuid4()), "method": "GET", "path": "/api/v1/admin", - "status_code": 403, + "status_code": status_code, "elapsed_ms": 8.5, "client_ip": ip_address, "user_agent": "pytest", @@ -658,18 +787,23 @@ async def test_ip_locations_includes_abnormal_security_ips(db_session, monkeypat locations = {item["country"]: item for item in result["items"]} assert result["summary"]["total_count"] == 3 - assert result["summary"]["denied_count"] == 2 + assert result["summary"]["denied_count"] == 1 assert result["summary"]["unique_ip_count"] == 3 assert result["summary"]["unique_user_count"] == 1 assert locations["United States"]["total_count"] == 1 assert locations["United States"]["denied_count"] == 1 + assert locations["United States"]["risk_level"] == "high" + assert locations["United States"]["country_code"] == "US" assert locations["Australia"]["total_count"] == 1 + assert locations["Australia"]["denied_count"] == 0 + assert locations["Australia"]["risk_level"] == "normal" @pytest.mark.asyncio async def test_access_logs_include_user_behavior_summary(db_session): """访问日志应返回用户行为审计汇总和用户排行。""" await db_session.execute(text("DELETE FROM permission_access_logs")) + await db_session.execute(text("DELETE FROM security_access_logs")) await db_session.commit() study_id = "00000000-0000-0000-0000-000000000201" @@ -784,10 +918,300 @@ def test_access_logs_user_stats_query_avoids_postgresql_uuid_max(): assert "func.max(PermissionAccessLog.user_id)" not in source +@pytest.mark.asyncio +async def test_access_logs_filter_by_account_location_and_client_source(db_session, monkeypatch): + """访问审计应支持按账号、IP 属地和客户端来源排查。""" + await db_session.execute(text("DELETE FROM permission_access_logs")) + await db_session.execute(text("DELETE FROM security_access_logs")) + await db_session.commit() + + study_id = "00000000-0000-0000-0000-000000001401" + user_a = "00000000-0000-0000-0000-000000001501" + user_b = "00000000-0000-0000-0000-000000001502" + + await db_session.execute( + text( + """ + INSERT INTO studies (id, code, name, status, is_locked, visit_schedule, active_roles) + VALUES (:id, :code, :name, :status, :is_locked, :visit_schedule, :active_roles) + """ + ), + { + "id": study_id, + "code": "ACCESS-CONTEXT-STUDY", + "name": "Access Context Study", + "status": "ACTIVE", + "is_locked": False, + "visit_schedule": "[]", + "active_roles": "[]", + }, + ) + for user_id, email, name in [ + (user_a, "audit-shanghai@example.com", "上海审计用户"), + (user_b, "audit-beijing@example.com", "北京审计用户"), + ]: + await db_session.execute( + text( + """ + INSERT INTO users (id, email, password_hash, full_name, clinical_department, is_admin, status) + VALUES (:id, :email, :password_hash, :full_name, :clinical_department, :is_admin, :status) + """ + ), + { + "id": user_id, + "email": email, + "password_hash": "hash", + "full_name": name, + "clinical_department": "临床运营", + "is_admin": False, + "status": "ACTIVE", + }, + ) + + rows = [ + (user_a, "203.0.113.10", "web", "Chrome", "macos", {"user-agent": "Chrome", "x-request-id": "req-web"}), + (user_b, "198.51.100.20", "desktop", "CTMS Desktop", "windows", {"user-agent": "CTMS Desktop"}), + (user_a, "10.10.10.10", None, "curl/8.0", None, {"user-agent": "curl/8.0"}), + ] + for user, ip_address, client_type, user_agent, platform, request_headers in rows: + await db_session.execute( + text( + """ + INSERT INTO permission_access_logs + (id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, + user_agent, client_type, client_version, client_platform, build_channel, build_commit, request_headers, created_at) + VALUES + (:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address, + :user_agent, :client_type, :client_version, :client_platform, :build_channel, :build_commit, :request_headers, CURRENT_TIMESTAMP) + """ + ), + { + "id": str(uuid.uuid4()), + "study_id": study_id, + "user_id": user, + "endpoint_key": "admin.permissions.read", + "role": "PM", + "allowed": True, + "elapsed_ms": 4.0, + "ip_address": ip_address, + "user_agent": user_agent, + "client_type": client_type, + "client_version": "0.1.0" if client_type else None, + "client_platform": platform, + "build_channel": "local" if client_type else None, + "build_commit": "abcdef1" if client_type else None, + "request_headers": json.dumps(request_headers), + }, + ) + await db_session.commit() + + ip_info_by_address = { + "203.0.113.10": FakeIpInfo("上海市", "上海市", "电信"), + "198.51.100.20": FakeIpInfo("北京市", "北京市", "联通"), + "10.10.10.10": FakeIpInfo("", "", "", "局域网"), + } + monkeypatch.setattr( + permission_monitoring, + "resolve_ip_location", + lambda ip: ip_info_by_address[ip], + ) + + result = await permission_monitoring.get_access_logs( + db=db_session, + _=AdminUserStub(), + study_id=None, + user_id=None, + endpoint_key=None, + role=None, + allowed=None, + start_time=None, + end_time=None, + client_ip=None, + client_type="web", + keyword="上海", + page=1, + page_size=50, + ) + + assert result["total"] == 1 + assert result["items"][0]["user_email"] == "audit-shanghai@example.com" + assert result["items"][0]["client_type"] == "web" + assert result["items"][0]["client_platform"] == "macos" + assert result["items"][0]["user_agent"] == "Chrome" + assert result["items"][0]["request_headers"]["x-request-id"] == "req-web" + assert result["items"][0]["ip_city"] == "上海市" + + unknown_result = await permission_monitoring.get_access_logs( + db=db_session, + _=AdminUserStub(), + study_id=None, + user_id=None, + endpoint_key=None, + role=None, + allowed=None, + start_time=None, + end_time=None, + client_ip=None, + client_type="unknown", + keyword="curl", + page=1, + page_size=50, + ) + assert unknown_result["total"] == 1 + assert unknown_result["items"][0]["ip_address"] == "10.10.10.10" + + +@pytest.mark.asyncio +async def test_access_logs_include_security_events_for_admin(db_session, monkeypatch): + """访问日志应合并业务权限日志和安全访问日志，避免匿名/探测请求审计盲区。""" + await db_session.execute(text("DELETE FROM permission_access_logs")) + await db_session.execute(text("DELETE FROM security_access_logs")) + await db_session.commit() + + study_id = "00000000-0000-0000-0000-000000001601" + user_id = "00000000-0000-0000-0000-000000001701" + await db_session.execute( + text( + """ + INSERT INTO studies (id, code, name, status, is_locked, visit_schedule, active_roles) + VALUES (:id, :code, :name, :status, :is_locked, :visit_schedule, :active_roles) + """ + ), + { + "id": study_id, + "code": "ACCESS-SECURITY-FEED", + "name": "Access Security Feed", + "status": "ACTIVE", + "is_locked": False, + "visit_schedule": "[]", + "active_roles": "[]", + }, + ) + await db_session.execute( + text( + """ + INSERT INTO users (id, email, password_hash, full_name, clinical_department, is_admin, status) + VALUES (:id, :email, :password_hash, :full_name, :clinical_department, :is_admin, :status) + """ + ), + { + "id": user_id, + "email": "access-feed@example.com", + "password_hash": "hash", + "full_name": "统一访问用户", + "clinical_department": "临床运营", + "is_admin": False, + "status": "ACTIVE", + }, + ) + await db_session.execute( + text( + """ + INSERT INTO permission_access_logs + (id, study_id, user_id, endpoint_key, role, allowed, elapsed_ms, ip_address, + request_snapshot, created_at) + VALUES + (:id, :study_id, :user_id, :endpoint_key, :role, :allowed, :elapsed_ms, :ip_address, + :request_snapshot, CURRENT_TIMESTAMP) + """ + ), + { + "id": str(uuid.uuid4()), + "study_id": study_id, + "user_id": user_id, + "endpoint_key": "project_overview:read", + "role": "PM", + "allowed": True, + "elapsed_ms": 3.0, + "ip_address": "192.168.10.2", + "request_snapshot": json.dumps( + { + "method": "GET", + "path": "/api/v1/projects/overview", + "query_string": "tab=summary", + "headers": {"user-agent": "Chrome", "x-request-id": "perm-1"}, + "body": {"captured": False, "reason": "body_not_captured_by_audit_policy"}, + } + ), + }, + ) + await db_session.execute( + text( + """ + INSERT INTO security_access_logs + (id, method, path, status_code, elapsed_ms, client_ip, user_agent, auth_status, + user_identifier, request_headers, request_snapshot, created_at) + VALUES + (:id, :method, :path, :status_code, :elapsed_ms, :client_ip, :user_agent, :auth_status, + :user_identifier, :request_headers, :request_snapshot, CURRENT_TIMESTAMP) + """ + ), + { + "id": str(uuid.uuid4()), + "method": "GET", + "path": "/api/.git/config", + "status_code": 404, + "elapsed_ms": 1.5, + "client_ip": "45.148.10.95", + "user_agent": "probe-bot/1.0", + "auth_status": "ANONYMOUS", + "user_identifier": None, + "request_headers": json.dumps({"user-agent": "probe-bot/1.0", "x-request-id": "sec-1"}), + "request_snapshot": json.dumps( + { + "method": "GET", + "path": "/api/.git/config", + "query_string": "token=[redacted]", + "headers": {"user-agent": "probe-bot/1.0", "authorization": "[redacted]"}, + "body": {"captured": False, "reason": "body_not_captured_by_audit_policy"}, + } + ), + }, + ) + await db_session.commit() + + ip_info_by_address = { + "192.168.10.2": FakeIpInfo("", "", "", "局域网"), + "45.148.10.95": FakeIpInfo("South Holland", "", "", "Netherlands"), + } + monkeypatch.setattr(permission_monitoring, "resolve_ip_location", lambda ip: ip_info_by_address[ip]) + + result = await permission_monitoring.get_access_logs( + db=db_session, + _=AdminUserStub(), + study_id=None, + user_id=None, + endpoint_key=None, + role=None, + allowed=None, + start_time=None, + end_time=None, + client_ip=None, + client_type=None, + keyword=None, + page=1, + page_size=50, + ) + + assert result["total"] == 2 + assert result["summary"]["total_count"] == 2 + assert result["summary"]["denied_count"] == 1 + items_by_type = {item["event_type"]: item for item in result["items"]} + assert items_by_type["permission"]["endpoint_key"] == "project_overview:read" + assert items_by_type["security"]["status_code"] == 404 + assert items_by_type["security"]["allowed"] is False + assert items_by_type["security"]["category"] == "PROBE" + assert items_by_type["security"]["severity"] == "CRITICAL" + assert items_by_type["security"]["request_headers"]["x-request-id"] == "sec-1" + assert items_by_type["permission"]["request_snapshot"]["path"] == "/api/v1/projects/overview" + assert items_by_type["security"]["request_snapshot"]["headers"]["authorization"] == "[redacted]" + + @pytest.mark.asyncio async def test_access_logs_ip_ranking_aggregates_by_ip_total(db_session): """IP 访问排行应按同一 IP 的整体访问量聚合排序，而不是按用户/IP/角色拆分。""" await db_session.execute(text("DELETE FROM permission_access_logs")) + await db_session.execute(text("DELETE FROM security_access_logs")) await db_session.commit() study_id = "00000000-0000-0000-0000-000000001201" @@ -896,10 +1320,13 @@ async def test_security_access_logs_include_anonymous_ip_attempts(db_session, mo text( """ INSERT INTO security_access_logs - (id, method, path, status_code, elapsed_ms, client_ip, user_agent, auth_status, user_identifier, created_at) + (id, method, path, status_code, elapsed_ms, client_ip, user_agent, auth_status, + user_identifier, request_snapshot, created_at) VALUES ('00000000-0000-4000-8000-000000000501', 'POST', '/api/v1/auth/login', 401, 12.5, - '203.0.113.10', 'attack-bot/1.0', 'ANONYMOUS', NULL, CURRENT_TIMESTAMP) + '203.0.113.10', 'attack-bot/1.0', 'ANONYMOUS', NULL, + '{"method":"POST","path":"/api/v1/auth/login","headers":{"authorization":"[redacted]"}}', + CURRENT_TIMESTAMP) """ ) ) @@ -925,6 +1352,7 @@ async def test_security_access_logs_include_anonymous_ip_attempts(db_session, mo assert result["items"][0]["ip_isp"] == "联通" assert result["items"][0]["account_label"] == "未知账号" assert result["items"][0]["status_code"] == 401 + assert result["items"][0]["request_snapshot"]["headers"]["authorization"] == "[redacted]" @pytest.mark.asyncio @@ -993,8 +1421,8 @@ async def test_security_access_logs_prioritize_sensitive_probe_over_foreign_ip(d @pytest.mark.asyncio -async def test_security_access_logs_classify_non_china_ip_as_abnormal(db_session, monkeypatch): - """安全事件明细应把非中国公网 IP 归类为异常 IP。""" +async def test_security_access_logs_classify_foreign_invalid_token_by_behavior(db_session, monkeypatch): + """海外来源不应单凭地域升级风险，仍按认证行为分类。""" await db_session.execute(text("DELETE FROM security_access_logs")) monkeypatch.setattr( permission_monitoring, @@ -1024,8 +1452,8 @@ async def test_security_access_logs_classify_non_china_ip_as_abnormal(db_session ) item = result["items"][0] - assert item["category"] == "ABNORMAL_IP" - assert item["severity"] == "HIGH" + assert item["category"] == "INVALID_TOKEN" + assert item["severity"] == "MEDIUM" assert item["ip_location"] == "United States / California / San Jose / Google" assert item["ip_country"] == "United States" @@ -1062,4 +1490,108 @@ async def test_security_access_logs_resolve_public_ip_with_packaged_ip2region(db assert item["ip_country"] == "United States" assert item["ip_province"] == "California" assert item["ip_city"] == "San Jose" - assert item["category"] == "ABNORMAL_IP" + assert item["category"] == "NOT_FOUND_NOISE" + + +@pytest.mark.asyncio +async def test_access_logs_deduplicate_permission_and_security_rows_by_request_id(db_session): + await db_session.execute(text("DELETE FROM permission_access_logs")) + await db_session.execute(text("DELETE FROM security_access_logs")) + request_id = str(uuid.uuid4()) + study_id = str(uuid.uuid4()) + user_id = str(uuid.uuid4()) + await _seed_permission_log(db_session, uuid.UUID(study_id), uuid.UUID(user_id), allowed=False, elapsed_ms=4.0) + await db_session.execute( + text("UPDATE permission_access_logs SET request_id = :request_id"), + {"request_id": request_id}, + ) + await db_session.execute( + text( + """ + INSERT INTO security_access_logs + (id, method, path, status_code, elapsed_ms, client_ip, auth_status, user_identifier, + request_id, category, severity, created_at) + VALUES + (:id, 'GET', '/api/v1/studies/denied', 403, 5.0, '203.0.113.20', 'AUTHENTICATED', :user_id, + :request_id, 'OTHER', 'LOW', CURRENT_TIMESTAMP) + """ + ), + {"id": str(uuid.uuid4()), "user_id": user_id, "request_id": request_id}, + ) + await db_session.commit() + + result = await permission_monitoring.get_access_logs( + db=db_session, + _=AdminUserStub(), + study_id=None, + user_id=None, + endpoint_key=None, + role=None, + allowed=None, + start_time=None, + end_time=None, + client_ip=None, + client_type=None, + keyword=None, + page=1, + page_size=50, + ) + + assert result["total"] == 1 + assert result["items"][0]["event_type"] == "security" + assert result["items"][0]["request_id"] == request_id + + +@pytest.mark.asyncio +async def test_security_events_only_includes_persisted_probe(db_session): + await db_session.execute(text("DELETE FROM security_access_logs")) + for category, ip in [("PROBE", "52.53.218.145"), ("OTHER", "203.0.113.20")]: + await db_session.execute( + text( + """ + INSERT INTO security_access_logs + (id, method, path, status_code, elapsed_ms, client_ip, auth_status, + category, severity, created_at) + VALUES + (:id, 'GET', '/api/v1/studies/', 200, 3.0, :ip, 'AUTHENTICATED', + :category, :severity, CURRENT_TIMESTAMP) + """ + ), + { + "id": str(uuid.uuid4()), + "ip": ip, + "category": category, + "severity": "CRITICAL" if category == "PROBE" else "LOW", + }, + ) + await db_session.commit() + + result = await permission_monitoring.get_security_access_logs( + db=db_session, + _=AdminUserStub(), + status_min=None, + auth_status=None, + events_only=True, + page=1, + page_size=20, + ) + + assert result["total"] == 1 + assert result["items"][0]["category"] == "PROBE" + assert result["summary"]["high_risk_count"] == 1 + + +def test_server_error_classification_takes_priority_over_foreign_ip(): + log = SimpleNamespace( + path="/api/v1/studies/", + status_code=503, + auth_status="AUTHENTICATED", + category=None, + severity=None, + ) + location = FakeIpInfo("California", "San Jose", "Google", "United States") + + assert permission_monitoring._classify_security_access_log(log, location) == { + "category": "SERVER_ERROR", + "severity": "HIGH", + } diff --git a/backend/tests/test_registration.py b/backend/tests/test_registration.py index 32f96064..e0d75f31 100644 --- a/backend/tests/test_registration.py +++ b/backend/tests/test_registration.py @@ -12,6 +12,7 @@ from datetime import datetime, timedelta, timezone import base64 import json import os +import uuid from app.main import create_app from app.core.config import settings @@ -397,6 +398,7 @@ async def test_root_returns_service_metadata(client_and_db): client, _ = client_and_db resp = await client.get("/") assert resp.status_code == 200 + assert uuid.UUID(resp.headers["x-request-id"]) assert resp.json() == { "service": "ctms-backend", "status": "ok", @@ -406,6 +408,17 @@ async def test_root_returns_service_metadata(client_and_db): } +@pytest.mark.asyncio +async def test_readiness_checks_database(client_and_db): + client, _ = client_and_db + resp = await client.get("/readyz") + assert resp.status_code == 200 + payload = resp.json() + assert payload["status"] == "ready" + assert payload["database"]["status"] == "healthy" + assert payload["database"]["latency_ms"] >= 0 + + @pytest.mark.asyncio async def test_registered_user_can_login_after_email_verification(client_and_db): client, SessionLocal = client_and_db diff --git a/backend/tests/test_source_location_aggregator.py b/backend/tests/test_source_location_aggregator.py new file mode 100644 index 00000000..727c38d5 --- /dev/null +++ b/backend/tests/test_source_location_aggregator.py @@ -0,0 +1,68 @@ +from datetime import datetime, timedelta, timezone + +import pytest + +from app.models.source_location_snapshot import SourceLocationSnapshot +from app.services.source_location_aggregator import _identity_hash, get_source_location_timeline + + +def _snapshot(bucket_time: datetime, *, allowed: int, denied: int) -> SourceLocationSnapshot: + return SourceLocationSnapshot( + bucket_time=bucket_time, + ip_hash=_identity_hash("ip", "203.0.113.10"), + user_hash=_identity_hash("user", "user-1"), + country="United States", + country_code="US", + province="California", + region_code="", + city="San Jose", + isp="Example ISP", + location="United States / California / San Jose", + longitude=-121.8863, + latitude=37.3382, + accuracy_level="city", + allowed_count=allowed, + denied_count=denied, + security_event_count=denied, + high_risk_count=0, + auth_failure_count=denied, + first_seen_at=bucket_time, + last_seen_at=bucket_time + timedelta(minutes=30), + ) + + +def test_source_location_identity_hash_is_stable_and_does_not_expose_ip(): + first = _identity_hash("ip", "203.0.113.10") + second = _identity_hash("ip", "203.0.113.10") + + assert first == second + assert len(first) == 64 + assert "203.0.113.10" not in first + + +@pytest.mark.asyncio +async def test_source_location_timeline_supports_hour_and_day_rollups(db_session): + start = datetime(2026, 7, 10, 8, tzinfo=timezone.utc) + db_session.add(_snapshot(start, allowed=4, denied=1)) + db_session.add(_snapshot(start + timedelta(hours=1), allowed=3, denied=2)) + await db_session.commit() + + hourly = await get_source_location_timeline( + db_session, + start_at=start, + end_at=start + timedelta(hours=2), + granularity="hour", + ) + daily = await get_source_location_timeline( + db_session, + start_at=start, + end_at=start + timedelta(hours=2), + granularity="day", + ) + + assert [item["total_count"] for item in hourly] == [5, 5] + assert len(daily) == 1 + assert daily[0]["total_count"] == 10 + assert daily[0]["denied_count"] == 3 + assert daily[0]["unique_ip_count"] == 1 + assert daily[0]["unique_user_count"] == 1 diff --git a/backend/tests/test_user_login_sessions.py b/backend/tests/test_user_login_sessions.py new file mode 100644 index 00000000..9d740fe9 --- /dev/null +++ b/backend/tests/test_user_login_sessions.py @@ -0,0 +1,61 @@ +from datetime import datetime, timedelta, timezone +import uuid + +import pytest + +from app.core.config import settings +from app.models.user import User, UserStatus +from app.models.user_login_session import UserLoginSession +from app.services.user_login_sessions import get_login_summaries, session_id_from_payload + + +def test_legacy_session_id_is_stable_without_storing_a_token(): + payload = {"sub": "00000000-0000-0000-0000-000000000101", "orig_iat": 1_700_000_000, "client_type": "web"} + + first = session_id_from_payload(payload) + second = session_id_from_payload(payload) + + assert first == second + assert isinstance(first, uuid.UUID) + + +@pytest.mark.asyncio +async def test_login_summary_marks_recent_unended_sessions_online(db_session, monkeypatch): + now = datetime.now(timezone.utc) + monkeypatch.setattr(settings, "USER_SESSION_ONLINE_SECONDS", 300) + user = User( + id=uuid.uuid4(), + email="login-summary@example.com", + password_hash="hash", + full_name="Login Summary", + clinical_department="IT", + status=UserStatus.ACTIVE, + ) + db_session.add(user) + db_session.add_all( + [ + UserLoginSession( + id=uuid.uuid4(), + user_id=user.id, + client_type="desktop", + login_at=now - timedelta(minutes=10), + last_seen_at=now - timedelta(seconds=20), + ), + UserLoginSession( + id=uuid.uuid4(), + user_id=user.id, + client_type="web", + login_at=now - timedelta(days=1), + last_seen_at=now - timedelta(days=1), + ended_at=now - timedelta(days=1), + end_reason="logout", + ), + ] + ) + await db_session.commit() + + summaries = await get_login_summaries(db_session, [user.id]) + + assert summaries[user.id].status == "ONLINE" + assert summaries[user.id].active_session_count == 1 + assert summaries[user.id].client_type == "desktop" diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 80b45dba..37fbaf80 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -68,7 +68,7 @@ services: - ./frontend/public/favicon.ico:/usr/share/nginx/html/favicon.ico:ro depends_on: backend: - condition: service_started + condition: service_healthy frontend-dev: condition: service_healthy networks: diff --git a/docker-compose.yaml b/docker-compose.yaml index f5e747a8..49da1b89 100755 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -33,9 +33,26 @@ services: FRONTEND_PUBLIC_URL: ${FRONTEND_PUBLIC_URL:-http://localhost:8888} LOGIN_RSA_PRIVATE_KEY: ${LOGIN_RSA_PRIVATE_KEY:-} LOGIN_RSA_KEY_ID: ${LOGIN_RSA_KEY_ID:-default} + MONITORING_ACCESS_LOG_RETENTION_DAYS: ${MONITORING_ACCESS_LOG_RETENTION_DAYS:-90} + MONITORING_METRIC_RETENTION_DAYS: ${MONITORING_METRIC_RETENTION_DAYS:-400} + MONITORING_RETENTION_INTERVAL_SECONDS: ${MONITORING_RETENTION_INTERVAL_SECONDS:-86400} + USER_LOGIN_ACTIVITY_RETENTION_DAYS: ${USER_LOGIN_ACTIVITY_RETENTION_DAYS:-180} + USER_SESSION_ONLINE_SECONDS: ${USER_SESSION_ONLINE_SECONDS:-300} + MONITORING_SERVER_PUBLIC_IP: ${MONITORING_SERVER_PUBLIC_IP:-} + MONITORING_PUBLIC_IP_DISCOVERY_URLS: ${MONITORING_PUBLIC_IP_DISCOVERY_URLS:-https://api64.ipify.org,https://icanhazip.com} depends_on: db: condition: service_healthy + healthcheck: + test: + - CMD + - python + - -c + - import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/readyz', timeout=2).read() + interval: 10s + timeout: 3s + retries: 5 + start_period: 20s networks: - ctms_net @@ -76,7 +93,8 @@ services: - ./nginx/certs:/etc/nginx/certs:ro - ./frontend/public/favicon.ico:/usr/share/nginx/html/favicon.ico:ro depends_on: - - backend + backend: + condition: service_healthy networks: - ctms_net diff --git a/docs/guides/system-monitoring-operations.md b/docs/guides/system-monitoring-operations.md new file mode 100644 index 00000000..9471be86 --- /dev/null +++ b/docs/guides/system-monitoring-operations.md @@ -0,0 +1,82 @@ +# 系统监测简易运维指南 + +## 定位与访问边界 + +系统监测用于单实例或小规模 CTMS 部署的日常巡检、访问追踪和初步故障定位，不替代集中式指标、日志和告警平台。监测页面及 `/api/v1/permission-monitoring/*` 仅允许系统管理员访问，项目 PM 不具备系统级监测权限。 + +## 健康检查 + +- `GET /health`：进程存活探针，不访问数据库，适合判断 HTTP 服务是否仍在响应。 +- `GET /readyz`：服务就绪探针，会执行数据库查询；数据库不可用时返回 `503`。Nginx、安装脚本和后端容器健康检查均使用或暴露此探针。 +- `GET /api/v1/permission-monitoring/health`：管理员健康详情，包含权限检查质量、缓存、进程内告警、数据库延迟、权限/安全日志写入队列、留存任务和监测表数据量。 + +部署后的最小检查： + +```bash +curl -fsS http://127.0.0.1:8888/health +curl -fsS http://127.0.0.1:8888/readyz +docker compose ps +``` + +## 数据留存 + +后台任务在应用启动后立即清理一次，之后按固定间隔执行。默认策略： + +- 权限访问日志和安全访问日志：90 天。 +- 权限小时指标：400 天。 +- 账号登录活动：180 天。 +- 清理周期：86400 秒（每天）。 + +可通过后端环境变量调整： + +```text +MONITORING_ACCESS_LOG_RETENTION_DAYS=90 +MONITORING_METRIC_RETENTION_DAYS=400 +MONITORING_RETENTION_INTERVAL_SECONDS=86400 +USER_LOGIN_ACTIVITY_RETENTION_DAYS=180 +USER_SESSION_ONLINE_SECONDS=300 +``` + +访问日志留存范围为 7–3650 天，指标留存范围为 30–3650 天，清理周期范围为 60–604800 秒。修改后需重启后端。正式环境部署新版本前必须先执行 Alembic migration。 + +账号管理页的“在线”状态由服务端会话心跳计算：最近 `USER_SESSION_ONLINE_SECONDS` 秒内成功心跳且未退出的会话视为在线。登录记录仅保存客户端类型、版本、登录/最近活动/退出时间；不保存 Token、密码或原始 IP。 + +## 来源地图服务器位置 + +来源地图中的服务器标记不使用固定城市或前端坐标。后端启动时按以下顺序确定部署服务器的公网地址，并缓存定位结果： + +1. `MONITORING_SERVER_PUBLIC_IP` 明确指定的公网 IP，适用于禁止主动访问公网探测服务的生产网络。 +2. `FRONTEND_PUBLIC_URL` 主机名解析出的公网 IP。 +3. `MONITORING_PUBLIC_IP_DISCOVERY_URLS` 配置的公网出口 IP 查询服务。 + +默认公网查询服务为 `https://api64.ipify.org,https://icanhazip.com`，单次超时 2.5 秒。解析成功后通过本地 ip2region 数据库确定属地，再由后端转换为地图坐标；解析失败时 API 返回 `server_location: null`，地图不会使用任意默认城市代替。 + +受限网络建议显式配置： + +```text +MONITORING_SERVER_PUBLIC_IP=<部署服务器公网IP> +MONITORING_PUBLIC_IP_DISCOVERY_URLS= +``` + +公网 IP 仅用于服务端定位，不写入来源分析 API 响应或浏览器界面。 + +## 日志完整性与隐私 + +- 每个请求由服务端生成 UUID 请求标识，并通过响应头 `X-Request-ID` 返回；权限日志和安全日志以该标识消除同一请求的重复统计。 +- 写入队列保持非阻塞；队列满或数据库批次写入最终失败时不阻塞业务请求，但会累计丢弃量、失败批次、队列占用和最近错误时间，并在管理员健康页显示降级。 +- 请求正文不采集。请求头仅保留运维白名单字段，认证、Cookie、密钥类字段统一脱敏；查询参数中的令牌、账号、受试者/患者、姓名、邮箱、电话、证件和地址类值会替换为 `[redacted]`。 +- IP、User-Agent 和请求路径仍属于运维审计数据，应按管理员最小授权和上述留存策略管理，不应复制到公开工单或通知正文。 + +## 常见异常处理 + +| 现象 | 首要检查 | 建议处理 | +| --- | --- | --- | +| `/health` 正常、`/readyz` 返回 503 | 数据库容器、连接串、迁移状态 | 检查 `docker compose ps`、数据库日志和 `alembic current` | +| 日志写入器降级 | 队列占用、丢弃量、失败批次、最近错误 | 检查数据库连接和容量；恢复后确认队列回落并重启以清零进程累计计数 | +| 留存任务异常 | 最近成功/错误时间 | 检查数据库权限、表结构和后端日志，确认 migration 已升级 | +| 页面显示“可能是上次成功结果” | 对应 API 或网络请求 | 使用刷新按钮重试，并结合 `/readyz` 与浏览器网络面板定位 | +| 安全事件突然增多 | 分类、来源 IP、路径和状态码 | 优先核对敏感路径探测、5xx 和无效令牌，不要仅按 4xx 总量或地理位置判断 | + +## 当前边界与后续升级 + +权限缓存指标、告警列表和写入器累计计数仍为单进程内存状态，重启后会重置；当前也不负责短信、邮件、Slack 等外部通知。需要多实例部署、跨重启趋势、值班通知或长期容量分析时，应接入 Prometheus/OpenTelemetry、集中日志和 Alertmanager 类告警链路，并保留本模块作为管理员快速诊断入口。 diff --git a/frontend/src/api/authClient.ts b/frontend/src/api/authClient.ts index 56cd1d60..e7d6da99 100644 --- a/frontend/src/api/authClient.ts +++ b/frontend/src/api/authClient.ts @@ -33,6 +33,11 @@ export type LoginKeyResponse = { expires_at: string; }; +export type SessionHeartbeatResponse = { + status: "online"; + last_seen_at: string; +}; + export type EncryptedPasswordRequest = { key_id: string; challenge: string; @@ -53,4 +58,18 @@ export const extendToken = (token: string): Promise> => authClient.get("/api/v1/auth/login-key"); +export const heartbeatSession = (token: string): Promise> => + authClient.post( + "/api/v1/auth/session/heartbeat", + {}, + { headers: { Authorization: `Bearer ${token}` }, timeout: 5000 }, + ); + +export const logoutSession = (token: string): Promise> => + authClient.post( + "/api/v1/auth/session/logout", + {}, + { headers: { Authorization: `Bearer ${token}` } }, + ); + export default authClient; diff --git a/frontend/src/api/projectPermissions.ts b/frontend/src/api/projectPermissions.ts index d30a07d4..67ece223 100644 --- a/frontend/src/api/projectPermissions.ts +++ b/frontend/src/api/projectPermissions.ts @@ -78,6 +78,9 @@ export const fetchAccessLogs = (params: { allowed?: boolean; start_time?: string; end_time?: string; + client_ip?: string; + client_type?: string; + keyword?: string; page?: number; page_size?: number; }) => apiGet(`/api/v1/permission-monitoring/access-logs`, { params }); @@ -87,6 +90,13 @@ export const fetchSecurityAccessLogs = (params?: { auth_status?: string; client_type?: string; client_version?: string; + client_ip?: string; + category?: string; + severity?: string; + keyword?: string; + events_only?: boolean; + start_time?: string; + end_time?: string; page?: number; page_size?: number; }) => apiGet(`/api/v1/permission-monitoring/security-logs`, { params }); @@ -97,7 +107,7 @@ export const fetchPermissionTrends = (period: "24h" | "7d" | "30d" = "24h") => export const fetchTopDenied = (params?: { days?: number; limit?: number }) => apiGet(`/api/v1/permission-monitoring/top-denied`, { params }); -export const fetchIpLocations = (params?: { days?: number; limit?: number }) => +export const fetchIpLocations = (params?: { days?: number; all_time?: boolean; limit?: number }) => apiGet(`/api/v1/permission-monitoring/ip-locations`, { params }); export const fetchStatsSummary = () => diff --git a/frontend/src/api/users.ts b/frontend/src/api/users.ts index fdbcef2f..ed08c339 100644 --- a/frontend/src/api/users.ts +++ b/frontend/src/api/users.ts @@ -1,5 +1,5 @@ import { apiGet, apiPatch, apiPost } from "./axios"; -import type { ApiListResponse, UserInfo } from "../types/api"; +import type { ApiListResponse, UserInfo, UserLoginActivity } from "../types/api"; import { apiDelete } from "./axios"; export const fetchUsers = (params?: Record) => @@ -21,3 +21,8 @@ export const updateUser = ( export const deleteUser = (userId: string) => apiDelete(`/api/v1/users/${userId}`, { suppressErrorMessage: true }); + +export const fetchUserLoginActivities = (userId: string, limit = 30) => + apiGet(`/api/v1/users/${userId}/login-activities`, { + params: { limit }, + }); diff --git a/frontend/src/components/AccountConnectionStatus.test.ts b/frontend/src/components/AccountConnectionStatus.test.ts new file mode 100644 index 00000000..5b30f13b --- /dev/null +++ b/frontend/src/components/AccountConnectionStatus.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const readSource = (path: string) => readFileSync(resolve(__dirname, path), "utf8"); + +describe("account connection status", () => { + it("keeps the compact indicator and detailed account diagnostics in both shells", () => { + const component = readSource("./AccountConnectionStatus.vue"); + const webLayout = readSource("./WebLayout.vue"); + const desktopLayout = readSource("./DesktopLayout.vue"); + + expect(component).toContain('mode === \'indicator\''); + expect(component).toContain("当前账号与连接状态"); + expect(component).toContain("API 延迟"); + expect(component).toContain("会话状态"); + expect(component).toContain("不代表系统健康评分"); + expect(component).not.toContain("ip_address"); + expect(component).not.toContain("access_token"); + expect(webLayout).toContain(''); + expect(webLayout).toContain(''); + expect(desktopLayout).toContain(''); + expect(desktopLayout).toContain(''); + }); +}); diff --git a/frontend/src/components/AccountConnectionStatus.vue b/frontend/src/components/AccountConnectionStatus.vue new file mode 100644 index 00000000..72cca195 --- /dev/null +++ b/frontend/src/components/AccountConnectionStatus.vue @@ -0,0 +1,267 @@ + + + + + diff --git a/frontend/src/components/DesktopLayout.vue b/frontend/src/components/DesktopLayout.vue index ea075910..0a334184 100644 --- a/frontend/src/components/DesktopLayout.vue +++ b/frontend/src/components/DesktopLayout.vue @@ -246,10 +246,12 @@