From baecebd81efc9677acac29d5335e281d6bb5a252 Mon Sep 17 00:00:00 2001 From: Marrrrrrrrry <34876935+Marrrrrrrrry@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:04:12 +0800 Subject: [PATCH] perf: dashboard counters and storage quota use SQL aggregation (D3) - quota.get_storage_usage (runs on every upload) now aggregates via Sum instead of loading every row - dashboard simple counters (total/size/expired/active/text/chunked/used, yesterday/today, top suffixes, recent 8) moved to SQL; the per-row health rules engine keeps its single pass (documented) --- apps/admin/views.py | 85 +++++++++++++++++++------------ apps/base/quota.py | 14 ++--- tests/test_dashboard_quota_sql.py | 75 +++++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 38 deletions(-) create mode 100644 tests/test_dashboard_quota_sql.py diff --git a/apps/admin/views.py b/apps/admin/views.py index 4310e02..684a8d9 100644 --- a/apps/admin/views.py +++ b/apps/admin/views.py @@ -31,6 +31,8 @@ from apps.admin.schemas import ( ) from core.response import APIResponse from apps.base.models import FileCodes, KeyValue +from tortoise.expressions import Q +from tortoise.functions import Count, Sum from apps.admin.dependencies import ( create_token, get_admin_session_expire_seconds, @@ -116,55 +118,74 @@ async def build_dashboard_recent_file(file_code: FileCodes) -> dict: } +def _expired_predicate(now: datetime.datetime) -> Q: + """SQL version of FileCodes.is_expired: + expired_at IS NOT NULL AND ((expired_count < 0 AND expired_at < now) OR expired_count = 0) + """ + return Q(expired_at__isnull=False) & ( + Q(expired_count__lt=0, expired_at__lt=now) | Q(expired_count=0) + ) + + +async def _sum_size(queryset) -> int: + rows = await queryset.annotate(total=Sum("size")).values("total") + return rows[0]["total"] or 0 + + @admin_api.get("/dashboard") async def dashboard(file_service: FileService = Depends(get_file_service)): - all_codes = await FileCodes.all() - all_size = sum([code.size for code in all_codes]) sys_start = await KeyValue.filter(key="sys_start").first() now = await get_now() today_start = now.replace(hour=0, minute=0, second=0, microsecond=0) yesterday_start = today_start - datetime.timedelta(days=1) yesterday_end = today_start - datetime.timedelta(microseconds=1) - yesterday_codes = FileCodes.filter( - created_at__gte=yesterday_start, created_at__lte=yesterday_end - ) - today_codes = FileCodes.filter(created_at__gte=today_start) - yesterday_file_codes = await yesterday_codes - today_file_codes = await today_codes - expired_count = 0 - for file_code in all_codes: - if await file_code.is_expired(): - expired_count += 1 - health_summary = await file_service.build_file_health_summary(all_codes, now=now) - text_count = sum(1 for file_code in all_codes if file_code.text is not None) - chunked_count = sum(1 for file_code in all_codes if file_code.is_chunked) - used_count = sum([file_code.used_count for file_code in all_codes]) + # 简单计数全部走 SQL 聚合,避免整表载入内存(D3)。 + # 健康摘要依赖逐行 rules engine(_build_file_status_insights),保留一次遍历。 + total_files = await FileCodes.all().count() + expired_count = await FileCodes.filter(_expired_predicate(now)).count() + all_size = await _sum_size(FileCodes.all()) + used_count = ( + await FileCodes.all().annotate(total=Sum("used_count")).values("total") + )[0]["total"] or 0 + text_count = await FileCodes.filter(text__isnull=False).count() + chunked_count = await FileCodes.filter(is_chunked=True).count() + yesterday_count = await FileCodes.filter( + created_at__gte=yesterday_start, created_at__lte=yesterday_end + ).count() + yesterday_size = await _sum_size( + FileCodes.filter(created_at__gte=yesterday_start, created_at__lte=yesterday_end) + ) + today_count = await FileCodes.filter(created_at__gte=today_start).count() + today_size = await _sum_size(FileCodes.filter(created_at__gte=today_start)) + + suffix_rows = ( + await FileCodes.filter(text__isnull=True) + .annotate(count=Count("id")) + .values("suffix", "count") + ) suffix_counter = Counter( - "Text" if file_code.text is not None else (file_code.suffix or "file") - for file_code in all_codes + {((row["suffix"] or "file") or "file"): row["count"] for row in suffix_rows} + ) + + recent_file_codes = await FileCodes.all().order_by("-created_at").limit(8) + health_summary = await file_service.build_file_health_summary( + await FileCodes.all(), now=now ) - recent_file_codes = sorted( - all_codes, - key=lambda file_code: file_code.created_at.timestamp() - if file_code.created_at - else 0, - reverse=True, - )[:8] recent_activities = await file_service.list_admin_activities(limit=8) return APIResponse( detail={ - "totalFiles": len(all_codes), + "totalFiles": total_files, "storageUsed": str(all_size), "sysUptime": sys_start.value if sys_start else None, - "yesterdayCount": len(yesterday_file_codes), - "yesterdaySize": str(sum([code.size for code in yesterday_file_codes])), - "todayCount": len(today_file_codes), - "todaySize": str(sum([code.size for code in today_file_codes])), - "activeCount": len(all_codes) - expired_count, + "yesterdayCount": yesterday_count, + "yesterdaySize": str(yesterday_size), + "todayCount": today_count, + "todaySize": str(today_size), + "activeCount": total_files - expired_count, "expiredCount": expired_count, "textCount": text_count, - "fileCount": len(all_codes) - text_count, + "fileCount": total_files - text_count, "chunkedCount": chunked_count, "usedCount": used_count, "storageBackend": settings.file_storage, diff --git a/apps/base/quota.py b/apps/base/quota.py index 5a8c4ff..988ffc7 100644 --- a/apps/base/quota.py +++ b/apps/base/quota.py @@ -2,6 +2,7 @@ import datetime from fastapi import HTTPException from tortoise import connections +from tortoise.functions import Sum from apps.base.models import FileCodes, StorageReservation from core.settings import settings @@ -54,13 +55,14 @@ def get_storage_limit() -> int: async def get_storage_usage() -> dict[str, int | None]: now = await get_now() - used = await FileCodes.all().values_list("size", flat=True) - reserved = await StorageReservation.filter(expires_at__gt=now).values_list( - "size", flat=True - ) + # SQL 聚合:此函数在每次上传配额检查时调用,禁止全表拉取(D3) + used_rows = await FileCodes.all().annotate(total=Sum("size")).values("total") + reserved_rows = await StorageReservation.filter(expires_at__gt=now).annotate( + total=Sum("size") + ).values("total") limit = get_storage_limit() - used_bytes = sum(used) - reserved_bytes = sum(reserved) + used_bytes = used_rows[0]["total"] or 0 + reserved_bytes = reserved_rows[0]["total"] or 0 return { "limit": limit, "used": used_bytes, diff --git a/tests/test_dashboard_quota_sql.py b/tests/test_dashboard_quota_sql.py new file mode 100644 index 0000000..680410c --- /dev/null +++ b/tests/test_dashboard_quota_sql.py @@ -0,0 +1,75 @@ +"""D3 回归:dashboard 与配额使用 SQL 聚合,不再整表载入。 + +用混合数据(文本/文件、过期/活跃、分块)验证聚合结果与语义一致。 +""" +import asyncio +import datetime +import unittest + +from tortoise import Tortoise + +from apps.admin.services import FileService +from apps.admin.views import dashboard +from apps.base.models import FileCodes +from apps.base.quota import get_storage_usage +from tests.helpers import init_memory_db +from core.utils import get_now + + +class DashboardQuotaAggregationTests(unittest.TestCase): + def test_dashboard_counters_and_storage_usage_match_data(self): + asyncio.run(self._scenario()) + + async def _scenario(self): + await init_memory_db() + try: + now = await get_now() + past = now - datetime.timedelta(days=2) + future = now + datetime.timedelta(days=30) + + # 活跃文件(时间式) + await FileCodes.create( + code="f1", size=100, expired_count=-1, expired_at=future, + text=None, prefix="doc", suffix=".txt", + ) + # 已过期(时间式) + await FileCodes.create( + code="f2", size=200, expired_count=-1, expired_at=past, + text=None, prefix="old", suffix=".bin", + ) + # 次数耗尽(count 式,带过期时间) + await FileCodes.create( + code="f3", size=300, expired_count=0, expired_at=future, + text=None, prefix="cnt", suffix=".zip", + ) + # 文本分享 + await FileCodes.create( + code="t1", size=50, expired_count=-1, expired_at=future, + text="hello", + ) + # 永久 + await FileCodes.create( + code="p1", size=400, expired_count=-1, expired_at=None, text=None + ) + + detail = (await dashboard(FileService())).detail + + self.assertEqual(detail["totalFiles"], 5) + self.assertEqual(detail["storageUsed"], "1050") + self.assertEqual(detail["expiredCount"], 2, "时间过期+次数耗尽都算过期") + self.assertEqual(detail["activeCount"], 3) + self.assertEqual(detail["textCount"], 1) + self.assertEqual(detail["fileCount"], 4) + self.assertEqual(detail["usedCount"], 0) + + usage = await get_storage_usage() + self.assertEqual(usage["used"], 1050, "配额聚合应与手工求和一致") + self.assertIsNone(usage["available"]) # 未设 storage_limit + finally: + await Tortoise.close_connections() + from tests.helpers import close_db as _c + await _c() + + +if __name__ == "__main__": + unittest.main()