perf: dashboard counters and storage quota use SQL aggregation (D3)

- quota.get_storage_usage (runs on every upload) now aggregates via Sum
  instead of loading every row
- dashboard simple counters (total/size/expired/active/text/chunked/used,
  yesterday/today, top suffixes, recent 8) moved to SQL; the per-row health
  rules engine keeps its single pass (documented)
This commit is contained in:
Marrrrrrrrry committed 2026-09-09 19:04:12 +08:00
1 parent bb95017aa3
commit baecebd81e
3 files changed
+136 -38

No files matched your search

+53 -32
View File
@@ -31,6 +31,8 @@ from apps.admin.schemas import (
)
from core.response import APIResponse
from apps.base.models import FileCodes, KeyValue
from tortoise.expressions import Q
from tortoise.functions import Count, Sum
from apps.admin.dependencies import (
create_token,
get_admin_session_expire_seconds,
@@ -116,55 +118,74 @@ async def build_dashboard_recent_file(file_code: FileCodes) -> dict:
}
def _expired_predicate(now: datetime.datetime) -> Q:
"""SQL version of FileCodes.is_expired:
expired_at IS NOT NULL AND ((expired_count < 0 AND expired_at < now) OR expired_count = 0)
"""
return Q(expired_at__isnull=False) & (
Q(expired_count__lt=0, expired_at__lt=now) | Q(expired_count=0)
)
async def _sum_size(queryset) -> int:
rows = await queryset.annotate(total=Sum("size")).values("total")
return rows[0]["total"] or 0
@admin_api.get("/dashboard")
async def dashboard(file_service: FileService = Depends(get_file_service)):
all_codes = await FileCodes.all()
all_size = sum([code.size for code in all_codes])
sys_start = await KeyValue.filter(key="sys_start").first()
now = await get_now()
today_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
yesterday_start = today_start - datetime.timedelta(days=1)
yesterday_end = today_start - datetime.timedelta(microseconds=1)
yesterday_codes = FileCodes.filter(
created_at__gte=yesterday_start, created_at__lte=yesterday_end
)
today_codes = FileCodes.filter(created_at__gte=today_start)
yesterday_file_codes = await yesterday_codes
today_file_codes = await today_codes
expired_count = 0
for file_code in all_codes:
if await file_code.is_expired():
expired_count += 1
health_summary = await file_service.build_file_health_summary(all_codes, now=now)
text_count = sum(1 for file_code in all_codes if file_code.text is not None)
chunked_count = sum(1 for file_code in all_codes if file_code.is_chunked)
used_count = sum([file_code.used_count for file_code in all_codes])
# 简单计数全部走 SQL 聚合,避免整表载入内存(D3)。
# 健康摘要依赖逐行 rules engine(_build_file_status_insights),保留一次遍历。
total_files = await FileCodes.all().count()
expired_count = await FileCodes.filter(_expired_predicate(now)).count()
all_size = await _sum_size(FileCodes.all())
used_count = (
await FileCodes.all().annotate(total=Sum("used_count")).values("total")
)[0]["total"] or 0
text_count = await FileCodes.filter(text__isnull=False).count()
chunked_count = await FileCodes.filter(is_chunked=True).count()
yesterday_count = await FileCodes.filter(
created_at__gte=yesterday_start, created_at__lte=yesterday_end
).count()
yesterday_size = await _sum_size(
FileCodes.filter(created_at__gte=yesterday_start, created_at__lte=yesterday_end)
)
today_count = await FileCodes.filter(created_at__gte=today_start).count()
today_size = await _sum_size(FileCodes.filter(created_at__gte=today_start))
suffix_rows = (
await FileCodes.filter(text__isnull=True)
.annotate(count=Count("id"))
.values("suffix", "count")
)
suffix_counter = Counter(
"Text" if file_code.text is not None else (file_code.suffix or "file")
for file_code in all_codes
{((row["suffix"] or "file") or "file"): row["count"] for row in suffix_rows}
)
recent_file_codes = await FileCodes.all().order_by("-created_at").limit(8)
health_summary = await file_service.build_file_health_summary(
await FileCodes.all(), now=now
)
recent_file_codes = sorted(
all_codes,
key=lambda file_code: file_code.created_at.timestamp()
if file_code.created_at
else 0,
reverse=True,
)[:8]
recent_activities = await file_service.list_admin_activities(limit=8)
return APIResponse(
detail={
"totalFiles": len(all_codes),
"totalFiles": total_files,
"storageUsed": str(all_size),
"sysUptime": sys_start.value if sys_start else None,
"yesterdayCount": len(yesterday_file_codes),
"yesterdaySize": str(sum([code.size for code in yesterday_file_codes])),
"todayCount": len(today_file_codes),
"todaySize": str(sum([code.size for code in today_file_codes])),
"activeCount": len(all_codes) - expired_count,
"yesterdayCount": yesterday_count,
"yesterdaySize": str(yesterday_size),
"todayCount": today_count,
"todaySize": str(today_size),
"activeCount": total_files - expired_count,
"expiredCount": expired_count,
"textCount": text_count,
"fileCount": len(all_codes) - text_count,
"fileCount": total_files - text_count,
"chunkedCount": chunked_count,
"usedCount": used_count,
"storageBackend": settings.file_storage,
+8 -6
View File
@@ -2,6 +2,7 @@ import datetime
from fastapi import HTTPException
from tortoise import connections
from tortoise.functions import Sum
from apps.base.models import FileCodes, StorageReservation
from core.settings import settings
@@ -54,13 +55,14 @@ def get_storage_limit() -> int:
async def get_storage_usage() -> dict[str, int | None]:
now = await get_now()
used = await FileCodes.all().values_list("size", flat=True)
reserved = await StorageReservation.filter(expires_at__gt=now).values_list(
"size", flat=True
)
# SQL 聚合:此函数在每次上传配额检查时调用,禁止全表拉取(D3)
used_rows = await FileCodes.all().annotate(total=Sum("size")).values("total")
reserved_rows = await StorageReservation.filter(expires_at__gt=now).annotate(
total=Sum("size")
).values("total")
limit = get_storage_limit()
used_bytes = sum(used)
reserved_bytes = sum(reserved)
used_bytes = used_rows[0]["total"] or 0
reserved_bytes = reserved_rows[0]["total"] or 0
return {
"limit": limit,
"used": used_bytes,
+75
View File
@@ -0,0 +1,75 @@
"""D3 回归:dashboard 与配额使用 SQL 聚合,不再整表载入。
用混合数据(文本/文件、过期/活跃、分块)验证聚合结果与语义一致。
"""
import asyncio
import datetime
import unittest
from tortoise import Tortoise
from apps.admin.services import FileService
from apps.admin.views import dashboard
from apps.base.models import FileCodes
from apps.base.quota import get_storage_usage
from tests.helpers import init_memory_db
from core.utils import get_now
class DashboardQuotaAggregationTests(unittest.TestCase):
def test_dashboard_counters_and_storage_usage_match_data(self):
asyncio.run(self._scenario())
async def _scenario(self):
await init_memory_db()
try:
now = await get_now()
past = now - datetime.timedelta(days=2)
future = now + datetime.timedelta(days=30)
# 活跃文件(时间式)
await FileCodes.create(
code="f1", size=100, expired_count=-1, expired_at=future,
text=None, prefix="doc", suffix=".txt",
)
# 已过期(时间式)
await FileCodes.create(
code="f2", size=200, expired_count=-1, expired_at=past,
text=None, prefix="old", suffix=".bin",
)
# 次数耗尽(count 式,带过期时间)
await FileCodes.create(
code="f3", size=300, expired_count=0, expired_at=future,
text=None, prefix="cnt", suffix=".zip",
)
# 文本分享
await FileCodes.create(
code="t1", size=50, expired_count=-1, expired_at=future,
text="hello",
)
# 永久
await FileCodes.create(
code="p1", size=400, expired_count=-1, expired_at=None, text=None
)
detail = (await dashboard(FileService())).detail
self.assertEqual(detail["totalFiles"], 5)
self.assertEqual(detail["storageUsed"], "1050")
self.assertEqual(detail["expiredCount"], 2, "时间过期+次数耗尽都算过期")
self.assertEqual(detail["activeCount"], 3)
self.assertEqual(detail["textCount"], 1)
self.assertEqual(detail["fileCount"], 4)
self.assertEqual(detail["usedCount"], 0)
usage = await get_storage_usage()
self.assertEqual(usage["used"], 1050, "配额聚合应与手工求和一致")
self.assertIsNone(usage["available"]) # 未设 storage_limit
finally:
await Tortoise.close_connections()
from tests.helpers import close_db as _c
await _c()
if __name__ == "__main__":
unittest.main()