mirror of
https://github.com/vastsa/FileCodeBox.git
synced 2026-11-22 06:57:56 +08:00
perf: dashboard counters and storage quota use SQL aggregation (D3)
- quota.get_storage_usage (runs on every upload) now aggregates via Sum instead of loading every row - dashboard simple counters (total/size/expired/active/text/chunked/used, yesterday/today, top suffixes, recent 8) moved to SQL; the per-row health rules engine keeps its single pass (documented)
This commit is contained in:
1 parent
bb95017aa3
commit
baecebd81e
3 files changed
+136
-38
No files matched your search
+53
-32
@@ -31,6 +31,8 @@ from apps.admin.schemas import (
|
||||
)
|
||||
from core.response import APIResponse
|
||||
from apps.base.models import FileCodes, KeyValue
|
||||
from tortoise.expressions import Q
|
||||
from tortoise.functions import Count, Sum
|
||||
from apps.admin.dependencies import (
|
||||
create_token,
|
||||
get_admin_session_expire_seconds,
|
||||
@@ -116,55 +118,74 @@ async def build_dashboard_recent_file(file_code: FileCodes) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _expired_predicate(now: datetime.datetime) -> Q:
|
||||
"""SQL version of FileCodes.is_expired:
|
||||
expired_at IS NOT NULL AND ((expired_count < 0 AND expired_at < now) OR expired_count = 0)
|
||||
"""
|
||||
return Q(expired_at__isnull=False) & (
|
||||
Q(expired_count__lt=0, expired_at__lt=now) | Q(expired_count=0)
|
||||
)
|
||||
|
||||
|
||||
async def _sum_size(queryset) -> int:
|
||||
rows = await queryset.annotate(total=Sum("size")).values("total")
|
||||
return rows[0]["total"] or 0
|
||||
|
||||
|
||||
@admin_api.get("/dashboard")
|
||||
async def dashboard(file_service: FileService = Depends(get_file_service)):
|
||||
all_codes = await FileCodes.all()
|
||||
all_size = sum([code.size for code in all_codes])
|
||||
sys_start = await KeyValue.filter(key="sys_start").first()
|
||||
now = await get_now()
|
||||
today_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
yesterday_start = today_start - datetime.timedelta(days=1)
|
||||
yesterday_end = today_start - datetime.timedelta(microseconds=1)
|
||||
yesterday_codes = FileCodes.filter(
|
||||
created_at__gte=yesterday_start, created_at__lte=yesterday_end
|
||||
)
|
||||
today_codes = FileCodes.filter(created_at__gte=today_start)
|
||||
yesterday_file_codes = await yesterday_codes
|
||||
today_file_codes = await today_codes
|
||||
expired_count = 0
|
||||
for file_code in all_codes:
|
||||
if await file_code.is_expired():
|
||||
expired_count += 1
|
||||
health_summary = await file_service.build_file_health_summary(all_codes, now=now)
|
||||
|
||||
text_count = sum(1 for file_code in all_codes if file_code.text is not None)
|
||||
chunked_count = sum(1 for file_code in all_codes if file_code.is_chunked)
|
||||
used_count = sum([file_code.used_count for file_code in all_codes])
|
||||
# 简单计数全部走 SQL 聚合,避免整表载入内存(D3)。
|
||||
# 健康摘要依赖逐行 rules engine(_build_file_status_insights),保留一次遍历。
|
||||
total_files = await FileCodes.all().count()
|
||||
expired_count = await FileCodes.filter(_expired_predicate(now)).count()
|
||||
all_size = await _sum_size(FileCodes.all())
|
||||
used_count = (
|
||||
await FileCodes.all().annotate(total=Sum("used_count")).values("total")
|
||||
)[0]["total"] or 0
|
||||
text_count = await FileCodes.filter(text__isnull=False).count()
|
||||
chunked_count = await FileCodes.filter(is_chunked=True).count()
|
||||
yesterday_count = await FileCodes.filter(
|
||||
created_at__gte=yesterday_start, created_at__lte=yesterday_end
|
||||
).count()
|
||||
yesterday_size = await _sum_size(
|
||||
FileCodes.filter(created_at__gte=yesterday_start, created_at__lte=yesterday_end)
|
||||
)
|
||||
today_count = await FileCodes.filter(created_at__gte=today_start).count()
|
||||
today_size = await _sum_size(FileCodes.filter(created_at__gte=today_start))
|
||||
|
||||
suffix_rows = (
|
||||
await FileCodes.filter(text__isnull=True)
|
||||
.annotate(count=Count("id"))
|
||||
.values("suffix", "count")
|
||||
)
|
||||
suffix_counter = Counter(
|
||||
"Text" if file_code.text is not None else (file_code.suffix or "file")
|
||||
for file_code in all_codes
|
||||
{((row["suffix"] or "file") or "file"): row["count"] for row in suffix_rows}
|
||||
)
|
||||
|
||||
recent_file_codes = await FileCodes.all().order_by("-created_at").limit(8)
|
||||
health_summary = await file_service.build_file_health_summary(
|
||||
await FileCodes.all(), now=now
|
||||
)
|
||||
recent_file_codes = sorted(
|
||||
all_codes,
|
||||
key=lambda file_code: file_code.created_at.timestamp()
|
||||
if file_code.created_at
|
||||
else 0,
|
||||
reverse=True,
|
||||
)[:8]
|
||||
recent_activities = await file_service.list_admin_activities(limit=8)
|
||||
return APIResponse(
|
||||
detail={
|
||||
"totalFiles": len(all_codes),
|
||||
"totalFiles": total_files,
|
||||
"storageUsed": str(all_size),
|
||||
"sysUptime": sys_start.value if sys_start else None,
|
||||
"yesterdayCount": len(yesterday_file_codes),
|
||||
"yesterdaySize": str(sum([code.size for code in yesterday_file_codes])),
|
||||
"todayCount": len(today_file_codes),
|
||||
"todaySize": str(sum([code.size for code in today_file_codes])),
|
||||
"activeCount": len(all_codes) - expired_count,
|
||||
"yesterdayCount": yesterday_count,
|
||||
"yesterdaySize": str(yesterday_size),
|
||||
"todayCount": today_count,
|
||||
"todaySize": str(today_size),
|
||||
"activeCount": total_files - expired_count,
|
||||
"expiredCount": expired_count,
|
||||
"textCount": text_count,
|
||||
"fileCount": len(all_codes) - text_count,
|
||||
"fileCount": total_files - text_count,
|
||||
"chunkedCount": chunked_count,
|
||||
"usedCount": used_count,
|
||||
"storageBackend": settings.file_storage,
|
||||
|
||||
+8
-6
@@ -2,6 +2,7 @@ import datetime
|
||||
|
||||
from fastapi import HTTPException
|
||||
from tortoise import connections
|
||||
from tortoise.functions import Sum
|
||||
|
||||
from apps.base.models import FileCodes, StorageReservation
|
||||
from core.settings import settings
|
||||
@@ -54,13 +55,14 @@ def get_storage_limit() -> int:
|
||||
|
||||
async def get_storage_usage() -> dict[str, int | None]:
|
||||
now = await get_now()
|
||||
used = await FileCodes.all().values_list("size", flat=True)
|
||||
reserved = await StorageReservation.filter(expires_at__gt=now).values_list(
|
||||
"size", flat=True
|
||||
)
|
||||
# SQL 聚合:此函数在每次上传配额检查时调用,禁止全表拉取(D3)
|
||||
used_rows = await FileCodes.all().annotate(total=Sum("size")).values("total")
|
||||
reserved_rows = await StorageReservation.filter(expires_at__gt=now).annotate(
|
||||
total=Sum("size")
|
||||
).values("total")
|
||||
limit = get_storage_limit()
|
||||
used_bytes = sum(used)
|
||||
reserved_bytes = sum(reserved)
|
||||
used_bytes = used_rows[0]["total"] or 0
|
||||
reserved_bytes = reserved_rows[0]["total"] or 0
|
||||
return {
|
||||
"limit": limit,
|
||||
"used": used_bytes,
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
"""D3 回归:dashboard 与配额使用 SQL 聚合,不再整表载入。
|
||||
|
||||
用混合数据(文本/文件、过期/活跃、分块)验证聚合结果与语义一致。
|
||||
"""
|
||||
import asyncio
|
||||
import datetime
|
||||
import unittest
|
||||
|
||||
from tortoise import Tortoise
|
||||
|
||||
from apps.admin.services import FileService
|
||||
from apps.admin.views import dashboard
|
||||
from apps.base.models import FileCodes
|
||||
from apps.base.quota import get_storage_usage
|
||||
from tests.helpers import init_memory_db
|
||||
from core.utils import get_now
|
||||
|
||||
|
||||
class DashboardQuotaAggregationTests(unittest.TestCase):
|
||||
def test_dashboard_counters_and_storage_usage_match_data(self):
|
||||
asyncio.run(self._scenario())
|
||||
|
||||
async def _scenario(self):
|
||||
await init_memory_db()
|
||||
try:
|
||||
now = await get_now()
|
||||
past = now - datetime.timedelta(days=2)
|
||||
future = now + datetime.timedelta(days=30)
|
||||
|
||||
# 活跃文件(时间式)
|
||||
await FileCodes.create(
|
||||
code="f1", size=100, expired_count=-1, expired_at=future,
|
||||
text=None, prefix="doc", suffix=".txt",
|
||||
)
|
||||
# 已过期(时间式)
|
||||
await FileCodes.create(
|
||||
code="f2", size=200, expired_count=-1, expired_at=past,
|
||||
text=None, prefix="old", suffix=".bin",
|
||||
)
|
||||
# 次数耗尽(count 式,带过期时间)
|
||||
await FileCodes.create(
|
||||
code="f3", size=300, expired_count=0, expired_at=future,
|
||||
text=None, prefix="cnt", suffix=".zip",
|
||||
)
|
||||
# 文本分享
|
||||
await FileCodes.create(
|
||||
code="t1", size=50, expired_count=-1, expired_at=future,
|
||||
text="hello",
|
||||
)
|
||||
# 永久
|
||||
await FileCodes.create(
|
||||
code="p1", size=400, expired_count=-1, expired_at=None, text=None
|
||||
)
|
||||
|
||||
detail = (await dashboard(FileService())).detail
|
||||
|
||||
self.assertEqual(detail["totalFiles"], 5)
|
||||
self.assertEqual(detail["storageUsed"], "1050")
|
||||
self.assertEqual(detail["expiredCount"], 2, "时间过期+次数耗尽都算过期")
|
||||
self.assertEqual(detail["activeCount"], 3)
|
||||
self.assertEqual(detail["textCount"], 1)
|
||||
self.assertEqual(detail["fileCount"], 4)
|
||||
self.assertEqual(detail["usedCount"], 0)
|
||||
|
||||
usage = await get_storage_usage()
|
||||
self.assertEqual(usage["used"], 1050, "配额聚合应与手工求和一致")
|
||||
self.assertIsNone(usage["available"]) # 未设 storage_limit
|
||||
finally:
|
||||
await Tortoise.close_connections()
|
||||
from tests.helpers import close_db as _c
|
||||
await _c()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user