mirror of
https://github.com/vastsa/FileCodeBox.git
synced 2026-11-20 13:29:56 +08:00
Use compatible S3 checksum settings, retry failed cleanup without reusing completed uploads, and include legacy attachment filenames. Correct admin-page URLs in both documentation languages. Refs #556, #531, #502
436 lines
16 KiB
Python
436 lines
16 KiB
Python
"""apps.base service layer: upload orchestration shared by share/chunk/presign.
|
|
|
|
Handlers in views.py keep HTTP concerns (form parsing, session lookup, status
|
|
codes for malformed requests, rate limiting); this module owns the business
|
|
workflow: quota reservation, storage writes, share-record creation, and
|
|
failure rollback/cleanup.
|
|
"""
|
|
import os
|
|
|
|
from fastapi import HTTPException, UploadFile
|
|
from fastapi.responses import FileResponse, Response, StreamingResponse
|
|
from tortoise.transactions import in_transaction
|
|
|
|
from core.logger import logger
|
|
from core.settings import settings
|
|
from core.storage import FileStorageInterface, StoredDownload, StoredFile, storages
|
|
|
|
from apps.base.file_validation import validate_upload_file
|
|
from apps.base.local_share import build_local_ref_download, is_local_ref
|
|
from apps.base.models import FileCodes, PresignUploadSession, UploadChunk
|
|
from apps.base.quota import release_storage, reserve_storage
|
|
from apps.base.utils import build_file_path, get_expire_info
|
|
|
|
import uuid
|
|
|
|
# 预签名上传会话有效期(秒)
|
|
PRESIGN_SESSION_EXPIRES = 900 # 15分钟
|
|
|
|
|
|
def stored_file_of(code: FileCodes) -> StoredFile:
|
|
"""Adapter: project an ORM FileCodes row onto the storage-layer contract."""
|
|
return StoredFile(
|
|
file_path=code.file_path,
|
|
uuid_file_name=code.uuid_file_name,
|
|
code=code.code,
|
|
prefix=code.prefix,
|
|
suffix=code.suffix,
|
|
text=code.text or "",
|
|
)
|
|
|
|
|
|
async def get_stored_download(file_code, file_storage: FileStorageInterface | None = None) -> StoredDownload:
|
|
if is_local_ref(file_code):
|
|
return build_local_ref_download(file_code)
|
|
storage = file_storage or storages[settings.file_storage]()
|
|
return await storage.get_file_response(stored_file_of(file_code))
|
|
|
|
|
|
async def rollback_saved_file(
|
|
storage: FileStorageInterface,
|
|
file_path: str,
|
|
uuid_file_name: str,
|
|
*,
|
|
context: str,
|
|
upload_id: str | None = None,
|
|
) -> None:
|
|
"""Best-effort delete of a stored file after a failed upload workflow.
|
|
|
|
Rollback failures are logged (with traceback) and swallowed: the original
|
|
error must propagate; the file, if left behind, is cleaned by the expiry
|
|
task once its orphaned record ages out.
|
|
"""
|
|
try:
|
|
await storage.delete_file(
|
|
StoredFile(file_path=file_path, uuid_file_name=uuid_file_name)
|
|
)
|
|
except Exception:
|
|
logger.warning(
|
|
"%s:回滚删除已保存文件失败%s",
|
|
context,
|
|
f" upload_id={upload_id}" if upload_id else "",
|
|
exc_info=True,
|
|
)
|
|
|
|
|
|
async def validate_file_size(file: UploadFile, max_size: int) -> int:
|
|
"""Return the upload's size, rejecting anything above max_size."""
|
|
size = file.size
|
|
if size is None:
|
|
# 无长度声明的上传(如 chunked 传输):必须走底层文件对象拿末尾偏移。
|
|
# 注意两层都不能用:UploadFile.seek 只接受单参数(TypeError);
|
|
# 底层 SpooledTemporaryFile.seek 是同步方法(不能 await)。
|
|
file.file.seek(0, 2)
|
|
size = file.file.tell()
|
|
file.file.seek(0)
|
|
if size > max_size:
|
|
max_size_mb = max_size / (1024 * 1024)
|
|
raise HTTPException(
|
|
status_code=403, detail=f"大小超过限制,最大为{max_size_mb:.2f} MB"
|
|
)
|
|
return size
|
|
|
|
|
|
def chunk_reservation_ttl() -> int:
|
|
ttl = max(1, int(getattr(settings, "chunk_expire_hours", 24))) * 3600
|
|
return ttl
|
|
|
|
|
|
class FileUploadService:
|
|
"""统一的文件上传服务"""
|
|
|
|
@staticmethod
|
|
def _storage() -> FileStorageInterface:
|
|
return storages[settings.file_storage]()
|
|
|
|
@staticmethod
|
|
async def generate_file_path(
|
|
file_name: str, upload_id: str | None = None
|
|
) -> tuple[str, str, str, str, str]:
|
|
"""Delegates path generation to apps.base.utils.build_file_path."""
|
|
return await build_file_path(file_name, upload_id or uuid.uuid4().hex)
|
|
|
|
@staticmethod
|
|
async def create_file_record(
|
|
file_name: str,
|
|
file_size: int,
|
|
file_path: str,
|
|
expire_value: int,
|
|
expire_style: str,
|
|
**extra_fields,
|
|
) -> str:
|
|
"""统一创建FileCodes记录,返回code"""
|
|
expired_at, expired_count, used_count, code = await get_expire_info(
|
|
expire_value, expire_style
|
|
)
|
|
prefix, suffix = os.path.splitext(file_name)
|
|
|
|
await FileCodes.create(
|
|
code=code,
|
|
prefix=prefix,
|
|
suffix=suffix,
|
|
uuid_file_name=file_name,
|
|
file_path=file_path,
|
|
size=file_size,
|
|
expired_at=expired_at,
|
|
expired_count=expired_count,
|
|
used_count=used_count,
|
|
**extra_fields,
|
|
)
|
|
return code
|
|
|
|
@staticmethod
|
|
async def create_text_share(
|
|
text: str, expire_value: int, expire_style: str
|
|
) -> str:
|
|
"""文本分享:配额预留 → 建分享记录 → 释放配额。"""
|
|
text_size = len(text.encode("utf-8"))
|
|
token = f"text:{uuid.uuid4().hex}"
|
|
await reserve_storage(token, text_size, ttl_seconds=300)
|
|
try:
|
|
expired_at, expired_count, used_count, code = await get_expire_info(
|
|
expire_value, expire_style
|
|
)
|
|
await FileCodes.create(
|
|
code=code,
|
|
text=text,
|
|
expired_at=expired_at,
|
|
expired_count=expired_count,
|
|
used_count=used_count,
|
|
size=text_size,
|
|
prefix="Text",
|
|
)
|
|
finally:
|
|
await release_storage(token)
|
|
return code
|
|
|
|
@staticmethod
|
|
async def create_file_share(
|
|
file: UploadFile, *, size: int, expire_value: int, expire_style: str
|
|
) -> dict[str, str]:
|
|
"""文件分享:路径生成 → 配额预留 → 存储写入 → 建分享记录,失败回滚已存文件。"""
|
|
path, suffix, prefix, uuid_file_name, save_path = (
|
|
await FileUploadService.generate_file_path(file.filename or "")
|
|
)
|
|
token = f"file:{uuid.uuid4().hex}"
|
|
await reserve_storage(token, size, ttl_seconds=3600)
|
|
storage = FileUploadService._storage()
|
|
try:
|
|
expired_at, expired_count, used_count, code = await get_expire_info(
|
|
expire_value, expire_style
|
|
)
|
|
await storage.save_file(file.file, save_path, file.content_type)
|
|
await FileCodes.create(
|
|
code=code,
|
|
prefix=prefix,
|
|
suffix=suffix,
|
|
uuid_file_name=uuid_file_name,
|
|
file_path=path,
|
|
size=size,
|
|
expired_at=expired_at,
|
|
expired_count=expired_count,
|
|
used_count=used_count,
|
|
)
|
|
except Exception:
|
|
await rollback_saved_file(
|
|
storage, path, uuid_file_name, context="分享上传"
|
|
)
|
|
raise
|
|
finally:
|
|
await release_storage(token)
|
|
# UploadFile.filename 类型为 str | None;路径生成已用 "" 兜底,
|
|
# 响应处保持 str 以匹配 dict[str, str] 契约
|
|
return {"code": code, "name": file.filename or ""}
|
|
|
|
@staticmethod
|
|
async def complete_chunked_upload(
|
|
upload_id: str, chunk_info: UploadChunk, expire_value: int, expire_style: str
|
|
) -> dict[str, str]:
|
|
"""分片合并:配额 → 完整性/大小校验 → 合并 → 建分享记录 → 清理分片。
|
|
|
|
失败路径的配额释放与清理范围与原实现逐一对齐:
|
|
完整性校验失败仅抛 400(预留由 TTL 兜底);合并失败清理分片文件后抛 500。
|
|
"""
|
|
if chunk_info.completed:
|
|
raise HTTPException(409, "上传会话已完成,仅待清理")
|
|
storage = FileUploadService._storage()
|
|
await reserve_storage(
|
|
f"chunk:{upload_id}", chunk_info.file_size, ttl_seconds=chunk_reservation_ttl()
|
|
)
|
|
|
|
completed_chunks = await UploadChunk.filter(
|
|
upload_id=upload_id, completed=True, chunk_index__gte=0
|
|
).all()
|
|
if len(completed_chunks) != chunk_info.total_chunks:
|
|
raise HTTPException(400, "分片不完整")
|
|
|
|
# 用分片数 * chunk_size 校验最大可能大小
|
|
max_total_size = len(completed_chunks) * chunk_info.chunk_size
|
|
if max_total_size > settings.upload_size:
|
|
save_path = chunk_info.save_path
|
|
cleanup_succeeded = True
|
|
if save_path:
|
|
try:
|
|
await storage.clean_chunks(upload_id, save_path)
|
|
except Exception:
|
|
cleanup_succeeded = False
|
|
logger.warning(
|
|
"分片超限中止:清理分片文件失败 upload_id=%s",
|
|
upload_id,
|
|
exc_info=True,
|
|
)
|
|
if cleanup_succeeded:
|
|
await UploadChunk.filter(upload_id=upload_id).delete()
|
|
await release_storage(f"chunk:{upload_id}")
|
|
max_size_mb = settings.upload_size / (1024 * 1024)
|
|
raise HTTPException(
|
|
403, f"实际上传大小超过限制,最大为 {max_size_mb:.2f} MB"
|
|
)
|
|
|
|
save_path = chunk_info.save_path
|
|
path = os.path.dirname(save_path) if save_path else ""
|
|
safe_file_name = os.path.basename(save_path) if save_path else ""
|
|
prefix, suffix = os.path.splitext(safe_file_name)
|
|
|
|
try:
|
|
records = {r.chunk_index: r for r in completed_chunks}
|
|
_, file_hash = await storage.merge_chunks(
|
|
upload_id,
|
|
chunk_info.total_chunks,
|
|
chunk_info.chunk_size,
|
|
save_path,
|
|
records,
|
|
)
|
|
expired_at, expired_count, used_count, code = await get_expire_info(
|
|
expire_value, expire_style
|
|
)
|
|
# 发布记录与会话终态一起提交;即使分享过期删除,会话也不能被复用。
|
|
async with in_transaction() as connection:
|
|
await FileCodes.create(
|
|
code=code,
|
|
file_hash=file_hash, # 使用合并后计算的哈希
|
|
is_chunked=True,
|
|
upload_id=upload_id,
|
|
size=chunk_info.file_size,
|
|
expired_at=expired_at,
|
|
expired_count=expired_count,
|
|
used_count=used_count,
|
|
file_path=path,
|
|
uuid_file_name=safe_file_name,
|
|
prefix=prefix,
|
|
suffix=suffix,
|
|
using_db=connection,
|
|
)
|
|
chunk_info.completed = True
|
|
await chunk_info.save(using_db=connection, update_fields=["completed"])
|
|
try:
|
|
await storage.clean_chunks(upload_id, save_path)
|
|
except Exception:
|
|
# 分享已创建:不把清理故障伪装成合并失败,保留会话供定时任务重试。
|
|
logger.warning(
|
|
"分片合并完成:清理临时分片文件失败 upload_id=%s",
|
|
upload_id,
|
|
exc_info=True,
|
|
)
|
|
else:
|
|
await UploadChunk.filter(upload_id=upload_id).delete()
|
|
await release_storage(f"chunk:{upload_id}")
|
|
return {"code": code, "name": safe_file_name}
|
|
except ValueError as e:
|
|
raise HTTPException(400, str(e))
|
|
except Exception as e:
|
|
# 合并失败时清理临时文件
|
|
try:
|
|
await storage.clean_chunks(upload_id, save_path)
|
|
except Exception:
|
|
logger.warning(
|
|
"分片合并失败:清理临时分片文件失败 upload_id=%s",
|
|
upload_id,
|
|
exc_info=True,
|
|
)
|
|
raise HTTPException(500, f"文件合并失败: {str(e)}")
|
|
|
|
@staticmethod
|
|
async def commit_proxy_upload(
|
|
session: PresignUploadSession, file: UploadFile
|
|
) -> str:
|
|
"""预签名代理上传:配额 → 大小/类型/一致性校验 → 转存 → 建记录 → 会话清理。
|
|
|
|
校验失败不释放预留(与原实现一致,由 TTL 兜底)。
|
|
"""
|
|
await reserve_storage(
|
|
f"presign:{session.upload_id}",
|
|
session.file_size,
|
|
ttl_seconds=PRESIGN_SESSION_EXPIRES,
|
|
)
|
|
|
|
file_size = await validate_file_size(file, settings.upload_size)
|
|
await validate_upload_file(file)
|
|
if abs(file_size - session.file_size) > 1024:
|
|
raise HTTPException(400, "文件大小与声明不符")
|
|
|
|
storage = FileUploadService._storage()
|
|
try:
|
|
await storage.save_file(file.file, session.save_path, file.content_type)
|
|
except Exception as e:
|
|
raise HTTPException(500, f"文件保存失败: {str(e)}")
|
|
|
|
try:
|
|
code = await FileUploadService.create_file_record(
|
|
session.file_name,
|
|
file_size,
|
|
os.path.dirname(session.save_path),
|
|
session.expire_value,
|
|
session.expire_style,
|
|
)
|
|
except Exception:
|
|
await rollback_saved_file(
|
|
storage,
|
|
os.path.dirname(session.save_path),
|
|
os.path.basename(session.save_path),
|
|
context="预签名代理上传:记录创建失败",
|
|
upload_id=session.upload_id,
|
|
)
|
|
raise
|
|
|
|
await session.delete()
|
|
await release_storage(f"presign:{session.upload_id}")
|
|
return code
|
|
|
|
@staticmethod
|
|
async def confirm_direct_upload(session: PresignUploadSession) -> str:
|
|
"""预签名直传确认:配额 → 文件存在性 → 建记录 → 会话清理。
|
|
|
|
预留失败说明配额已耗尽,此时清理远端临时文件与会话后原样抛出。
|
|
"""
|
|
try:
|
|
await reserve_storage(
|
|
f"presign:{session.upload_id}",
|
|
session.file_size,
|
|
ttl_seconds=PRESIGN_SESSION_EXPIRES,
|
|
)
|
|
except HTTPException:
|
|
storage = FileUploadService._storage()
|
|
try:
|
|
if await storage.file_exists(session.save_path):
|
|
await storage.delete_file(
|
|
StoredFile(
|
|
file_path=os.path.dirname(session.save_path),
|
|
uuid_file_name=os.path.basename(session.save_path),
|
|
)
|
|
)
|
|
finally:
|
|
await session.delete()
|
|
await release_storage(f"presign:{session.upload_id}")
|
|
raise
|
|
|
|
storage = FileUploadService._storage()
|
|
if not await storage.file_exists(session.save_path):
|
|
raise HTTPException(404, "文件未上传或上传失败")
|
|
|
|
try:
|
|
code = await FileUploadService.create_file_record(
|
|
session.file_name,
|
|
session.file_size,
|
|
os.path.dirname(session.save_path),
|
|
session.expire_value,
|
|
session.expire_style,
|
|
)
|
|
except Exception:
|
|
await rollback_saved_file(
|
|
storage,
|
|
os.path.dirname(session.save_path),
|
|
os.path.basename(session.save_path),
|
|
context="预签名确认:记录创建失败",
|
|
upload_id=session.upload_id,
|
|
)
|
|
raise
|
|
|
|
await session.delete()
|
|
await release_storage(f"presign:{session.upload_id}")
|
|
return code
|
|
|
|
|
|
def response_from_download(download: StoredDownload):
|
|
"""Build the starlette Response for a StoredDownload (view-layer duty)."""
|
|
if download.path is not None:
|
|
return FileResponse(
|
|
download.path,
|
|
media_type=download.media_type,
|
|
headers=download.headers,
|
|
filename=download.filename,
|
|
)
|
|
if download.content is not None:
|
|
return Response(
|
|
download.content, media_type=download.media_type, headers=download.headers
|
|
)
|
|
if download.stream_factory is None:
|
|
# 数据契约要求三者至少其一;防御性 500 语义
|
|
raise ValueError("StoredDownload 缺少可用的下载载荷")
|
|
return StreamingResponse(
|
|
download.stream_factory(),
|
|
media_type=download.media_type,
|
|
headers=download.headers,
|
|
background=download.background,
|
|
)
|