mirror of
https://github.com/vastsa/FileCodeBox.git
synced 2026-11-22 06:57:56 +08:00
refactor: storage layer accepts plain StoredFile refs and caller-fetched chunk records (core/ is now ORM-free)
This commit is contained in:
1 parent
3396644da3
commit
892fb24ffc
6 files changed
+127
-77
No files matched your search
@@ -7,13 +7,14 @@ from datetime import datetime, timedelta
|
||||
from typing import Any, Optional
|
||||
|
||||
from core.response import APIResponse
|
||||
from core.storage import FileStorageInterface, storages
|
||||
from core.storage import FileStorageInterface, StoredFile, storages
|
||||
from core.settings import (
|
||||
ADMIN_SESSION_EXPIRE_MAX,
|
||||
ADMIN_SESSION_EXPIRE_MIN,
|
||||
settings,
|
||||
)
|
||||
from apps.base.config import refresh_settings
|
||||
from apps.base.services import stored_file_of
|
||||
from core.security import INTERNAL_CONFIG_KEYS, generate_jwt_secret
|
||||
from apps.base.models import FileCodes, KeyValue
|
||||
from apps.base.utils import get_expire_info, get_file_path_name
|
||||
@@ -91,7 +92,7 @@ class FileService:
|
||||
|
||||
async def _delete_file_code(self, file_code: FileCodes):
|
||||
if file_code.text is None:
|
||||
await self.file_storage.delete_file(file_code)
|
||||
await self.file_storage.delete_file(stored_file_of(file_code))
|
||||
await KeyValue.filter(key=self._file_metadata_key(file_code.id)).delete()
|
||||
await file_code.delete()
|
||||
|
||||
@@ -1440,7 +1441,7 @@ class FileService:
|
||||
if file_code.text:
|
||||
return APIResponse(detail=file_code.text)
|
||||
else:
|
||||
return await self.file_storage.get_file_response(file_code)
|
||||
return await self.file_storage.get_file_response(stored_file_of(file_code))
|
||||
|
||||
async def preview_file(self, file_id: int, max_chars: int = 4000):
|
||||
max_chars = min(max(max_chars, 1), 20000)
|
||||
@@ -1500,7 +1501,7 @@ class FileService:
|
||||
)
|
||||
except Exception:
|
||||
await self.file_storage.delete_file(
|
||||
FileCodes(file_path=path, uuid_file_name=uuid_file_name)
|
||||
StoredFile(file_path=path, uuid_file_name=uuid_file_name)
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
|
||||
+14
-2
@@ -7,7 +7,7 @@ writes, share-record creation, and failure rollback.
|
||||
import os
|
||||
|
||||
from core.logger import logger
|
||||
from core.storage import FileStorageInterface
|
||||
from core.storage import FileStorageInterface, StoredFile
|
||||
|
||||
from apps.base.models import FileCodes
|
||||
from apps.base.utils import build_file_path, get_expire_info
|
||||
@@ -15,6 +15,18 @@ from apps.base.utils import build_file_path, get_expire_info
|
||||
import uuid
|
||||
|
||||
|
||||
def stored_file_of(code: FileCodes) -> StoredFile:
|
||||
"""Adapter: project an ORM FileCodes row onto the storage-layer contract."""
|
||||
return StoredFile(
|
||||
file_path=code.file_path,
|
||||
uuid_file_name=code.uuid_file_name,
|
||||
code=code.code,
|
||||
prefix=code.prefix,
|
||||
suffix=code.suffix,
|
||||
text=code.text or "",
|
||||
)
|
||||
|
||||
|
||||
async def rollback_saved_file(
|
||||
storage: FileStorageInterface,
|
||||
file_path: str,
|
||||
@@ -31,7 +43,7 @@ async def rollback_saved_file(
|
||||
"""
|
||||
try:
|
||||
await storage.delete_file(
|
||||
FileCodes(file_path=file_path, uuid_file_name=uuid_file_name)
|
||||
StoredFile(file_path=file_path, uuid_file_name=uuid_file_name)
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
|
||||
+4
-3
@@ -18,7 +18,8 @@ from apps.base.utils import ip_limit, get_chunk_file_path_name
|
||||
from apps.base.config import refresh_settings
|
||||
from core.logger import logger
|
||||
from core.settings import settings, data_root
|
||||
from core.storage import FileStorageInterface, storages
|
||||
from core.storage import FileStorageInterface, StoredFile, storages
|
||||
from apps.base.services import stored_file_of
|
||||
from core.utils import get_now
|
||||
|
||||
|
||||
@@ -41,7 +42,7 @@ async def delete_expire_files():
|
||||
).all()
|
||||
for exp in expire_data:
|
||||
try:
|
||||
await file_storage.delete_file(exp)
|
||||
await file_storage.delete_file(stored_file_of(exp))
|
||||
except Exception as e:
|
||||
logger.error(f"删除过期文件失败 code={exp.code}: {e}")
|
||||
try:
|
||||
@@ -110,7 +111,7 @@ async def clean_expired_presign_sessions():
|
||||
try:
|
||||
if await storage.file_exists(session.save_path):
|
||||
await storage.delete_file(
|
||||
FileCodes(
|
||||
StoredFile(
|
||||
file_path=os.path.dirname(session.save_path),
|
||||
uuid_file_name=os.path.basename(session.save_path),
|
||||
)
|
||||
|
||||
+11
-7
@@ -15,7 +15,8 @@ from tortoise.expressions import Case, F, Q, When
|
||||
from apps.admin.dependencies import share_required_login
|
||||
from apps.base.models import FileCodes, UploadChunk, PresignUploadSession
|
||||
from apps.base.quota import release_storage, reserve_storage
|
||||
from apps.base.services import FileUploadService, rollback_saved_file
|
||||
from apps.base.services import FileUploadService, rollback_saved_file, stored_file_of
|
||||
from core.storage import StoredFile
|
||||
from core.logger import logger
|
||||
from apps.base.schemas import (
|
||||
SelectFileModel,
|
||||
@@ -211,7 +212,7 @@ async def build_select_detail(
|
||||
# 有次数限制的文件必须经过下载接口,第三方直链无法阻止重复使用。
|
||||
download_url = await get_proxy_file_url(file_code.code)
|
||||
else:
|
||||
download_url = await file_storage.get_file_url(file_code)
|
||||
download_url = await file_storage.get_file_url(stored_file_of(file_code))
|
||||
content = file_code.text if file_code.text is not None else None
|
||||
return {
|
||||
**metadata,
|
||||
@@ -269,7 +270,7 @@ async def get_code_file(code: str, ip: str = Depends(ip_limit["error"])):
|
||||
)
|
||||
},
|
||||
)
|
||||
return await file_storage.get_file_response(file_code)
|
||||
return await file_storage.get_file_response(stored_file_of(file_code))
|
||||
|
||||
|
||||
@share_api.post("/select/")
|
||||
@@ -314,7 +315,7 @@ async def download_file(key: str, code: str, ip: str = Depends(ip_limit["error"]
|
||||
return (
|
||||
APIResponse(detail=file_code.text)
|
||||
if file_code.text
|
||||
else await file_storage.get_file_response(file_code)
|
||||
else await file_storage.get_file_response(stored_file_of(file_code))
|
||||
)
|
||||
|
||||
|
||||
@@ -615,7 +616,10 @@ async def complete_upload(
|
||||
|
||||
try:
|
||||
# 合并文件并计算哈希
|
||||
_, file_hash = await storage.merge_chunks(upload_id, chunk_info, save_path)
|
||||
records = {r.chunk_index: r for r in completed_chunks_list}
|
||||
_, file_hash = await storage.merge_chunks(
|
||||
upload_id, chunk_info.total_chunks, chunk_info.chunk_size, save_path, records
|
||||
)
|
||||
# 创建文件记录
|
||||
expired_at, expired_count, used_count, code = await get_expire_info(
|
||||
data.expire_value, data.expire_style
|
||||
@@ -808,7 +812,7 @@ async def presign_upload_confirm(upload_id: str, ip: str = Depends(ip_limit["upl
|
||||
try:
|
||||
if await storage.file_exists(session.save_path):
|
||||
await storage.delete_file(
|
||||
FileCodes(
|
||||
StoredFile(
|
||||
file_path=os.path.dirname(session.save_path),
|
||||
uuid_file_name=os.path.basename(session.save_path),
|
||||
)
|
||||
@@ -879,7 +883,7 @@ async def presign_upload_cancel(upload_id: str):
|
||||
storage: FileStorageInterface = storages[settings.file_storage]()
|
||||
try:
|
||||
if await storage.file_exists(session.save_path):
|
||||
temp_file_code = FileCodes(
|
||||
temp_file_code = StoredFile(
|
||||
file_path=os.path.dirname(session.save_path),
|
||||
uuid_file_name=os.path.basename(session.save_path),
|
||||
)
|
||||
|
||||
+78
-56
@@ -16,25 +16,47 @@ import aiohttp
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
import datetime
|
||||
from dataclasses import dataclass
|
||||
import re
|
||||
import aioboto3
|
||||
from botocore.config import Config
|
||||
from fastapi import HTTPException, Response, UploadFile
|
||||
from core.response import APIResponse
|
||||
from core.settings import data_root, settings
|
||||
from apps.base.models import FileCodes, UploadChunk
|
||||
from core.utils import get_file_url, sanitize_filename
|
||||
from fastapi.responses import FileResponse, StreamingResponse
|
||||
from starlette.background import BackgroundTask
|
||||
|
||||
|
||||
@dataclass
|
||||
class StoredFile:
|
||||
"""Plain, framework- and ORM-free description of a stored file.
|
||||
|
||||
Storage backends accept this instead of ORM models so core/ never imports
|
||||
apps/. Callers (views/tasks/services) build it from their own records.
|
||||
"""
|
||||
|
||||
file_path: str
|
||||
uuid_file_name: str
|
||||
code: str = ""
|
||||
prefix: str = ""
|
||||
suffix: str = ""
|
||||
text: str = ""
|
||||
|
||||
def get_file_path(self) -> str:
|
||||
return f"{self.file_path}/{self.uuid_file_name}"
|
||||
|
||||
|
||||
class FileStorageInterface:
|
||||
|
||||
async def _load_chunk_record(self, upload_id: str, index: int) -> UploadChunk:
|
||||
"""Fetch the UploadChunk row for chunk `index`, shared by all backends."""
|
||||
chunk_record = await UploadChunk.filter(
|
||||
upload_id=upload_id, chunk_index=index
|
||||
).first()
|
||||
@staticmethod
|
||||
def _get_chunk_record(chunk_records: dict, index: int):
|
||||
"""Look up the caller-provided record for chunk `index`.
|
||||
|
||||
Records are plain objects exposing ``chunk_hash``; fetching them from
|
||||
the DB is the caller's job (keeps storage ORM-free).
|
||||
"""
|
||||
chunk_record = chunk_records.get(index)
|
||||
if not chunk_record:
|
||||
raise ValueError(f"分片{index}记录不存在")
|
||||
return chunk_record
|
||||
@@ -42,7 +64,7 @@ class FileStorageInterface:
|
||||
def _verify_and_hash_chunk(
|
||||
self,
|
||||
index: int,
|
||||
chunk_record: UploadChunk,
|
||||
chunk_record,
|
||||
chunk_data: bytes,
|
||||
file_sha256,
|
||||
) -> None:
|
||||
@@ -61,13 +83,13 @@ class FileStorageInterface:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def delete_file(self, file_code: FileCodes):
|
||||
async def delete_file(self, file_code: StoredFile):
|
||||
"""
|
||||
删除文件
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_file_url(self, file_code: FileCodes):
|
||||
async def get_file_url(self, file_code: StoredFile):
|
||||
"""
|
||||
获取文件分享的url
|
||||
|
||||
@@ -76,7 +98,7 @@ class FileStorageInterface:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_file_response(self, file_code: FileCodes):
|
||||
async def get_file_response(self, file_code: StoredFile):
|
||||
"""
|
||||
获取文件响应
|
||||
|
||||
@@ -96,7 +118,7 @@ class FileStorageInterface:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def merge_chunks(self, upload_id: str, chunk_info: UploadChunk, save_path: str) -> tuple[str, str]:
|
||||
async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]:
|
||||
"""
|
||||
合并分片文件并返回文件路径和完整哈希值
|
||||
:param upload_id: 上传会话ID
|
||||
@@ -169,16 +191,16 @@ class SystemFileStorage(FileStorageInterface):
|
||||
safe_save_path.parent.mkdir(parents=True)
|
||||
await asyncio.to_thread(self._save, file.file, safe_save_path)
|
||||
|
||||
async def delete_file(self, file_code: FileCodes):
|
||||
save_path = self._resolve_safe_path(await file_code.get_file_path())
|
||||
async def delete_file(self, file_code: StoredFile):
|
||||
save_path = self._resolve_safe_path(file_code.get_file_path())
|
||||
if save_path.exists():
|
||||
save_path.unlink()
|
||||
|
||||
async def get_file_url(self, file_code: FileCodes):
|
||||
async def get_file_url(self, file_code: StoredFile):
|
||||
return await get_file_url(file_code.code)
|
||||
|
||||
async def get_file_response(self, file_code: FileCodes):
|
||||
file_path = self._resolve_safe_path(await file_code.get_file_path())
|
||||
async def get_file_response(self, file_code: StoredFile):
|
||||
file_path = self._resolve_safe_path(file_code.get_file_path())
|
||||
if not file_path.exists():
|
||||
return APIResponse(code=404, detail="文件已过期删除")
|
||||
filename = f"{file_code.prefix}{file_code.suffix}"
|
||||
@@ -229,7 +251,7 @@ class SystemFileStorage(FileStorageInterface):
|
||||
temp_path.unlink()
|
||||
raise e
|
||||
|
||||
async def merge_chunks(self, upload_id: str, chunk_info: UploadChunk, save_path: str) -> tuple[str, str]:
|
||||
async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]:
|
||||
"""
|
||||
合并本地文件系统的分片文件并返回文件路径和完整哈希值
|
||||
:param upload_id: 上传会话ID
|
||||
@@ -248,9 +270,9 @@ class SystemFileStorage(FileStorageInterface):
|
||||
temp_output = output_path.with_suffix('.merging')
|
||||
try:
|
||||
async with aiofiles.open(temp_output, "wb") as out_file:
|
||||
for i in range(chunk_info.total_chunks):
|
||||
for i in range(total_chunks):
|
||||
# 获取分片记录
|
||||
chunk_record = await self._load_chunk_record(upload_id, i)
|
||||
chunk_record = self._get_chunk_record(chunk_records, i)
|
||||
chunk_path = chunk_base_dir / f"{i}.part"
|
||||
if not chunk_path.exists():
|
||||
raise ValueError(f"分片{i}文件不存在")
|
||||
@@ -351,13 +373,13 @@ class S3FileStorage(FileStorageInterface):
|
||||
ExtraArgs={"ContentType": file.content_type or "application/octet-stream"},
|
||||
)
|
||||
|
||||
async def delete_file(self, file_code: FileCodes):
|
||||
async def delete_file(self, file_code: StoredFile):
|
||||
async with self._client() as s3:
|
||||
await s3.delete_object(
|
||||
Bucket=self.bucket_name, Key=await file_code.get_file_path()
|
||||
Bucket=self.bucket_name, Key=file_code.get_file_path()
|
||||
)
|
||||
|
||||
async def get_file_response(self, file_code: FileCodes):
|
||||
async def get_file_response(self, file_code: StoredFile):
|
||||
try:
|
||||
filename = file_code.prefix + file_code.suffix
|
||||
content_length = None # 初始化为 None,表示未知大小
|
||||
@@ -367,7 +389,7 @@ class S3FileStorage(FileStorageInterface):
|
||||
try:
|
||||
head_response = await s3.head_object(
|
||||
Bucket=self.bucket_name,
|
||||
Key=await file_code.get_file_path()
|
||||
Key=file_code.get_file_path()
|
||||
)
|
||||
# 从HEAD响应中获取Content-Length
|
||||
if 'ContentLength' in head_response:
|
||||
@@ -382,7 +404,7 @@ class S3FileStorage(FileStorageInterface):
|
||||
"get_object",
|
||||
Params={
|
||||
"Bucket": self.bucket_name,
|
||||
"Key": await file_code.get_file_path(),
|
||||
"Key": file_code.get_file_path(),
|
||||
},
|
||||
ExpiresIn=3600,
|
||||
)
|
||||
@@ -427,7 +449,7 @@ class S3FileStorage(FileStorageInterface):
|
||||
except Exception:
|
||||
raise HTTPException(status_code=503, detail="服务代理下载异常,请稍后再试")
|
||||
|
||||
async def get_file_url(self, file_code: FileCodes):
|
||||
async def get_file_url(self, file_code: StoredFile):
|
||||
if file_code.prefix == "文本分享":
|
||||
return file_code.text
|
||||
if self.proxy:
|
||||
@@ -438,7 +460,7 @@ class S3FileStorage(FileStorageInterface):
|
||||
"get_object",
|
||||
Params={
|
||||
"Bucket": self.bucket_name,
|
||||
"Key": await file_code.get_file_path(),
|
||||
"Key": file_code.get_file_path(),
|
||||
},
|
||||
ExpiresIn=3600,
|
||||
)
|
||||
@@ -462,7 +484,7 @@ class S3FileStorage(FileStorageInterface):
|
||||
}
|
||||
)
|
||||
|
||||
async def merge_chunks(self, upload_id: str, chunk_info: UploadChunk, save_path: str) -> tuple[str, str]:
|
||||
async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]:
|
||||
"""
|
||||
合并 S3 上的分片文件
|
||||
使用 S3 的 multipart upload API 实现流式合并,避免内存问题
|
||||
@@ -482,9 +504,9 @@ class S3FileStorage(FileStorageInterface):
|
||||
|
||||
try:
|
||||
# 按顺序读取、验证并上传每个分片
|
||||
for i in range(chunk_info.total_chunks):
|
||||
for i in range(total_chunks):
|
||||
chunk_key = f"{chunk_dir}/{i}.part"
|
||||
chunk_record = await self._load_chunk_record(upload_id, i)
|
||||
chunk_record = self._get_chunk_record(chunk_records, i)
|
||||
|
||||
try:
|
||||
response = await s3.get_object(
|
||||
@@ -683,8 +705,8 @@ class OneDriveFileStorage(FileStorageInterface):
|
||||
else:
|
||||
raise e
|
||||
|
||||
async def delete_file(self, file_code: FileCodes):
|
||||
await asyncio.to_thread(self._delete, await file_code.get_file_path())
|
||||
async def delete_file(self, file_code: StoredFile):
|
||||
await asyncio.to_thread(self._delete, file_code.get_file_path())
|
||||
|
||||
def _convert_link_to_download_link(self, link):
|
||||
p1 = re.search(r"https://(.+)\.sharepoint\.com", link).group(1)
|
||||
@@ -705,11 +727,11 @@ class OneDriveFileStorage(FileStorageInterface):
|
||||
).execute_query()
|
||||
return self._convert_link_to_download_link(permission.link.webUrl)
|
||||
|
||||
async def get_file_response(self, file_code: FileCodes):
|
||||
async def get_file_response(self, file_code: StoredFile):
|
||||
try:
|
||||
filename = file_code.prefix + file_code.suffix
|
||||
link = await asyncio.to_thread(
|
||||
self._get_file_url, await file_code.get_file_path(), filename
|
||||
self._get_file_url, file_code.get_file_path(), filename
|
||||
)
|
||||
|
||||
content_length = None # 初始化为 None,表示未知大小
|
||||
@@ -761,13 +783,13 @@ class OneDriveFileStorage(FileStorageInterface):
|
||||
except Exception:
|
||||
raise HTTPException(status_code=503, detail="服务代理下载异常,请稍后再试")
|
||||
|
||||
async def get_file_url(self, file_code: FileCodes):
|
||||
async def get_file_url(self, file_code: StoredFile):
|
||||
if self.proxy:
|
||||
return await get_file_url(file_code.code)
|
||||
else:
|
||||
return await asyncio.to_thread(
|
||||
self._get_file_url,
|
||||
await file_code.get_file_path(),
|
||||
file_code.get_file_path(),
|
||||
f"{file_code.prefix}{file_code.suffix}",
|
||||
)
|
||||
|
||||
@@ -823,7 +845,7 @@ class OneDriveFileStorage(FileStorageInterface):
|
||||
|
||||
current_folder.upload(filename, data).execute_query()
|
||||
|
||||
async def merge_chunks(self, upload_id: str, chunk_info: UploadChunk, save_path: str) -> tuple[str, str]:
|
||||
async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]:
|
||||
"""合并 OneDrive 上的分片文件,使用临时文件避免内存问题"""
|
||||
file_sha256 = hashlib.sha256()
|
||||
chunk_dir = str(Path(save_path).parent / "chunks" / upload_id)
|
||||
@@ -834,9 +856,9 @@ class OneDriveFileStorage(FileStorageInterface):
|
||||
|
||||
try:
|
||||
async with aiofiles.open(temp_path, 'wb') as out_file:
|
||||
for i in range(chunk_info.total_chunks):
|
||||
for i in range(total_chunks):
|
||||
chunk_path = f"{chunk_dir}/{i}.part"
|
||||
chunk_record = await self._load_chunk_record(upload_id, i)
|
||||
chunk_record = self._get_chunk_record(chunk_records, i)
|
||||
|
||||
try:
|
||||
chunk_data = await asyncio.to_thread(self._read_chunk, chunk_path)
|
||||
@@ -916,20 +938,20 @@ class OpenDALFileStorage(FileStorageInterface):
|
||||
content = await asyncio.to_thread(file.file.read)
|
||||
await self.operator.write(save_path, content)
|
||||
|
||||
async def delete_file(self, file_code: FileCodes):
|
||||
await self.operator.delete(await file_code.get_file_path())
|
||||
async def delete_file(self, file_code: StoredFile):
|
||||
await self.operator.delete(file_code.get_file_path())
|
||||
|
||||
async def get_file_url(self, file_code: FileCodes):
|
||||
async def get_file_url(self, file_code: StoredFile):
|
||||
return await get_file_url(file_code.code)
|
||||
|
||||
async def get_file_response(self, file_code: FileCodes):
|
||||
async def get_file_response(self, file_code: StoredFile):
|
||||
try:
|
||||
filename = file_code.prefix + file_code.suffix
|
||||
content_length = None # 初始化为 None,表示未知大小
|
||||
|
||||
# 尝试获取文件大小
|
||||
try:
|
||||
stat_result = await self.operator.stat(await file_code.get_file_path())
|
||||
stat_result = await self.operator.stat(file_code.get_file_path())
|
||||
if hasattr(stat_result, 'content_length') and stat_result.content_length:
|
||||
content_length = stat_result.content_length
|
||||
elif hasattr(stat_result, 'size') and stat_result.size:
|
||||
@@ -941,10 +963,10 @@ class OpenDALFileStorage(FileStorageInterface):
|
||||
# 尝试使用流式读取器
|
||||
try:
|
||||
# OpenDAL 可能提供 reader 方法返回一个异步读取器
|
||||
reader = await self.operator.reader(await file_code.get_file_path())
|
||||
reader = await self.operator.reader(file_code.get_file_path())
|
||||
except AttributeError:
|
||||
# 如果 reader 方法不存在,回退到全量读取(兼容旧版本)
|
||||
content = await self.operator.read(await file_code.get_file_path())
|
||||
content = await self.operator.read(file_code.get_file_path())
|
||||
encoded_filename = quote(filename, safe='')
|
||||
headers = {
|
||||
"Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}"
|
||||
@@ -983,7 +1005,7 @@ class OpenDALFileStorage(FileStorageInterface):
|
||||
chunk_path = str(Path(save_path).parent / "chunks" / upload_id / f"{chunk_index}.part")
|
||||
await self.operator.write(chunk_path, chunk_data)
|
||||
|
||||
async def merge_chunks(self, upload_id: str, chunk_info: UploadChunk, save_path: str) -> tuple[str, str]:
|
||||
async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]:
|
||||
"""合并 OpenDAL 存储上的分片文件,使用临时文件避免内存问题"""
|
||||
file_sha256 = hashlib.sha256()
|
||||
chunk_dir = str(Path(save_path).parent / "chunks" / upload_id)
|
||||
@@ -994,9 +1016,9 @@ class OpenDALFileStorage(FileStorageInterface):
|
||||
|
||||
try:
|
||||
async with aiofiles.open(temp_path, 'wb') as out_file:
|
||||
for i in range(chunk_info.total_chunks):
|
||||
for i in range(total_chunks):
|
||||
chunk_path = f"{chunk_dir}/{i}.part"
|
||||
chunk_record = await self._load_chunk_record(upload_id, i)
|
||||
chunk_record = self._get_chunk_record(chunk_records, i)
|
||||
|
||||
try:
|
||||
chunk_data = await self.operator.read(chunk_path)
|
||||
@@ -1146,9 +1168,9 @@ class WebDAVFileStorage(FileStorageInterface):
|
||||
raise HTTPException(
|
||||
status_code=503, detail=f"WebDAV连接异常: {str(e)}")
|
||||
|
||||
async def delete_file(self, file_code: FileCodes):
|
||||
async def delete_file(self, file_code: StoredFile):
|
||||
"""删除WebDAV文件及空目录"""
|
||||
file_path = await file_code.get_file_path()
|
||||
file_path = file_code.get_file_path()
|
||||
url = self._build_url(file_path)
|
||||
try:
|
||||
async with aiohttp.ClientSession(auth=self.auth) as session:
|
||||
@@ -1168,14 +1190,14 @@ class WebDAVFileStorage(FileStorageInterface):
|
||||
raise HTTPException(
|
||||
status_code=503, detail=f"WebDAV连接异常: {str(e)}")
|
||||
|
||||
async def get_file_url(self, file_code: FileCodes):
|
||||
async def get_file_url(self, file_code: StoredFile):
|
||||
return await get_file_url(file_code.code)
|
||||
|
||||
async def get_file_response(self, file_code: FileCodes):
|
||||
async def get_file_response(self, file_code: StoredFile):
|
||||
"""获取文件响应(代理模式)"""
|
||||
try:
|
||||
filename = file_code.prefix + file_code.suffix
|
||||
url = self._build_url(await file_code.get_file_path())
|
||||
url = self._build_url(file_code.get_file_path())
|
||||
content_length = None # 初始化为 None,表示未知大小
|
||||
|
||||
# 创建ClientSession并复用(包含认证头)
|
||||
@@ -1244,7 +1266,7 @@ class WebDAVFileStorage(FileStorageInterface):
|
||||
detail=f"分片上传失败: {content[:200]}"
|
||||
)
|
||||
|
||||
async def merge_chunks(self, upload_id: str, chunk_info: UploadChunk, save_path: str) -> tuple[str, str]:
|
||||
async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]:
|
||||
"""
|
||||
合并 WebDAV 上的分片文件
|
||||
使用临时文件避免内存问题
|
||||
@@ -1260,12 +1282,12 @@ class WebDAVFileStorage(FileStorageInterface):
|
||||
async with aiohttp.ClientSession(auth=self.auth) as session:
|
||||
# 按顺序读取并验证每个分片,写入临时文件
|
||||
async with aiofiles.open(temp_path, 'wb') as out_file:
|
||||
for i in range(chunk_info.total_chunks):
|
||||
for i in range(total_chunks):
|
||||
chunk_path = f"{chunk_dir}/{i}.part"
|
||||
chunk_url = self._build_url(chunk_path)
|
||||
|
||||
# 获取分片记录
|
||||
chunk_record = await self._load_chunk_record(upload_id, i)
|
||||
chunk_record = self._get_chunk_record(chunk_records, i)
|
||||
|
||||
# 下载分片数据
|
||||
async with session.get(chunk_url) as resp:
|
||||
|
||||
@@ -3,6 +3,9 @@
|
||||
The happy path is already exercised end-to-end by the ASGI journey tests;
|
||||
these target the guard rails: missing chunk records, missing chunk files,
|
||||
hash mismatches, and that a failed merge leaves no half-written output.
|
||||
|
||||
Contract note: storage is ORM-free — the caller fetches chunk records and
|
||||
passes them as a ``{chunk_index: record}`` dict keyed by index.
|
||||
"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
@@ -38,7 +41,11 @@ class MergeChunksFailureTests(SettingsOverrideMixin, unittest.TestCase):
|
||||
(chunk_dir / "0.part").write_bytes(chunk_a)
|
||||
(chunk_dir / "1.part").write_bytes(chunk_b)
|
||||
|
||||
session = await UploadChunk.create(
|
||||
async def records():
|
||||
rows = await UploadChunk.filter(upload_id="uid-merge").all()
|
||||
return {r.chunk_index: r for r in rows}
|
||||
|
||||
await UploadChunk.create(
|
||||
upload_id="uid-merge",
|
||||
chunk_index=-1,
|
||||
chunk_hash="0" * 64,
|
||||
@@ -62,7 +69,7 @@ class MergeChunksFailureTests(SettingsOverrideMixin, unittest.TestCase):
|
||||
|
||||
# 1) 分片 1 记录缺失
|
||||
with self.assertRaisesRegex(ValueError, "分片1记录不存在"):
|
||||
await storage.merge_chunks("uid-merge", session, save_path)
|
||||
await storage.merge_chunks("uid-merge", 2, 4, save_path, await records())
|
||||
self.assertFalse((Path(tmpdir) / save_path).exists(), "失败不得产出半成品文件")
|
||||
|
||||
# 2) 分片 1 记录存在但哈希与文件内容不匹配
|
||||
@@ -78,13 +85,13 @@ class MergeChunksFailureTests(SettingsOverrideMixin, unittest.TestCase):
|
||||
completed=True,
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "哈希不匹配"):
|
||||
await storage.merge_chunks("uid-merge", session, save_path)
|
||||
await storage.merge_chunks("uid-merge", 2, 4, save_path, await records())
|
||||
self.assertFalse((Path(tmpdir) / save_path).exists())
|
||||
|
||||
# 3) 记录匹配但磁盘文件缺失
|
||||
(chunk_dir / "1.part").unlink()
|
||||
with self.assertRaisesRegex(ValueError, "分片1文件不存在"):
|
||||
await storage.merge_chunks("uid-merge", session, save_path)
|
||||
await storage.merge_chunks("uid-merge", 2, 4, save_path, await records())
|
||||
|
||||
# 4) happy path:修正分片1记录哈希后合并成功,且输出哈希正确
|
||||
(chunk_dir / "1.part").write_bytes(chunk_b)
|
||||
@@ -93,7 +100,9 @@ class MergeChunksFailureTests(SettingsOverrideMixin, unittest.TestCase):
|
||||
).first()
|
||||
wrong_record.chunk_hash = _sha(chunk_b)
|
||||
await wrong_record.save()
|
||||
out_path, file_hash = await storage.merge_chunks("uid-merge", session, save_path)
|
||||
out_path, file_hash = await storage.merge_chunks(
|
||||
"uid-merge", 2, 4, save_path, await records()
|
||||
)
|
||||
self.assertTrue(Path(out_path).exists())
|
||||
self.assertEqual(file_hash, _sha(chunk_a + chunk_b))
|
||||
self.assertFalse((chunk_dir / "1.part").with_suffix(".merging").exists(), "临时合并文件应被重命名消费")
|
||||
@@ -104,5 +113,6 @@ class MergeChunksFailureTests(SettingsOverrideMixin, unittest.TestCase):
|
||||
finally:
|
||||
await close_db()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user