mirror of
https://github.com/vastsa/FileCodeBox.git
synced 2026-11-22 06:57:56 +08:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
42b2c29ccb | ||
|
|
f9cc16c805 | ||
|
|
8d3e4ad904 | ||
|
|
381a539c92 |
No files matched your search
@@ -1,3 +1,3 @@
|
|||||||
{
|
{
|
||||||
".": "2.8.4"
|
".": "2.9.0"
|
||||||
}
|
}
|
||||||
@@ -13,12 +13,11 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from urllib.parse import quote
|
|
||||||
|
|
||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
|
|
||||||
from core.errors import StorageError
|
from core.errors import StorageError
|
||||||
from core.storage import StoredDownload
|
from core.storage import StoredDownload, build_attachment_headers
|
||||||
|
|
||||||
LOCAL_REF_MARKER = "local-ref"
|
LOCAL_REF_MARKER = "local-ref"
|
||||||
MAX_RELPATH_LEN = 255
|
MAX_RELPATH_LEN = 255
|
||||||
@@ -97,8 +96,7 @@ def build_local_ref_download(file_code) -> StoredDownload:
|
|||||||
filename = f"{getattr(file_code, 'prefix', '') or ''}{getattr(file_code, 'suffix', '') or ''}"
|
filename = f"{getattr(file_code, 'prefix', '') or ''}{getattr(file_code, 'suffix', '') or ''}"
|
||||||
if not filename:
|
if not filename:
|
||||||
filename = path.name
|
filename = path.name
|
||||||
encoded_filename = quote(filename, safe="")
|
headers = build_attachment_headers(filename)
|
||||||
headers = {"Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}"}
|
|
||||||
try:
|
try:
|
||||||
headers["Content-Length"] = str(path.stat().st_size)
|
headers["Content-Length"] = str(path.stat().st_size)
|
||||||
except OSError:
|
except OSError:
|
||||||
|
|||||||
+38
-18
@@ -9,6 +9,7 @@ import os
|
|||||||
|
|
||||||
from fastapi import HTTPException, UploadFile
|
from fastapi import HTTPException, UploadFile
|
||||||
from fastapi.responses import FileResponse, Response, StreamingResponse
|
from fastapi.responses import FileResponse, Response, StreamingResponse
|
||||||
|
from tortoise.transactions import in_transaction
|
||||||
|
|
||||||
from core.logger import logger
|
from core.logger import logger
|
||||||
from core.settings import settings
|
from core.settings import settings
|
||||||
@@ -210,13 +211,15 @@ class FileUploadService:
|
|||||||
失败路径的配额释放与清理范围与原实现逐一对齐:
|
失败路径的配额释放与清理范围与原实现逐一对齐:
|
||||||
完整性校验失败仅抛 400(预留由 TTL 兜底);合并失败清理分片文件后抛 500。
|
完整性校验失败仅抛 400(预留由 TTL 兜底);合并失败清理分片文件后抛 500。
|
||||||
"""
|
"""
|
||||||
|
if chunk_info.completed:
|
||||||
|
raise HTTPException(409, "上传会话已完成,仅待清理")
|
||||||
storage = FileUploadService._storage()
|
storage = FileUploadService._storage()
|
||||||
await reserve_storage(
|
await reserve_storage(
|
||||||
f"chunk:{upload_id}", chunk_info.file_size, ttl_seconds=chunk_reservation_ttl()
|
f"chunk:{upload_id}", chunk_info.file_size, ttl_seconds=chunk_reservation_ttl()
|
||||||
)
|
)
|
||||||
|
|
||||||
completed_chunks = await UploadChunk.filter(
|
completed_chunks = await UploadChunk.filter(
|
||||||
upload_id=upload_id, completed=True
|
upload_id=upload_id, completed=True, chunk_index__gte=0
|
||||||
).all()
|
).all()
|
||||||
if len(completed_chunks) != chunk_info.total_chunks:
|
if len(completed_chunks) != chunk_info.total_chunks:
|
||||||
raise HTTPException(400, "分片不完整")
|
raise HTTPException(400, "分片不完整")
|
||||||
@@ -225,16 +228,19 @@ class FileUploadService:
|
|||||||
max_total_size = len(completed_chunks) * chunk_info.chunk_size
|
max_total_size = len(completed_chunks) * chunk_info.chunk_size
|
||||||
if max_total_size > settings.upload_size:
|
if max_total_size > settings.upload_size:
|
||||||
save_path = chunk_info.save_path
|
save_path = chunk_info.save_path
|
||||||
|
cleanup_succeeded = True
|
||||||
if save_path:
|
if save_path:
|
||||||
try:
|
try:
|
||||||
await storage.clean_chunks(upload_id, save_path)
|
await storage.clean_chunks(upload_id, save_path)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
cleanup_succeeded = False
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"分片超限中止:清理分片文件失败 upload_id=%s",
|
"分片超限中止:清理分片文件失败 upload_id=%s",
|
||||||
upload_id,
|
upload_id,
|
||||||
exc_info=True,
|
exc_info=True,
|
||||||
)
|
)
|
||||||
await UploadChunk.filter(upload_id=upload_id).delete()
|
if cleanup_succeeded:
|
||||||
|
await UploadChunk.filter(upload_id=upload_id).delete()
|
||||||
await release_storage(f"chunk:{upload_id}")
|
await release_storage(f"chunk:{upload_id}")
|
||||||
max_size_mb = settings.upload_size / (1024 * 1024)
|
max_size_mb = settings.upload_size / (1024 * 1024)
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
@@ -258,22 +264,36 @@ class FileUploadService:
|
|||||||
expired_at, expired_count, used_count, code = await get_expire_info(
|
expired_at, expired_count, used_count, code = await get_expire_info(
|
||||||
expire_value, expire_style
|
expire_value, expire_style
|
||||||
)
|
)
|
||||||
await FileCodes.create(
|
# 发布记录与会话终态一起提交;即使分享过期删除,会话也不能被复用。
|
||||||
code=code,
|
async with in_transaction() as connection:
|
||||||
file_hash=file_hash, # 使用合并后计算的哈希
|
await FileCodes.create(
|
||||||
is_chunked=True,
|
code=code,
|
||||||
upload_id=upload_id,
|
file_hash=file_hash, # 使用合并后计算的哈希
|
||||||
size=chunk_info.file_size,
|
is_chunked=True,
|
||||||
expired_at=expired_at,
|
upload_id=upload_id,
|
||||||
expired_count=expired_count,
|
size=chunk_info.file_size,
|
||||||
used_count=used_count,
|
expired_at=expired_at,
|
||||||
file_path=path,
|
expired_count=expired_count,
|
||||||
uuid_file_name=safe_file_name,
|
used_count=used_count,
|
||||||
prefix=prefix,
|
file_path=path,
|
||||||
suffix=suffix,
|
uuid_file_name=safe_file_name,
|
||||||
)
|
prefix=prefix,
|
||||||
await storage.clean_chunks(upload_id, save_path)
|
suffix=suffix,
|
||||||
await UploadChunk.filter(upload_id=upload_id).delete()
|
using_db=connection,
|
||||||
|
)
|
||||||
|
chunk_info.completed = True
|
||||||
|
await chunk_info.save(using_db=connection, update_fields=["completed"])
|
||||||
|
try:
|
||||||
|
await storage.clean_chunks(upload_id, save_path)
|
||||||
|
except Exception:
|
||||||
|
# 分享已创建:不把清理故障伪装成合并失败,保留会话供定时任务重试。
|
||||||
|
logger.warning(
|
||||||
|
"分片合并完成:清理临时分片文件失败 upload_id=%s",
|
||||||
|
upload_id,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
await UploadChunk.filter(upload_id=upload_id).delete()
|
||||||
await release_storage(f"chunk:{upload_id}")
|
await release_storage(f"chunk:{upload_id}")
|
||||||
return {"code": code, "name": safe_file_name}
|
return {"code": code, "name": safe_file_name}
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ def build_public_config() -> dict:
|
|||||||
"notify_content": settings.notify_content,
|
"notify_content": settings.notify_content,
|
||||||
"show_admin_address": normalize_public_flag(settings.show_admin_addr),
|
"show_admin_address": normalize_public_flag(settings.show_admin_addr),
|
||||||
"max_save_seconds": settings.max_save_seconds,
|
"max_save_seconds": settings.max_save_seconds,
|
||||||
|
"background": str(settings.background or ""),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -80,6 +80,8 @@ async def clean_incomplete_uploads():
|
|||||||
logger.error(
|
logger.error(
|
||||||
f"清理分片文件失败 upload_id={session.upload_id}: {e}"
|
f"清理分片文件失败 upload_id={session.upload_id}: {e}"
|
||||||
)
|
)
|
||||||
|
# 存储清理失败(包括部分删除)时保留记录,下轮继续重试。
|
||||||
|
continue
|
||||||
|
|
||||||
try:
|
try:
|
||||||
await UploadChunk.filter(upload_id=session.upload_id).delete()
|
await UploadChunk.filter(upload_id=session.upload_id).delete()
|
||||||
|
|||||||
+12
-10
@@ -3,7 +3,7 @@ import hmac
|
|||||||
import os
|
import os
|
||||||
import uuid
|
import uuid
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from urllib.parse import quote, unquote
|
from urllib.parse import unquote
|
||||||
|
|
||||||
from typing import Optional, Tuple, Union
|
from typing import Optional, Tuple, Union
|
||||||
|
|
||||||
@@ -24,7 +24,7 @@ from apps.base.services import (
|
|||||||
stored_file_of,
|
stored_file_of,
|
||||||
validate_file_size,
|
validate_file_size,
|
||||||
)
|
)
|
||||||
from core.storage import StoredFile
|
from core.storage import StoredFile, build_attachment_headers
|
||||||
from core.logger import logger
|
from core.logger import logger
|
||||||
from apps.base.schemas import (
|
from apps.base.schemas import (
|
||||||
SelectFileModel,
|
SelectFileModel,
|
||||||
@@ -212,11 +212,7 @@ async def get_code_file(code: str, ip: str = Depends(ip_limit["error"])):
|
|||||||
return Response(
|
return Response(
|
||||||
content=file_code.text,
|
content=file_code.text,
|
||||||
media_type="text/plain",
|
media_type="text/plain",
|
||||||
headers={
|
headers=build_attachment_headers(filename),
|
||||||
"Content-Disposition": (
|
|
||||||
f"attachment; filename*=UTF-8''{quote(filename, safe='')}"
|
|
||||||
)
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
return response_from_download(await get_stored_download(file_code, file_storage))
|
return response_from_download(await get_stored_download(file_code, file_storage))
|
||||||
|
|
||||||
@@ -313,6 +309,7 @@ async def init_chunk_upload(data: InitChunkUploadModel = Depends(parse_init_chun
|
|||||||
existing_session = await UploadChunk.filter(
|
existing_session = await UploadChunk.filter(
|
||||||
chunk_hash=data.file_hash,
|
chunk_hash=data.file_hash,
|
||||||
chunk_index=-1,
|
chunk_index=-1,
|
||||||
|
completed=False,
|
||||||
file_size=data.file_size,
|
file_size=data.file_size,
|
||||||
file_name=safe_file_name,
|
file_name=safe_file_name,
|
||||||
).first()
|
).first()
|
||||||
@@ -331,7 +328,7 @@ async def init_chunk_upload(data: InitChunkUploadModel = Depends(parse_init_chun
|
|||||||
* 3600,
|
* 3600,
|
||||||
)
|
)
|
||||||
uploaded_chunks = await UploadChunk.filter(
|
uploaded_chunks = await UploadChunk.filter(
|
||||||
upload_id=existing_session.upload_id, completed=True
|
upload_id=existing_session.upload_id, completed=True, chunk_index__gte=0
|
||||||
).values_list("chunk_index", flat=True)
|
).values_list("chunk_index", flat=True)
|
||||||
return APIResponse(
|
return APIResponse(
|
||||||
detail={
|
detail={
|
||||||
@@ -391,6 +388,8 @@ async def upload_chunk(
|
|||||||
chunk_info = await UploadChunk.filter(upload_id=upload_id, chunk_index=-1).first()
|
chunk_info = await UploadChunk.filter(upload_id=upload_id, chunk_index=-1).first()
|
||||||
if not chunk_info:
|
if not chunk_info:
|
||||||
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="上传会话不存在")
|
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="上传会话不存在")
|
||||||
|
if chunk_info.completed:
|
||||||
|
raise HTTPException(409, "上传会话已完成,仅待清理")
|
||||||
await reserve_storage(
|
await reserve_storage(
|
||||||
f"chunk:{upload_id}",
|
f"chunk:{upload_id}",
|
||||||
chunk_info.file_size,
|
chunk_info.file_size,
|
||||||
@@ -425,7 +424,7 @@ async def upload_chunk(
|
|||||||
|
|
||||||
# 计算已上传分片数,校验累计大小不超限(用分片数 * chunk_size 估算)
|
# 计算已上传分片数,校验累计大小不超限(用分片数 * chunk_size 估算)
|
||||||
uploaded_count = await UploadChunk.filter(
|
uploaded_count = await UploadChunk.filter(
|
||||||
upload_id=upload_id, completed=True
|
upload_id=upload_id, completed=True, chunk_index__gte=0
|
||||||
).count()
|
).count()
|
||||||
# 已上传分片的最大可能大小 + 当前分片
|
# 已上传分片的最大可能大小 + 当前分片
|
||||||
max_uploaded_size = uploaded_count * chunk_info.chunk_size + chunk_size
|
max_uploaded_size = uploaded_count * chunk_info.chunk_size + chunk_size
|
||||||
@@ -483,6 +482,7 @@ async def cancel_upload(upload_id: str):
|
|||||||
await storage.clean_chunks(upload_id, save_path)
|
await storage.clean_chunks(upload_id, save_path)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.warning("取消分片上传:清理分片文件失败 upload_id=%s", upload_id, exc_info=True)
|
logger.warning("取消分片上传:清理分片文件失败 upload_id=%s", upload_id, exc_info=True)
|
||||||
|
raise HTTPException(503, "分片清理失败,请稍后重试")
|
||||||
|
|
||||||
# 清理数据库记录
|
# 清理数据库记录
|
||||||
await UploadChunk.filter(upload_id=upload_id).delete()
|
await UploadChunk.filter(upload_id=upload_id).delete()
|
||||||
@@ -502,7 +502,7 @@ async def get_upload_status(upload_id: str):
|
|||||||
|
|
||||||
# 获取已上传的分片列表
|
# 获取已上传的分片列表
|
||||||
uploaded_chunks = await UploadChunk.filter(
|
uploaded_chunks = await UploadChunk.filter(
|
||||||
upload_id=upload_id, completed=True
|
upload_id=upload_id, completed=True, chunk_index__gte=0
|
||||||
).values_list("chunk_index", flat=True)
|
).values_list("chunk_index", flat=True)
|
||||||
|
|
||||||
return APIResponse(
|
return APIResponse(
|
||||||
@@ -530,6 +530,8 @@ async def complete_upload(
|
|||||||
chunk_info = await UploadChunk.filter(upload_id=upload_id, chunk_index=-1).first()
|
chunk_info = await UploadChunk.filter(upload_id=upload_id, chunk_index=-1).first()
|
||||||
if not chunk_info:
|
if not chunk_info:
|
||||||
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="上传会话不存在")
|
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="上传会话不存在")
|
||||||
|
if chunk_info.completed:
|
||||||
|
raise HTTPException(409, "上传会话已完成,仅待清理")
|
||||||
validate_expire_style(data.expire_style)
|
validate_expire_style(data.expire_style)
|
||||||
detail = await FileUploadService.complete_chunked_upload(
|
detail = await FileUploadService.complete_chunked_upload(
|
||||||
upload_id, chunk_info, data.expire_value, data.expire_style
|
upload_id, chunk_info, data.expire_value, data.expire_style
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
# @File : storage.py
|
# @File : storage.py
|
||||||
# @Software: PyCharm
|
# @Software: PyCharm
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import re
|
||||||
from typing import BinaryIO, Optional
|
from typing import BinaryIO, Optional
|
||||||
from urllib.parse import quote
|
from urllib.parse import quote
|
||||||
|
|
||||||
@@ -66,8 +67,15 @@ def build_attachment_headers(filename: str, content_length=None) -> dict:
|
|||||||
(/share/download)因此永不内联渲染 HTML/SVG。此函数是唯一构造点,
|
(/share/download)因此永不内联渲染 HTML/SVG。此函数是唯一构造点,
|
||||||
新增后端必须复用(tests/test_attachment_guard.py 有源码级 tripwire)。
|
新增后端必须复用(tests/test_attachment_guard.py 有源码级 tripwire)。
|
||||||
"""
|
"""
|
||||||
|
# Older Android download managers ignore filename*. Keep an ASCII fallback
|
||||||
|
# with the extension, without allowing quoted-string/header injection.
|
||||||
|
fallback = re.sub(r"[^A-Za-z0-9._ -]", "_", filename).strip(" .") or "download"
|
||||||
encoded_filename = quote(filename, safe="")
|
encoded_filename = quote(filename, safe="")
|
||||||
headers = {"Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}"}
|
headers = {
|
||||||
|
"Content-Disposition": (
|
||||||
|
f'attachment; filename="{fallback}"; filename*=UTF-8\'\'{encoded_filename}'
|
||||||
|
)
|
||||||
|
}
|
||||||
if content_length is not None:
|
if content_length is not None:
|
||||||
headers["Content-Length"] = str(content_length)
|
headers["Content-Length"] = str(content_length)
|
||||||
return headers
|
return headers
|
||||||
|
|||||||
+14
-15
@@ -45,7 +45,12 @@ class S3FileStorage(FileStorageInterface):
|
|||||||
self.endpoint_url = settings.s3_endpoint_url
|
self.endpoint_url = settings.s3_endpoint_url
|
||||||
|
|
||||||
def _client_config(self) -> Config:
|
def _client_config(self) -> Config:
|
||||||
config = {"signature_version": self.signature_version}
|
# OSS 等兼容服务不支持默认 flexible checksum 的 aws-chunked trailer。
|
||||||
|
config = {
|
||||||
|
"signature_version": self.signature_version,
|
||||||
|
"request_checksum_calculation": "when_required",
|
||||||
|
"response_checksum_validation": "when_required",
|
||||||
|
}
|
||||||
s3_config = {}
|
s3_config = {}
|
||||||
if self.addressing_style in {"path", "virtual", "auto"}:
|
if self.addressing_style in {"path", "virtual", "auto"}:
|
||||||
s3_config["addressing_style"] = self.addressing_style
|
s3_config["addressing_style"] = self.addressing_style
|
||||||
@@ -277,21 +282,15 @@ class S3FileStorage(FileStorageInterface):
|
|||||||
:param upload_id: 上传会话ID
|
:param upload_id: 上传会话ID
|
||||||
:param save_path: 文件保存路径
|
:param save_path: 文件保存路径
|
||||||
"""
|
"""
|
||||||
chunk_dir = str(Path(save_path).parent / "chunks" / upload_id)
|
# 尾部斜杠防止误删 upload_id 相同前缀的其他会话。
|
||||||
|
chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) + "/"
|
||||||
async with self._client() as s3:
|
async with self._client() as s3:
|
||||||
try:
|
paginator = s3.get_paginator("list_objects_v2")
|
||||||
# 列出并删除所有分片对象
|
async for page in paginator.paginate(Bucket=self.bucket_name, Prefix=chunk_dir):
|
||||||
paginator = s3.get_paginator('list_objects_v2')
|
for obj in page.get("Contents", []):
|
||||||
async for page in paginator.paginate(Bucket=self.bucket_name, Prefix=chunk_dir):
|
# OSS 批量删除要求 Content-MD5;单对象删除不依赖该扩展。
|
||||||
objects = page.get('Contents', [])
|
# 失败向调用方传播,保留会话供重试,不能当作清理成功。
|
||||||
if objects:
|
await s3.delete_object(Bucket=self.bucket_name, Key=obj["Key"])
|
||||||
delete_objects = [{'Key': obj['Key']} for obj in objects]
|
|
||||||
await s3.delete_objects(
|
|
||||||
Bucket=self.bucket_name,
|
|
||||||
Delete={'Objects': delete_objects}
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"清理 S3 分片数据时出错: {e}")
|
|
||||||
|
|
||||||
async def generate_presigned_upload_url(self, save_path: str, expires_in: int = 900) -> Optional[str]:
|
async def generate_presigned_upload_url(self, save_path: str, expires_in: int = 900) -> Optional[str]:
|
||||||
"""
|
"""
|
||||||
|
|||||||
+1
-1
@@ -1,7 +1,7 @@
|
|||||||
version: "3"
|
version: "3"
|
||||||
services:
|
services:
|
||||||
file-code-box:
|
file-code-box:
|
||||||
image: lanol/filecodebox:2.8.4 # x-release-please-version
|
image: lanol/filecodebox:2.9.0 # x-release-please-version
|
||||||
volumes:
|
volumes:
|
||||||
- fcb-data:/app/data:rw
|
- fcb-data:/app/data:rw
|
||||||
# NAS 已有文件:把共享目录挂到 /app/data/local,即可在后台「本地分享」里生成提取码,无需再上传
|
# NAS 已有文件:把共享目录挂到 /app/data/local,即可在后台「本地分享」里生成提取码,无需再上传
|
||||||
|
|||||||
@@ -1,5 +1,19 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## [2.9.0](https://github.com/vastsa/FileCodeBox/compare/v2.8.5...v2.9.0) (2026-11-16)
|
||||||
|
|
||||||
|
|
||||||
|
### Features
|
||||||
|
|
||||||
|
* expose background in public config for 2024 theme ([#546](https://github.com/vastsa/FileCodeBox/issues/546)) ([f9cc16c](https://github.com/vastsa/FileCodeBox/commit/f9cc16c805757dc4dcf198456f4a5e84922b9b8f))
|
||||||
|
|
||||||
|
## [2.8.5](https://github.com/vastsa/FileCodeBox/compare/v2.8.4...v2.8.5) (2026-10-27)
|
||||||
|
|
||||||
|
|
||||||
|
### Bug Fixes
|
||||||
|
|
||||||
|
* resolve OSS chunk cleanup and legacy download issues ([381a539](https://github.com/vastsa/FileCodeBox/commit/381a539c92ba85681a1e820a4150fff17984147a))
|
||||||
|
|
||||||
## [2.8.4](https://github.com/vastsa/FileCodeBox/compare/v2.8.3...v2.8.4) (2026-10-25)
|
## [2.8.4](https://github.com/vastsa/FileCodeBox/compare/v2.8.3...v2.8.4) (2026-10-25)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
<!-- Current release: 2.8.4 x-release-please-version -->
|
<!-- Current release: 2.9.0 x-release-please-version -->
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ FileCodeBox provides rich configuration options. Prefer the admin panel; directl
|
|||||||
|
|
||||||
FileCodeBox supports two configuration methods:
|
FileCodeBox supports two configuration methods:
|
||||||
|
|
||||||
1. **Admin Panel Configuration** (Recommended): Access `/admin` to enter the admin panel and modify settings on the settings page
|
1. **Admin Panel Configuration** (Recommended): Access `/#/admin` to enter the admin panel and modify settings on the settings page
|
||||||
2. **Database Configuration**: Configuration is stored in the `data/filecodebox.db` database
|
2. **Database Configuration**: Configuration is stored in the `data/filecodebox.db` database
|
||||||
|
|
||||||
::: tip Note
|
::: tip Note
|
||||||
|
|||||||
@@ -179,7 +179,7 @@ python main.py
|
|||||||
3. Download file or view text
|
3. Download file or view text
|
||||||
|
|
||||||
### Admin Panel
|
### Admin Panel
|
||||||
1. Visit `/admin`
|
1. Visit `/#/admin`
|
||||||
2. Enter admin password
|
2. Enter admin password
|
||||||
3. Manage files and settings
|
3. Manage files and settings
|
||||||
|
|
||||||
|
|||||||
@@ -10,10 +10,10 @@ FileCodeBox provides a fully-featured admin panel that allows administrators to
|
|||||||
|
|
||||||
### Login Method
|
### Login Method
|
||||||
|
|
||||||
The admin panel is located at the `/admin` path. Access method:
|
The admin panel is located at `/#/admin` (a frontend hash route). Access method:
|
||||||
|
|
||||||
1. On first run, set the admin password on the setup page
|
1. On first run, set the admin password on the setup page
|
||||||
2. Visit `http://your-domain.com/admin` in your browser
|
2. Visit `http://your-domain.com/#/admin` in your browser
|
||||||
3. Enter the admin password (the value of the `admin_token` configuration)
|
3. Enter the admin password (the value of the `admin_token` configuration)
|
||||||
4. Click the login button
|
4. Click the login button
|
||||||
|
|
||||||
@@ -30,7 +30,7 @@ By default, the admin panel entry is not shown on the homepage. You can control
|
|||||||
| `show_admin_addr` | int | `0` | Show admin entry on homepage (1=show, 0=hide) |
|
| `show_admin_addr` | int | `0` | Show admin entry on homepage (1=show, 0=hide) |
|
||||||
|
|
||||||
::: warning Security Recommendation
|
::: warning Security Recommendation
|
||||||
For public services, it's recommended to keep `show_admin_addr` at `0` and access the admin panel directly via the `/admin` path to reduce the risk of malicious scanning.
|
For public services, keep `show_admin_addr` at `0` and access the admin panel via `/#/admin`. Hiding the link is not access control; a strong password is still required.
|
||||||
:::
|
:::
|
||||||
|
|
||||||
### Authentication Mechanism
|
### Authentication Mechanism
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ FileCodeBox does not ship with a default admin password. Open the site in your b
|
|||||||
|
|
||||||
Change the admin password through the admin panel:
|
Change the admin password through the admin panel:
|
||||||
|
|
||||||
1. Access `/admin` to enter the admin panel
|
1. Access `/#/admin` to enter the admin panel
|
||||||
2. Log in with the current password
|
2. Log in with the current password
|
||||||
3. Go to the "System Settings" page
|
3. Go to the "System Settings" page
|
||||||
4. Find the `admin_token` configuration item
|
4. Find the `admin_token` configuration item
|
||||||
@@ -41,7 +41,7 @@ By default, the admin panel entry is hidden. You can control whether to show the
|
|||||||
| `show_admin_addr` | int | `0` | Show admin entry (1=show, 0=hide) |
|
| `show_admin_addr` | int | `0` | Show admin entry (1=show, 0=hide) |
|
||||||
|
|
||||||
::: tip Recommendation
|
::: tip Recommendation
|
||||||
For public services, it's recommended to keep `show_admin_addr` at `0` and access the admin panel directly via the `/admin` path.
|
For public services, it's recommended to keep `show_admin_addr` at `0` and access the admin panel directly via `/#/admin`.
|
||||||
:::
|
:::
|
||||||
|
|
||||||
## IP Rate Limiting
|
## IP Rate Limiting
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ Supports all S3-compatible object storage services, including AWS S3, Aliyun OSS
|
|||||||
| `s3_endpoint_url` | string | `""` | S3 endpoint URL |
|
| `s3_endpoint_url` | string | `""` | S3 endpoint URL |
|
||||||
| `s3_region_name` | string | `auto` | Region name |
|
| `s3_region_name` | string | `auto` | Region name |
|
||||||
| `s3_signature_version` | string | `s3v2` | Signature version (`s3v2` or `s3v4`) |
|
| `s3_signature_version` | string | `s3v2` | Signature version (`s3v2` or `s3v4`) |
|
||||||
|
| `s3_addressing_style` | string | `auto` | Addressing style (`auto`, `path`, or `virtual`); use `virtual` for OSS |
|
||||||
| `s3_hostname` | string | `""` | S3 hostname (alternative) |
|
| `s3_hostname` | string | `""` | S3 hostname (alternative) |
|
||||||
| `s3_proxy` | int | `0` | Download through server proxy (1=yes, 0=no) |
|
| `s3_proxy` | int | `0` | Download through server proxy (1=yes, 0=no) |
|
||||||
| `aws_session_token` | string | `""` | AWS session token (optional) |
|
| `aws_session_token` | string | `""` | AWS session token (optional) |
|
||||||
@@ -76,13 +77,16 @@ file_storage=s3
|
|||||||
s3_access_key_id=YourAccessKeyId
|
s3_access_key_id=YourAccessKeyId
|
||||||
s3_secret_access_key=YourSecretAccessKey
|
s3_secret_access_key=YourSecretAccessKey
|
||||||
s3_bucket_name=bucket-name
|
s3_bucket_name=bucket-name
|
||||||
s3_endpoint_url=https://bucket-name.oss-cn-hangzhou.aliyuncs.com
|
s3_endpoint_url=https://oss-cn-hangzhou.aliyuncs.com
|
||||||
s3_region_name=oss-cn-hangzhou
|
s3_region_name=oss-cn-hangzhou
|
||||||
s3_signature_version=s3v4
|
s3_signature_version=s3v4
|
||||||
|
s3_addressing_style=virtual
|
||||||
```
|
```
|
||||||
|
|
||||||
::: tip Aliyun OSS Endpoint Format
|
::: tip Aliyun OSS Endpoint Format
|
||||||
Endpoint URL format: `https://<bucket-name>.<region>.aliyuncs.com`
|
Endpoint URL format: `https://oss-<region>.aliyuncs.com` (for example, region `cn-hangzhou`), **without the bucket name**. Set the bucket only in `s3_bucket_name` and select Virtual-host (`virtual`) addressing. The SDK adds the bucket to the hostname; including it in the endpoint can duplicate the bucket prefix or cause DNS failures.
|
||||||
|
|
||||||
|
The S3 client sets request and response checksums to `when_required` to avoid OSS's unsupported `STREAMING-UNSIGNED-PAYLOAD-TRAILER`. Chunk cleanup uses paginated, single-object deletion; no additional checksum environment variables are needed. Failed cleanup retains the upload session for retry by the scheduled task.
|
||||||
|
|
||||||
Common regions:
|
Common regions:
|
||||||
- Hangzhou: `oss-cn-hangzhou`
|
- Hangzhou: `oss-cn-hangzhou`
|
||||||
|
|||||||
+1
-1
@@ -42,7 +42,7 @@ docker run -d --restart unless-stopped \
|
|||||||
--log-opt max-size=10m \
|
--log-opt max-size=10m \
|
||||||
--log-opt max-file=3 \
|
--log-opt max-file=3 \
|
||||||
--name filecodebox \
|
--name filecodebox \
|
||||||
lanol/filecodebox:2.8.4 # x-release-please-version
|
lanol/filecodebox:2.9.0 # x-release-please-version
|
||||||
```
|
```
|
||||||
|
|
||||||
Open `http://localhost:12345` and complete the first-run setup.
|
Open `http://localhost:12345` and complete the first-run setup.
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ FileCodeBox 提供了丰富的配置选项。请优先通过管理面板修改
|
|||||||
|
|
||||||
FileCodeBox 支持两种配置方式:
|
FileCodeBox 支持两种配置方式:
|
||||||
|
|
||||||
1. **管理面板配置**(推荐):访问 `/admin` 进入管理面板,在设置页面修改配置
|
1. **管理面板配置**(推荐):访问 `/#/admin` 进入管理面板,在设置页面修改配置
|
||||||
2. **数据库配置**:配置存储在 `data/filecodebox.db` 数据库中
|
2. **数据库配置**:配置存储在 `data/filecodebox.db` 数据库中
|
||||||
|
|
||||||
::: tip 提示
|
::: tip 提示
|
||||||
|
|||||||
@@ -193,7 +193,7 @@ python main.py
|
|||||||
3. 下载文件或查看文本
|
3. 下载文件或查看文本
|
||||||
|
|
||||||
### 管理面板
|
### 管理面板
|
||||||
1. 访问 `/admin`
|
1. 访问 `/#/admin`
|
||||||
2. 输入管理员密码
|
2. 输入管理员密码
|
||||||
3. 管理文件和配置
|
3. 管理文件和配置
|
||||||
|
|
||||||
|
|||||||
@@ -10,10 +10,10 @@ FileCodeBox 提供了功能完善的管理面板,让管理员可以方便地
|
|||||||
|
|
||||||
### 登录方式
|
### 登录方式
|
||||||
|
|
||||||
管理面板位于 `/admin` 路径。访问方式:
|
管理面板位于 `/#/admin`(前端 Hash 路由)。访问方式:
|
||||||
|
|
||||||
1. 首次访问站点时,按初始化页面设置管理员密码
|
1. 首次访问站点时,按初始化页面设置管理员密码
|
||||||
2. 在浏览器中访问 `http://your-domain.com/admin`
|
2. 在浏览器中访问 `http://your-domain.com/#/admin`
|
||||||
3. 输入管理员密码(`admin_token` 配置项的值)
|
3. 输入管理员密码(`admin_token` 配置项的值)
|
||||||
4. 点击登录按钮
|
4. 点击登录按钮
|
||||||
|
|
||||||
@@ -30,7 +30,7 @@ FileCodeBox 提供了功能完善的管理面板,让管理员可以方便地
|
|||||||
| `show_admin_addr` | int | `0` | 是否在首页显示管理入口(1=显示,0=隐藏) |
|
| `show_admin_addr` | int | `0` | 是否在首页显示管理入口(1=显示,0=隐藏) |
|
||||||
|
|
||||||
::: warning 安全建议
|
::: warning 安全建议
|
||||||
在公开服务中,建议保持 `show_admin_addr` 为 `0`,通过直接访问 `/admin` 路径进入管理面板,减少被恶意扫描的风险。
|
在公开服务中,建议保持 `show_admin_addr` 为 `0`,通过直接访问 `/#/admin` 进入管理面板。隐藏入口不是访问控制,仍须使用强密码。
|
||||||
:::
|
:::
|
||||||
|
|
||||||
### 认证机制
|
### 认证机制
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ FileCodeBox 首次启动时不会生成默认管理员密码。请在浏览器
|
|||||||
|
|
||||||
请通过管理面板修改管理员密码:
|
请通过管理面板修改管理员密码:
|
||||||
|
|
||||||
1. 访问 `/admin` 进入管理面板
|
1. 访问 `/#/admin` 进入管理面板
|
||||||
2. 使用当前密码登录
|
2. 使用当前密码登录
|
||||||
3. 进入「系统设置」页面
|
3. 进入「系统设置」页面
|
||||||
4. 找到 `admin_token` 配置项
|
4. 找到 `admin_token` 配置项
|
||||||
@@ -41,7 +41,7 @@ FileCodeBox 首次启动时不会生成默认管理员密码。请在浏览器
|
|||||||
| `show_admin_addr` | int | `0` | 是否显示管理入口(1=显示,0=隐藏) |
|
| `show_admin_addr` | int | `0` | 是否显示管理入口(1=显示,0=隐藏) |
|
||||||
|
|
||||||
::: tip 建议
|
::: tip 建议
|
||||||
在公开服务中,建议保持 `show_admin_addr` 为 `0`,通过直接访问 `/admin` 路径进入管理面板。
|
在公开服务中,建议保持 `show_admin_addr` 为 `0`,通过直接访问 `/#/admin` 进入管理面板。
|
||||||
:::
|
:::
|
||||||
|
|
||||||
## IP 速率限制
|
## IP 速率限制
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ storage_path=
|
|||||||
| `s3_endpoint_url` | string | `""` | S3 端点 URL |
|
| `s3_endpoint_url` | string | `""` | S3 端点 URL |
|
||||||
| `s3_region_name` | string | `auto` | 区域名称 |
|
| `s3_region_name` | string | `auto` | 区域名称 |
|
||||||
| `s3_signature_version` | string | `s3v2` | 签名版本(`s3v2` 或 `s3v4`) |
|
| `s3_signature_version` | string | `s3v2` | 签名版本(`s3v2` 或 `s3v4`) |
|
||||||
|
| `s3_addressing_style` | string | `auto` | 地址风格(`auto`、`path` 或 `virtual`);OSS 使用 `virtual` |
|
||||||
| `s3_hostname` | string | `""` | S3 主机名(备用) |
|
| `s3_hostname` | string | `""` | S3 主机名(备用) |
|
||||||
| `s3_proxy` | int | `0` | 是否通过服务器代理下载(1=是,0=否) |
|
| `s3_proxy` | int | `0` | 是否通过服务器代理下载(1=是,0=否) |
|
||||||
| `aws_session_token` | string | `""` | AWS 会话令牌(可选) |
|
| `aws_session_token` | string | `""` | AWS 会话令牌(可选) |
|
||||||
@@ -75,13 +76,16 @@ file_storage=s3
|
|||||||
s3_access_key_id=您的AccessKeyId
|
s3_access_key_id=您的AccessKeyId
|
||||||
s3_secret_access_key=您的SecretAccessKey
|
s3_secret_access_key=您的SecretAccessKey
|
||||||
s3_bucket_name=bucket-name
|
s3_bucket_name=bucket-name
|
||||||
s3_endpoint_url=https://bucket-name.oss-cn-hangzhou.aliyuncs.com
|
s3_endpoint_url=https://oss-cn-hangzhou.aliyuncs.com
|
||||||
s3_region_name=oss-cn-hangzhou
|
s3_region_name=oss-cn-hangzhou
|
||||||
s3_signature_version=s3v4
|
s3_signature_version=s3v4
|
||||||
|
s3_addressing_style=virtual
|
||||||
```
|
```
|
||||||
|
|
||||||
::: tip 阿里云 OSS 端点格式
|
::: tip 阿里云 OSS 端点格式
|
||||||
端点 URL 格式为:`https://<bucket-name>.<region>.aliyuncs.com`
|
端点 URL 格式为:`https://oss-<region>.aliyuncs.com`(例如 region 为 `cn-hangzhou`),**不要包含桶名**。桶名仅填入 `s3_bucket_name`,地址风格选择 Virtual-host(`virtual`),由 SDK 自动拼接桶名;否则可能出现桶名前缀重复或 DNS 解析失败。
|
||||||
|
|
||||||
|
S3 客户端已将请求和响应校验和设为 `when_required`,避免 OSS 不支持的 `STREAMING-UNSIGNED-PAYLOAD-TRAILER`;分片采用分页、逐对象删除,无需配置额外的校验和环境变量。清理失败会保留上传会话,供定时任务重试。
|
||||||
|
|
||||||
常用区域:
|
常用区域:
|
||||||
- 杭州:`oss-cn-hangzhou`
|
- 杭州:`oss-cn-hangzhou`
|
||||||
|
|||||||
+1
-1
@@ -42,7 +42,7 @@ docker run -d --restart unless-stopped \
|
|||||||
--log-opt max-size=10m \
|
--log-opt max-size=10m \
|
||||||
--log-opt max-file=3 \
|
--log-opt max-file=3 \
|
||||||
--name filecodebox \
|
--name filecodebox \
|
||||||
lanol/filecodebox:2.8.4 # x-release-please-version
|
lanol/filecodebox:2.9.0 # x-release-please-version
|
||||||
```
|
```
|
||||||
|
|
||||||
打开 `http://localhost:12345`,按引导完成首次初始化。
|
打开 `http://localhost:12345`,按引导完成首次初始化。
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ docker run -d --restart unless-stopped \
|
|||||||
--log-opt max-size=10m \
|
--log-opt max-size=10m \
|
||||||
--log-opt max-file=3 \
|
--log-opt max-file=3 \
|
||||||
--name filecodebox \
|
--name filecodebox \
|
||||||
lanol/filecodebox:2.8.4 # x-release-please-version
|
lanol/filecodebox:2.9.0 # x-release-please-version
|
||||||
```
|
```
|
||||||
|
|
||||||
访问 `http://localhost:12345`,完成首次初始化。生产环境建议固定版本号;`latest` 指向最新正式版。
|
访问 `http://localhost:12345`,完成首次初始化。生产环境建议固定版本号;`latest` 指向最新正式版。
|
||||||
|
|||||||
+1
-1
@@ -32,7 +32,7 @@ docker run -d --restart unless-stopped \
|
|||||||
--log-opt max-size=10m \
|
--log-opt max-size=10m \
|
||||||
--log-opt max-file=3 \
|
--log-opt max-file=3 \
|
||||||
--name filecodebox \
|
--name filecodebox \
|
||||||
lanol/filecodebox:2.8.4 # x-release-please-version
|
lanol/filecodebox:2.9.0 # x-release-please-version
|
||||||
```
|
```
|
||||||
|
|
||||||
Open `http://localhost:12345` and complete first-run setup. Pin a version in production; `latest` tracks the newest stable release.
|
Open `http://localhost:12345` and complete first-run setup. Pin a version in production; `latest` tracks the newest stable release.
|
||||||
|
|||||||
@@ -24,3 +24,19 @@ class AdminAddressPublicConfigTests(unittest.TestCase):
|
|||||||
self.assertIs(type(public_value), int)
|
self.assertIs(type(public_value), int)
|
||||||
self.assertEqual(public_value, expected_value)
|
self.assertEqual(public_value, expected_value)
|
||||||
self.assertIs(feature_value, bool(expected_value))
|
self.assertIs(feature_value, bool(expected_value))
|
||||||
|
|
||||||
|
|
||||||
|
class BackgroundPublicConfigTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self._original_user_config = dict(settings.user_config)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
settings.user_config = self._original_user_config
|
||||||
|
|
||||||
|
def test_background_is_exposed_for_themes(self):
|
||||||
|
settings.background = "https://example.com/bg.jpg"
|
||||||
|
self.assertEqual(build_public_config()["background"], "https://example.com/bg.jpg")
|
||||||
|
|
||||||
|
def test_empty_background_is_string(self):
|
||||||
|
settings.background = None
|
||||||
|
self.assertEqual(build_public_config()["background"], "")
|
||||||
@@ -163,7 +163,7 @@ class ShareUsageSecurityTests(unittest.TestCase):
|
|||||||
self.assertEqual(response.media_type, "text/plain")
|
self.assertEqual(response.media_type, "text/plain")
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
response.headers["content-disposition"],
|
response.headers["content-disposition"],
|
||||||
"attachment; filename*=UTF-8''TAG.txt",
|
'attachment; filename="TAG.txt"; filename*=UTF-8\'\'TAG.txt',
|
||||||
)
|
)
|
||||||
await record.refresh_from_db()
|
await record.refresh_from_db()
|
||||||
self.assertEqual(record.used_count, 1)
|
self.assertEqual(record.used_count, 1)
|
||||||
|
|||||||
@@ -0,0 +1,84 @@
|
|||||||
|
"""Issue #531: keep a legacy filename alongside the RFC 5987 UTF-8 name."""
|
||||||
|
import re
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from urllib.parse import quote, unquote
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from apps.base import local_share, views
|
||||||
|
from apps.base.models import FileCodes
|
||||||
|
from core.settings import settings
|
||||||
|
from core.storage import build_attachment_headers
|
||||||
|
|
||||||
|
|
||||||
|
def legacy_filename(disposition):
|
||||||
|
match = re.search(r'(?:^|;\s*)filename="([^"]+)"', disposition)
|
||||||
|
assert match is not None, "legacy download clients cannot read filename*"
|
||||||
|
return match.group(1)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("filename", "fallback"),
|
||||||
|
[
|
||||||
|
("report.pdf", "report.pdf"),
|
||||||
|
("my archive.tar.gz", "my archive.tar.gz"),
|
||||||
|
("报告.pdf", "__.pdf"),
|
||||||
|
("图像 🖼.png", "__ _.png"),
|
||||||
|
("", "download"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_attachment_has_legacy_and_utf8_filenames(filename, fallback):
|
||||||
|
headers = build_attachment_headers(filename, 42)
|
||||||
|
disposition = headers["Content-Disposition"]
|
||||||
|
assert disposition.startswith("attachment;")
|
||||||
|
assert legacy_filename(disposition) == fallback
|
||||||
|
assert disposition.endswith("filename*=UTF-8''" + quote(filename, safe=""))
|
||||||
|
assert headers["Content-Length"] == "42"
|
||||||
|
assert disposition.isascii()
|
||||||
|
|
||||||
|
|
||||||
|
def test_legacy_fallback_cannot_inject_parameters_headers_or_paths():
|
||||||
|
filename = '../bad\\name"; inline;\r\nX-Evil: true\x00%20.html'
|
||||||
|
disposition = build_attachment_headers(filename)["Content-Disposition"]
|
||||||
|
fallback = legacy_filename(disposition)
|
||||||
|
assert re.fullmatch(r"[A-Za-z0-9._ -]+", fallback)
|
||||||
|
assert not fallback.startswith(".")
|
||||||
|
assert fallback.endswith(".html")
|
||||||
|
assert "\r" not in disposition and "\n" not in disposition and "\x00" not in disposition
|
||||||
|
assert len(disposition.split(";")) == 3
|
||||||
|
assert unquote(disposition.split("filename*=UTF-8''", 1)[1]) == filename
|
||||||
|
|
||||||
|
|
||||||
|
def test_local_reference_download_uses_compatible_headers(tmp_path, monkeypatch):
|
||||||
|
source = tmp_path / "报告.pdf"
|
||||||
|
source.write_bytes(b"pdf contents")
|
||||||
|
monkeypatch.setattr(local_share, "get_local_root", lambda: tmp_path)
|
||||||
|
record = SimpleNamespace(uuid_file_name=source.name, prefix="报告", suffix=".pdf")
|
||||||
|
|
||||||
|
download = local_share.build_local_ref_download(record)
|
||||||
|
|
||||||
|
assert download.path == source
|
||||||
|
assert download.headers == build_attachment_headers("报告.pdf", source.stat().st_size)
|
||||||
|
assert legacy_filename(download.headers["Content-Disposition"]) == "__.pdf"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_text_download_uses_compatible_headers(db, monkeypatch):
|
||||||
|
monkeypatch.setattr(settings, "user_config", dict(settings.user_config))
|
||||||
|
settings.file_storage = "local"
|
||||||
|
record = await FileCodes.create(
|
||||||
|
code="legacy-text",
|
||||||
|
prefix="文本",
|
||||||
|
text="hello, 文本直链",
|
||||||
|
expired_count=-1,
|
||||||
|
expired_at=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
response = await views.get_code_file(code=record.code, ip="127.0.0.1")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.body.decode("utf-8") == record.text
|
||||||
|
assert response.headers["content-disposition"] == build_attachment_headers("文本.txt")[
|
||||||
|
"Content-Disposition"
|
||||||
|
]
|
||||||
|
assert legacy_filename(response.headers["content-disposition"]) == "__.txt"
|
||||||
@@ -0,0 +1,314 @@
|
|||||||
|
"""OSS compatibility regressions: real SDK request preparation, no external I/O."""
|
||||||
|
from datetime import timedelta
|
||||||
|
from unittest.mock import AsyncMock
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
import aioboto3
|
||||||
|
import pytest
|
||||||
|
from botocore.exceptions import ClientError
|
||||||
|
from fastapi import HTTPException
|
||||||
|
|
||||||
|
from apps.base import services, tasks, views
|
||||||
|
from apps.base.models import FileCodes, StorageReservation, UploadChunk
|
||||||
|
from core.settings import settings
|
||||||
|
from core.storage.s3 import S3FileStorage
|
||||||
|
from core.utils import get_now
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def storage(monkeypatch):
|
||||||
|
monkeypatch.setattr(settings, "user_config", {
|
||||||
|
**settings.user_config, "file_storage": "s3", "upload_size": 1024,
|
||||||
|
"chunk_expire_hours": 24,
|
||||||
|
})
|
||||||
|
storage = S3FileStorage.__new__(S3FileStorage)
|
||||||
|
storage.signature_version = "s3v4"
|
||||||
|
storage.addressing_style = "virtual"
|
||||||
|
storage.bucket_name = "test-bucket"
|
||||||
|
# Reserved .invalid domain; before-send hooks stop requests before any I/O.
|
||||||
|
storage.endpoint_url = "https://oss-cn-hangzhou.example.invalid"
|
||||||
|
storage.region_name = "oss-cn-hangzhou"
|
||||||
|
storage.aws_session_token = None
|
||||||
|
storage.session = aioboto3.Session(
|
||||||
|
aws_access_key_id="testing", aws_secret_access_key="testing"
|
||||||
|
)
|
||||||
|
return storage
|
||||||
|
|
||||||
|
|
||||||
|
def test_checksum_config_preserves_signature_and_addressing(storage):
|
||||||
|
config = storage._client_config()
|
||||||
|
assert config.request_checksum_calculation == "when_required"
|
||||||
|
assert config.response_checksum_validation == "when_required"
|
||||||
|
assert config.signature_version == "s3v4"
|
||||||
|
assert config.s3["addressing_style"] == "virtual"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("operation", ["PutObject", "UploadPart"])
|
||||||
|
async def test_sdk_upload_has_no_flexible_checksum_trailer(storage, monkeypatch, operation):
|
||||||
|
monkeypatch.setenv("AWS_REQUEST_CHECKSUM_CALCULATION", "when_supported")
|
||||||
|
monkeypatch.setenv("AWS_RESPONSE_CHECKSUM_VALIDATION", "when_supported")
|
||||||
|
captured = []
|
||||||
|
|
||||||
|
class RequestCaptured(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def intercept(request, **kwargs):
|
||||||
|
captured.append(request)
|
||||||
|
raise RequestCaptured
|
||||||
|
|
||||||
|
# Register on the session so save_chunk's real _client() inherits the hook.
|
||||||
|
storage.session.events.register(f"before-send.s3.{operation}", intercept)
|
||||||
|
with pytest.raises(RequestCaptured):
|
||||||
|
if operation == "PutObject":
|
||||||
|
await storage.save_chunk("up", 0, b"payload", "hash", "share/file.bin")
|
||||||
|
else:
|
||||||
|
async with storage._client() as client:
|
||||||
|
assert client.meta.config.response_checksum_validation == "when_required"
|
||||||
|
await client.upload_part(
|
||||||
|
Bucket=storage.bucket_name, Key="share/file.bin", UploadId="up",
|
||||||
|
PartNumber=1, Body=b"payload",
|
||||||
|
)
|
||||||
|
assert len(captured) == 1
|
||||||
|
request = captured[0]
|
||||||
|
headers = {key.lower(): value for key, value in request.headers.items()}
|
||||||
|
assert "x-amz-trailer" not in headers
|
||||||
|
assert not any(key.startswith("x-amz-checksum-") for key in headers)
|
||||||
|
assert "x-amz-sdk-checksum-algorithm" not in headers
|
||||||
|
assert b"aws-chunked" not in headers.get("content-encoding", b"")
|
||||||
|
assert b"TRAILER" not in headers.get("x-amz-content-sha256", b"")
|
||||||
|
assert urlsplit(request.url).hostname == "test-bucket.oss-cn-hangzhou.example.invalid"
|
||||||
|
assert not urlsplit(request.url).path.startswith("/test-bucket/")
|
||||||
|
|
||||||
|
|
||||||
|
class FakeS3:
|
||||||
|
"""Only single-object deletion is available; paginate from a bounded snapshot."""
|
||||||
|
|
||||||
|
def __init__(self, keys=(), fail_key=None, list_error=False):
|
||||||
|
self.keys = set(keys)
|
||||||
|
self.fail_key = fail_key
|
||||||
|
self.list_error = list_error
|
||||||
|
self.deleted = []
|
||||||
|
self.prefixes = []
|
||||||
|
self.pages = 0
|
||||||
|
|
||||||
|
async def __aenter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, *args):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def get_paginator(self, operation):
|
||||||
|
assert operation == "list_objects_v2"
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def paginate(self, Bucket, Prefix):
|
||||||
|
assert Bucket == "test-bucket"
|
||||||
|
self.prefixes.append(Prefix)
|
||||||
|
if self.list_error:
|
||||||
|
raise RuntimeError("list failed")
|
||||||
|
# Also exercise a page without Contents.
|
||||||
|
yield {}
|
||||||
|
keys = sorted(key for key in self.keys if key.startswith(Prefix))
|
||||||
|
for offset in range(0, len(keys), 1000):
|
||||||
|
self.pages += 1
|
||||||
|
yield {"Contents": [{"Key": key} for key in keys[offset:offset + 1000]]}
|
||||||
|
|
||||||
|
async def delete_object(self, Bucket, Key):
|
||||||
|
assert Bucket == "test-bucket"
|
||||||
|
if Key == self.fail_key:
|
||||||
|
raise ClientError({"Error": {"Code": "AccessDenied"}}, "DeleteObject")
|
||||||
|
self.deleted.append(Key)
|
||||||
|
self.keys.discard(Key)
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def install_client(storage, monkeypatch, fake):
|
||||||
|
monkeypatch.setattr(storage, "_client", lambda: fake)
|
||||||
|
monkeypatch.setattr(tasks, "storages", {"s3": lambda: storage})
|
||||||
|
monkeypatch.setattr(views, "storages", {"s3": lambda: storage})
|
||||||
|
monkeypatch.setattr(services.FileUploadService, "_storage", lambda: storage)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cleanup_paginates_and_keeps_neighboring_upload(storage, monkeypatch):
|
||||||
|
prefix = "share/chunks/up/"
|
||||||
|
keys = {f"{prefix}{i}.part" for i in range(1005)}
|
||||||
|
neighbors = {"share/chunks/up-other/0.part", "share/file.bin"}
|
||||||
|
fake = FakeS3(keys | neighbors)
|
||||||
|
install_client(storage, monkeypatch, fake)
|
||||||
|
await storage.clean_chunks("up", "share/file.bin")
|
||||||
|
assert fake.prefixes == [prefix]
|
||||||
|
assert fake.pages == 2
|
||||||
|
assert set(fake.deleted) == keys
|
||||||
|
assert fake.keys == neighbors
|
||||||
|
await storage.clean_chunks("up", "share/file.bin") # Empty/idempotent retry.
|
||||||
|
assert fake.keys == neighbors
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cleanup_listing_failure_propagates(storage, monkeypatch):
|
||||||
|
install_client(storage, monkeypatch, FakeS3(list_error=True))
|
||||||
|
with pytest.raises(RuntimeError, match="list failed"):
|
||||||
|
await storage.clean_chunks("up", "share/file.bin")
|
||||||
|
|
||||||
|
|
||||||
|
async def seed_session(upload_id="up"):
|
||||||
|
now = await get_now()
|
||||||
|
values = dict(
|
||||||
|
upload_id=upload_id, chunk_hash="a" * 64, total_chunks=2,
|
||||||
|
file_size=8, chunk_size=4, file_name="file.bin", save_path="share/file.bin",
|
||||||
|
created_at=now - timedelta(hours=48),
|
||||||
|
)
|
||||||
|
session = await UploadChunk.create(chunk_index=-1, **values)
|
||||||
|
for index in range(2):
|
||||||
|
await UploadChunk.create(chunk_index=index, completed=True, **values)
|
||||||
|
await StorageReservation.create(
|
||||||
|
token=f"chunk:{upload_id}", size=8, expires_at=now + timedelta(hours=1)
|
||||||
|
)
|
||||||
|
return session
|
||||||
|
|
||||||
|
|
||||||
|
async def run_cleanup_round(monkeypatch):
|
||||||
|
class StopLoop(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
monkeypatch.setattr(tasks, "refresh_settings", AsyncMock())
|
||||||
|
monkeypatch.setattr(tasks.asyncio, "sleep", AsyncMock(side_effect=StopLoop))
|
||||||
|
with pytest.raises(StopLoop):
|
||||||
|
await tasks.clean_incomplete_uploads()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_partial_failure_keeps_records_then_task_retries(db, storage, monkeypatch):
|
||||||
|
await seed_session()
|
||||||
|
await seed_session("other")
|
||||||
|
keys = {"share/chunks/up/0.part", "share/chunks/up/1.part"}
|
||||||
|
fake = FakeS3(keys, fail_key="share/chunks/up/1.part")
|
||||||
|
install_client(storage, monkeypatch, fake)
|
||||||
|
with pytest.raises(ClientError):
|
||||||
|
await storage.clean_chunks("up", "share/file.bin")
|
||||||
|
assert fake.keys == {fake.fail_key}
|
||||||
|
await run_cleanup_round(monkeypatch)
|
||||||
|
assert await UploadChunk.filter(upload_id="up").count() == 3
|
||||||
|
assert await StorageReservation.filter(token="chunk:up").exists()
|
||||||
|
# Failure in one session must not prevent another session being cleaned.
|
||||||
|
assert not await UploadChunk.filter(upload_id="other").exists()
|
||||||
|
fake.fail_key = None
|
||||||
|
await run_cleanup_round(monkeypatch)
|
||||||
|
assert not fake.keys
|
||||||
|
assert not await UploadChunk.filter(upload_id="up").exists()
|
||||||
|
assert not await StorageReservation.filter(token="chunk:up").exists()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("cleanup_fails", [False, True])
|
||||||
|
async def test_completed_share_survives_cleanup_failure(db, storage, monkeypatch, cleanup_fails):
|
||||||
|
session = await seed_session()
|
||||||
|
fake = FakeS3(list_error=cleanup_fails)
|
||||||
|
install_client(storage, monkeypatch, fake)
|
||||||
|
monkeypatch.setattr(storage, "merge_chunks", AsyncMock(return_value=("share/file.bin", "a" * 64)))
|
||||||
|
monkeypatch.setattr(services, "reserve_storage", AsyncMock())
|
||||||
|
monkeypatch.setattr(services, "get_expire_info", AsyncMock(return_value=(
|
||||||
|
await get_now() + timedelta(days=1), -1, 0, "oss-code",
|
||||||
|
)))
|
||||||
|
result = await services.FileUploadService.complete_chunked_upload("up", session, 1, "day")
|
||||||
|
assert result == {"code": "oss-code", "name": "file.bin"}
|
||||||
|
assert await FileCodes.filter(code="oss-code").exists()
|
||||||
|
assert await UploadChunk.filter(upload_id="up").exists() is cleanup_fails
|
||||||
|
assert not await StorageReservation.filter(token="chunk:up").exists()
|
||||||
|
if cleanup_fails:
|
||||||
|
fake.list_error = False
|
||||||
|
await run_cleanup_round(monkeypatch)
|
||||||
|
assert not await UploadChunk.filter(upload_id="up").exists()
|
||||||
|
assert await FileCodes.filter(code="oss-code").exists()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_oversize_cleanup_failure_keeps_retry_records(db, storage, monkeypatch):
|
||||||
|
session = await seed_session()
|
||||||
|
install_client(storage, monkeypatch, FakeS3(list_error=True))
|
||||||
|
monkeypatch.setattr(services, "reserve_storage", AsyncMock())
|
||||||
|
settings.user_config = {**settings.user_config, "upload_size": 1}
|
||||||
|
with pytest.raises(HTTPException) as error:
|
||||||
|
await services.FileUploadService.complete_chunked_upload("up", session, 1, "day")
|
||||||
|
assert error.value.status_code == 403
|
||||||
|
assert await UploadChunk.filter(upload_id="up").count() == 3
|
||||||
|
assert not await FileCodes.all().exists()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cancel_cleanup_failure_keeps_retry_records(db, storage, monkeypatch):
|
||||||
|
await seed_session()
|
||||||
|
fake = FakeS3(list_error=True)
|
||||||
|
install_client(storage, monkeypatch, fake)
|
||||||
|
with pytest.raises(HTTPException) as error:
|
||||||
|
await views.cancel_upload("up")
|
||||||
|
assert error.value.status_code == 503
|
||||||
|
assert await UploadChunk.filter(upload_id="up").count() == 3
|
||||||
|
assert await StorageReservation.filter(token="chunk:up").exists()
|
||||||
|
fake.list_error = False
|
||||||
|
await views.cancel_upload("up")
|
||||||
|
assert not await UploadChunk.filter(upload_id="up").exists()
|
||||||
|
assert not await StorageReservation.filter(token="chunk:up").exists()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("delete_share", [False, True])
|
||||||
|
async def test_published_session_cannot_be_reused(db, storage, monkeypatch, delete_share):
|
||||||
|
session = await seed_session()
|
||||||
|
install_client(storage, monkeypatch, FakeS3(list_error=True))
|
||||||
|
merge = AsyncMock(return_value=("share/file.bin", "a" * 64))
|
||||||
|
save_chunk = AsyncMock()
|
||||||
|
reserve = AsyncMock()
|
||||||
|
monkeypatch.setattr(storage, "merge_chunks", merge)
|
||||||
|
monkeypatch.setattr(storage, "save_chunk", save_chunk)
|
||||||
|
monkeypatch.setattr(services, "reserve_storage", reserve)
|
||||||
|
monkeypatch.setattr(views, "reserve_storage", reserve)
|
||||||
|
monkeypatch.setattr(services, "get_expire_info", AsyncMock(return_value=(
|
||||||
|
await get_now() + timedelta(days=1), -1, 0, "oss-code",
|
||||||
|
)))
|
||||||
|
result = await services.FileUploadService.complete_chunked_upload("up", session, 1, "day")
|
||||||
|
assert result["code"] == "oss-code"
|
||||||
|
persisted_session = await UploadChunk.get(upload_id="up", chunk_index=-1)
|
||||||
|
assert persisted_session.completed is True
|
||||||
|
assert await FileCodes.filter(upload_id="up").count() == 1
|
||||||
|
if delete_share:
|
||||||
|
# The terminal marker must survive expiration/deletion of the share itself.
|
||||||
|
await FileCodes.filter(upload_id="up").delete()
|
||||||
|
|
||||||
|
with pytest.raises(HTTPException) as error:
|
||||||
|
await services.FileUploadService.complete_chunked_upload(
|
||||||
|
"up", persisted_session, 1, "day"
|
||||||
|
)
|
||||||
|
assert error.value.status_code == 409
|
||||||
|
with pytest.raises(HTTPException) as error:
|
||||||
|
await views.complete_upload(
|
||||||
|
"up", views.CompleteUploadModel(expire_value=1, expire_style="day"), "127.0.0.1"
|
||||||
|
)
|
||||||
|
assert error.value.status_code == 409
|
||||||
|
|
||||||
|
# Even when a cleaned fragment no longer has a row, it cannot be uploaded again.
|
||||||
|
await UploadChunk.filter(upload_id="up", chunk_index=0).delete()
|
||||||
|
chunk = AsyncMock()
|
||||||
|
with pytest.raises(HTTPException) as error:
|
||||||
|
await views.upload_chunk("up", 0, chunk)
|
||||||
|
assert error.value.status_code == 409
|
||||||
|
chunk.read.assert_not_awaited()
|
||||||
|
save_chunk.assert_not_awaited()
|
||||||
|
merge.assert_awaited_once()
|
||||||
|
reserve.assert_awaited_once()
|
||||||
|
assert await FileCodes.filter(upload_id="up").count() == (0 if delete_share else 1)
|
||||||
|
|
||||||
|
# The -1 cleanup marker is not a file fragment or progress credit.
|
||||||
|
status = await views.get_upload_status("up")
|
||||||
|
assert status.detail["uploaded_chunks"] == [1]
|
||||||
|
assert status.detail["progress"] == 50
|
||||||
|
|
||||||
|
# Initializing the same file must create a new upload, not revive the old ID.
|
||||||
|
initialized = await views.init_chunk_upload(views.InitChunkUploadModel(
|
||||||
|
file_name="file.bin", file_size=8, chunk_size=4, file_hash="a" * 64,
|
||||||
|
))
|
||||||
|
assert initialized.detail["upload_id"] != "up"
|
||||||
|
assert initialized.detail["uploaded_chunks"] == []
|
||||||
|
assert (await UploadChunk.get(upload_id="up", chunk_index=-1)).completed is True
|
||||||
Reference in new issue
Block a user