chore: engineering hardening - test tooling, lint, security fix, and cleanup

- add pyproject.toml with pytest config and dev dependency group (PEP 735)
- add ruff (core correctness rules) + pre-commit hook
- fix stored XSS: html.escape site config before index template injection,
  with regression tests
- log previously swallowed cleanup exceptions in upload rollback paths
- consolidate three duplicate path generators into build_file_path
  (also aligns chunk/presign date dirs to get_now() instead of local time)
- drop redundant ip_limit sync in load_config (refresh_settings covers all limiters)
- remove dead code (calculate_file_hash, commented-out block), two orphan
  guide docs, and stale .gitignore entries
- add dependabot for pip, docker, and github-actions
- bump starlette 1.3.1 -> 1.6.0 (no breaking API changes; verified with the
  full test suite plus an ASGI smoke test incl. Range downloads)
- skip theme asset tests when themes/ is not built locally
This commit is contained in:
Marrrrrrrrry committed 2026-09-07 23:43:50 +08:00
1 parent 6c76aabe1e
commit d701c6e00e
16 files changed
+169 -251

No files matched your search

+15
View File
@@ -0,0 +1,15 @@
version: 2
updates:
# Runtime deps (requirements.txt is the single source; once a lockfile lands, keep raising version PRs here)
- package-ecosystem: "pip"
directory: "/"
schedule:
interval: "weekly"
- package-ecosystem: "docker"
directory: "/"
schedule:
interval: "weekly"
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
+2 -7
View File
@@ -80,8 +80,8 @@ docs/_build/
.pybuilder/
target/
*.db
./filecodebox.db-shm
./filecodebox.db-wal
*.db-shm
*.db-wal
# Jupyter Notebook
.ipynb_checkpoints
@@ -147,13 +147,8 @@ cython_debug/
# Project
.vscode
.DS_Store
for_test.py
.html
/evaluate/temp.py
/evaluation/back.json
data/.env
.backup/
/cloc-1.64.exe
# Ignore node_modules
node_modules/
+8
View File
@@ -0,0 +1,8 @@
# First-time setup: pre-commit install
# Behind a proxy: HTTPS_PROXY=http://<proxy-host>:<port> pre-commit run --all-files
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.16.6
hooks:
- id: ruff-check
args: [--fix]
+1 -1
View File
@@ -2,7 +2,7 @@
# @Author : Lan
# @File : depends.py
# @Software: PyCharm
from fastapi import Header, HTTPException, Depends
from fastapi import Header, HTTPException
from fastapi.requests import Request
import base64
import hmac
-1
View File
@@ -1,5 +1,4 @@
from pydantic import BaseModel
from typing import Optional
class SelectFileModel(BaseModel):
+15 -25
View File
@@ -1,5 +1,4 @@
import datetime
import hashlib
import os
import uuid
from urllib.parse import unquote
@@ -26,29 +25,32 @@ def validate_expire_style(expire_style: str) -> str:
return expire_style
async def get_file_path_name(file: UploadFile) -> Tuple[str, str, str, str, str]:
async def build_file_path(
file_name: str, file_uuid: str
) -> Tuple[str, str, str, str, str]:
"""Single source of storage path generation (date dir + UUID), shared by
regular, chunked, and presigned uploads.
Always use get_now() (UTC+8); do not switch to server-local time.
"""
today = await get_now()
storage_path = settings.storage_path.strip("/")
file_uuid = uuid.uuid4().hex
filename = await sanitize_filename(unquote(file.filename or ""))
filename = await sanitize_filename(unquote(file_name or ""))
base_path = f"share/data/{today.strftime('%Y/%m/%d')}/{file_uuid}"
path = f"{storage_path}/{base_path}" if storage_path else base_path
prefix, suffix = os.path.splitext(filename)
save_path = f"{path}/{filename}"
save_path = f"{path}/{prefix}{suffix}"
return path, suffix, prefix, filename, save_path
async def get_file_path_name(file: UploadFile) -> Tuple[str, str, str, str, str]:
return await build_file_path(file.filename or "", uuid.uuid4().hex)
async def get_chunk_file_path_name(
file_name: str, upload_id: str
) -> Tuple[str, str, str, str, str]:
today = await get_now()
storage_path = settings.storage_path.strip("/")
file_name = await sanitize_filename(unquote(file_name or ""))
base_path = f"share/data/{today.strftime('%Y/%m/%d')}/{upload_id}"
path = f"{storage_path}/{base_path}" if storage_path else base_path
prefix, suffix = os.path.splitext(file_name)
save_path = f"{path}/{prefix}{suffix}"
return path, suffix, prefix, file_name, save_path
return await build_file_path(file_name, upload_id)
async def get_expire_info(
@@ -121,18 +123,6 @@ async def get_random_code(style: str | None = None) -> str:
return str(code)
async def calculate_file_hash(file: UploadFile, chunk_size=1024 * 1024) -> str:
sha = hashlib.sha256()
await file.seek(0)
while True:
chunk = await file.read(chunk_size)
if not chunk:
break
sha.update(chunk)
await file.seek(0)
return sha.hexdigest()
ip_limit = {
"error": IPRateLimit(count=settings.errorCount, minutes=settings.errorMinute),
"metadata": IPRateLimit(count=settings.errorCount, minutes=settings.errorMinute),
+12 -34
View File
@@ -1,4 +1,3 @@
import datetime
import hashlib
import os
import uuid
@@ -16,6 +15,7 @@ from tortoise.expressions import Case, F, Q, When
from apps.admin.dependencies import share_required_login
from apps.base.models import FileCodes, UploadChunk, PresignUploadSession
from apps.base.quota import release_storage, reserve_storage
from core.logger import logger
from apps.base.schemas import (
SelectFileModel,
InitChunkUploadModel,
@@ -24,6 +24,7 @@ from apps.base.schemas import (
)
from apps.base.file_validation import validate_file_type, validate_upload_file, validate_header_bytes
from apps.base.utils import (
build_file_path,
get_expire_info,
get_file_path_name,
ip_limit,
@@ -51,16 +52,8 @@ class FileUploadService:
async def generate_file_path(
file_name: str, upload_id: Optional[str] = None
) -> tuple[str, str, str, str, str]:
"""统一的路径生成"""
today = datetime.datetime.now()
storage_path = settings.storage_path.strip("/")
file_uuid = upload_id or uuid.uuid4().hex
filename = await sanitize_filename(unquote(file_name))
base_path = f"share/data/{today.strftime('%Y/%m/%d')}/{file_uuid}"
path = f"{storage_path}/{base_path}" if storage_path else base_path
prefix, suffix = os.path.splitext(filename)
save_path = f"{path}/{filename}"
return path, suffix, prefix, filename, save_path
"""Delegates path generation to apps.base.utils.build_file_path."""
return await build_file_path(file_name, upload_id or uuid.uuid4().hex)
@staticmethod
async def create_file_record(
@@ -186,7 +179,7 @@ async def share_file(
FileCodes(file_path=path, uuid_file_name=uuid_file_name)
)
except Exception:
pass
logger.warning("分享上传:记录创建失败,回滚删除已保存文件失败", exc_info=True)
raise
finally:
await release_storage(reservation_token)
@@ -404,21 +397,6 @@ async def init_chunk_upload(data: InitChunkUploadModel = Depends(parse_init_chun
status_code=403, detail=f"文件大小超过限制,最大为 {max_size_mb:.2f} MB"
)
# # 秒传检查
# existing = await FileCodes.filter(file_hash=data.file_hash).first()
# if existing:
# if await existing.is_expired():
# file_storage: FileStorageInterface = storages[settings.file_storage](
# )
# await file_storage.delete_file(existing)
# await existing.delete()
# else:
# return APIResponse(detail={
# "code": existing.code,
# "existed": True,
# "name": f'{existing.prefix}{existing.suffix}'
# })
# 断点续传:检查是否存在相同文件的未完成上传会话
existing_session = await UploadChunk.filter(
chunk_hash=data.file_hash,
@@ -591,8 +569,8 @@ async def cancel_upload(upload_id: str):
if save_path:
try:
await storage.clean_chunks(upload_id, save_path)
except Exception as e:
pass
except Exception:
logger.warning("取消分片上传:清理分片文件失败 upload_id=%s", upload_id, exc_info=True)
# 清理数据库记录
await UploadChunk.filter(upload_id=upload_id).delete()
@@ -663,7 +641,7 @@ async def complete_upload(
try:
await storage.clean_chunks(upload_id, save_path)
except Exception:
pass
logger.warning("分片超限中止:清理分片文件失败 upload_id=%s", upload_id, exc_info=True)
await UploadChunk.filter(upload_id=upload_id).delete()
await release_storage(f"chunk:{upload_id}")
max_size_mb = settings.uploadSize / (1024 * 1024)
@@ -711,7 +689,7 @@ async def complete_upload(
try:
await storage.clean_chunks(upload_id, save_path)
except Exception:
pass
logger.warning("分片合并失败:清理临时分片文件失败 upload_id=%s", upload_id, exc_info=True)
raise HTTPException(
status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"文件合并失败: {str(e)}"
)
@@ -847,7 +825,7 @@ async def presign_upload_proxy(
)
)
except Exception:
pass
logger.warning("预签名代理上传:记录创建失败,回滚删除已保存文件失败 upload_id=%s", session.upload_id, exc_info=True)
raise
await session.delete()
@@ -904,7 +882,7 @@ async def presign_upload_confirm(upload_id: str, ip: str = Depends(ip_limit["upl
)
)
except Exception:
pass
logger.warning("预签名确认:记录创建失败,回滚删除已保存文件失败 upload_id=%s", session.upload_id, exc_info=True)
raise
await session.delete()
@@ -952,7 +930,7 @@ async def presign_upload_cancel(upload_id: str):
)
await storage.delete_file(temp_file_code)
except Exception:
pass
logger.warning("取消预签名会话:清理临时文件失败 upload_id=%s", upload_id, exc_info=True)
await session.delete()
await release_storage(f"presign:{upload_id}")
-135
View File
@@ -1,135 +0,0 @@
# OneDrive作为存储的配置方法
**仅支持工作或学校账户,并且需要有管理员权限以授权API**
## 1. 需要配置的参数
```
file_storage=onedrive
onedrive_domain=XXXXXX
onedrive_client_id=XXXXXX-XXXXXX-XXXXXX-XXXXXX
onedrive_username=XXXXXX@XXXXXX
onedrive_password=XXXXXX
```
`onedrive_username`和`onedrive_password`是你的账户名(邮箱)和密码,另外两个参数需要在[微软Azure门户](https://portal.azure.com/#view/Microsoft_AAD_RegisteredApps/ApplicationsListBlade)中注册应用后获取。
## 2. 应用注册
1. 登录[https://portal.azure.com/#view/Microsoft_AAD_RegisteredApps/ApplicationsListBlade](https://portal.azure.com/#view/Microsoft_AAD_RegisteredApps/ApplicationsListBlade),鼠标置于右上角账号处,浮窗将显示的`域`即为`onedrive_domain`的值。
![onedrive_domain](https://api.onedrive.com/v1.0/shares/s!Au-BDzXcM6_VmGCiErO85doq9Tcu/root/content)
2. 点击左上角的`+新注册`,输入名称,
* 受支持的帐户类型:选择任何组织目录(任何 Azure AD 目录 - 多租户)中的帐户和个人 Microsoft 帐户(例如,Skype、Xbox)
* 重定向 URI (可选):选择`Web`,并输入`http://localhost`
3. 完成注册后进入概述页面,在概要中找到`应用程序(客户端)ID`,即为`onedrive_client_id`的值。
![onedrive_client_id](https://api.onedrive.com/v1.0/shares/s!Au-BDzXcM6_VmGHD4CNyJxm_QBb8/root/content)
4. 此时还需要配置允许公共客户端流和API权限
* 在左侧选择`身份验证`,找到`允许的客户端流`,选择`是`,并**点击`保存`**。
![允许的客户端流](https://api.onedrive.com/v1.0/shares/s!Au-BDzXcM6_VmGJQMOlOCb2-L0Lh/root/content)
* 在左侧选择`API权限`,点击`+添加权限`,选择`Microsoft Graph`->`委托的权限`,并勾选下述权限:openid、Files中所有权限、User.Read,如下图所示。最后**点击下方的`添加权限`**。
![添加权限](https://api.onedrive.com/v1.0/shares/s!Au-BDzXcM6_VmGOZzz7sIrdXkD4w/root/content)
* 最后点击`授予管理员同意`,并**点击`是`**,最终状态变为`已授予`。
![授予管理员同意](https://api.onedrive.com/v1.0/shares/s!Au-BDzXcM6_VmGSOAnjnHUlbirbU/root/content)
## 3. 使用下述代码测试是否配置成功
安装依赖:`pip install Office365-REST-Python-Client`
```python
# common.py
import msal
domain = 'XXXXXX'
client_id = 'XXXXXX'
username = 'XXXXXX'
password = 'XXXXXX'
def acquire_token_pwd():
authority_url = f'https://login.microsoftonline.com/{domain}'
app = msal.PublicClientApplication(
authority=authority_url,
client_id=client_id
)
result = app.acquire_token_by_username_password(
username=username,
password=password,
scopes=['https://graph.microsoft.com/.default']
)
return result
```
测试登录,如果成功打印出账户名,说明配置成功。
```python
from common import acquire_token_pwd
from office365.graph_client import GraphClient
try:
client = GraphClient(acquire_token_pwd)
me = client.me.get().execute_query()
print(me.user_principal_name)
except Exception as e:
print(e)
```
测试文件上传
```python
import os
from office365.graph_client import GraphClient
from common import acquire_token_pwd
remote_path = 'tmp'
local_path = '.tmp/1689843925000.png'
def convert_link_to_download_link(link):
import re
p1 = re.search(r'https:\/\/(.+)\.sharepoint\.com', link).group(1)
p2 = re.search(r'personal\/(.+)\/', link).group(1)
p3 = re.search(rf'{p2}\/(.+)', link).group(1)
return f'https://{p1}.sharepoint.com/personal/{p2}/_layouts/52/download.aspx?share={p3}'
client = GraphClient(acquire_token_pwd)
folder = client.me.drive.root.get_by_path(remote_path)
# 1. upload
file = folder.upload_file(local_path).execute_query()
print(f'File {file.web_url} has been uploaded')
# 2. create sharing link
remote_file = folder.get_by_path(os.path.basename(local_path))
permission = remote_file.create_link("view", "anonymous").execute_query()
print(f"sharing link: {convert_link_to_download_link(permission.link.webUrl)}")
```
测试文件下载
```python
import os
from office365.graph_client import GraphClient
from common import acquire_token_pwd
remote_path = 'tmp/1689843925000.png'
local_path = '.tmp'
if not os.path.exists(local_path):
os.makedirs(local_path)
client = GraphClient(acquire_token_pwd)
remote_file = client.me.drive.root.get_by_path(remote_path).get().execute_query()
with open(os.path.join(local_path, os.path.basename(remote_path)), 'wb') as local_file:
remote_file.download(local_file).execute_query()
print(f'{remote_file.name} has been downloaded into {local_file.name}')
```
测试删除文件
```python
from office365.graph_client import GraphClient
from common import acquire_token_pwd
remote_path = 'tmp/1689843925000.png'
client = GraphClient(acquire_token_pwd)
file = client.me.drive.root.get_by_path(remote_path)
file.delete_object().execute_query()
```
-30
View File
@@ -1,30 +0,0 @@
# 通过 OpenDAL 集成存储的配置方法
## 需要配置的参数
```dotenv
file_storage=opendal
opendal_scheme=<service_name>
opendal_<service_name>_<service_setting>=...
```
以 Gcs 为例,需要配置的参数如下:
```dotenv
file_storage=opendal
opendal_scheme=gcs
opendal_gcs_root=<root>
opendal_gcs_bucket=<bucket_name>
opendal_gcs_credential=<base64_credential>
```
所有支持的服务可以在[此处](https://opendal.apache.org/docs/rust/opendal/services/index.html)查看。
具体服务的配置参数与 OpenDAL 文档一致。
## 补充说明
通过 OpenDAL 集成的服务均通过服务器中转下载。因此,每次下载既消耗存储服务的流量,也消耗服务器的流量。
OpenDAL 和该项目本身都支持本地存储、`s3`、`onedrive`。不同之处有以下几点:
1. 项目的支持通过预签名实现,不消耗服务器流量。而 OpenDAL 通过服务器中转下载,消耗服务器流量。(本地存储除外)
2. 项目的支持对于异常情况可能会有更多的调试信息,方便排查问题。
3. OpenDAL 项目本身采用 Rust 编写,性能更好。
+11 -13
View File
@@ -16,7 +16,6 @@ from tortoise.contrib.fastapi import register_tortoise
from apps.admin.views import admin_api
from apps.base.models import KeyValue
from apps.base.utils import ip_limit
from apps.base.views import share_api, chunk_api, presign_api
from core.config import (
ensure_security_settings,
@@ -755,16 +754,12 @@ async def load_config():
await KeyValue.update_or_create(
key="sys_start", defaults={"value": int(time.time() * 1000)}
)
# refresh_settings already syncs every rate limiter (error/metadata/upload/login)
# via _sync_ip_limits; do not hand-sync a subset here — that once drifted by
# missing the metadata limiter.
await refresh_settings()
await ensure_security_settings()
ip_limit["error"].minutes = settings.errorMinute
ip_limit["error"].count = settings.errorCount
ip_limit["upload"].minutes = settings.uploadMinute
ip_limit["upload"].count = settings.uploadCount
ip_limit["login"].minutes = settings.loginMinute
ip_limit["login"].count = settings.loginCount
app = FastAPI(lifespan=lifespan, version=APP_VERSION)
@app.middleware("http")
@@ -879,14 +874,17 @@ async def theme_asset(asset_path: str):
@app.exception_handler(404)
@app.get("/")
async def index(request=None, exc=None):
# Site config is admin input (and during the setup window anyone can claim it);
# always escape before injecting into the theme template to prevent stored XSS
# (mirrors the setup page).
return HTMLResponse(
content=resolve_theme_file("index.html")
.read_text(encoding="utf-8")
.replace("{{title}}", str(settings.name))
.replace("{{description}}", str(settings.description))
.replace("{{keywords}}", str(settings.keywords))
.replace("{{opacity}}", str(settings.opacity))
.replace("{{background}}", str(settings.background)),
.replace("{{title}}", html.escape(str(settings.name)))
.replace("{{description}}", html.escape(str(settings.description)))
.replace("{{keywords}}", html.escape(str(settings.keywords)))
.replace("{{opacity}}", html.escape(str(settings.opacity)))
.replace("{{background}}", html.escape(str(settings.background))),
media_type="text/html",
headers={"Cache-Control": "no-cache"},
)
+30
View File
@@ -0,0 +1,30 @@
# FileCodeBox tooling config.
# Application repo (not a library): no [project] packaging metadata here, and
# requirements.txt remains the single source of runtime deps. This file only
# holds tool configs plus the dev dependency group (PEP 735, `uv sync --group dev`).
[tool.pytest.ini_options]
testpaths = ["tests"]
# Tests import top-level packages (core/apps) directly; put the repo root on
# sys.path so bare `pytest` collects without `python -m pytest`.
pythonpath = ["."]
[dependency-groups]
dev = [
"pytest",
"pytest-asyncio",
"httpx",
"ruff",
"pre-commit",
]
[tool.ruff]
line-length = 120
target-version = "py312"
[tool.ruff.lint]
# Core correctness rules only for now (same families as ruff defaults):
# E4/E7/E9 = syntax/indent errors and unused variables; F = pyflakes
# (unused imports, undefined names). Style families (pyupgrade, broad-except,
# import sorting) are deferred to keep the initial diff minimal.
select = ["E4", "E7", "E9", "F"]
+1 -1
View File
@@ -2,7 +2,7 @@ aioboto3==15.5.0
aiohttp==3.14.2
aiofiles==25.1.0
fastapi==0.139.2
starlette==1.3.1
starlette==1.6.0
pydantic==2.12.5
uvicorn==0.51.0
tortoise-orm==0.25.3
+68
View File
@@ -0,0 +1,68 @@
import asyncio
import tempfile
import unittest
from pathlib import Path
import main
from core.settings import settings
class IndexTemplateEscapingTests(unittest.TestCase):
"""Regression: site config must be html.escape-d before template injection
(stored XSS prevention).
Uses a temp file as a stand-in for themes/*/index.html so the test also
runs in a bare checkout without built frontends.
"""
def setUp(self):
self._original_user_config = dict(settings.user_config)
self._original_resolve_theme_file = main.resolve_theme_file
tmp = tempfile.NamedTemporaryFile(
"w", suffix=".html", delete=False, encoding="utf-8"
)
tmp.write(
"<title>{{title}}</title>"
'<meta name="description" content="{{description}}">'
"<div>{{opacity}}</div>"
)
tmp.close()
self._template_path = Path(tmp.name)
def tearDown(self):
settings.user_config = self._original_user_config
main.resolve_theme_file = self._original_resolve_theme_file
self._template_path.unlink()
def _patch_template(self):
main.resolve_theme_file = lambda *args, **kwargs: self._template_path
def _render_index_html(self) -> str:
return asyncio.run(main.index()).body.decode("utf-8")
def test_malicious_site_config_is_escaped(self):
self._patch_template()
settings.name = "<script>alert(1)</script>"
settings.description = '"><img src=x onerror=alert(2)>'
html = self._render_index_html()
self.assertNotIn("<script>alert", html)
self.assertNotIn("<img", html)
self.assertIn("&lt;script&gt;", html)
def test_plain_site_config_renders_unchanged(self):
self._patch_template()
settings.name = "FileCodeBox"
settings.description = "A simple file share service"
settings.opacity = 0.9
html = self._render_index_html()
self.assertIn("<title>FileCodeBox</title>", html)
self.assertIn('content="A simple file share service"', html)
self.assertIn("<div>0.9</div>", html)
if __name__ == "__main__":
unittest.main()
+5
View File
@@ -6,6 +6,9 @@ from fastapi import HTTPException
from core.settings import BASE_DIR, settings
from main import index, resolve_theme_file
# themes/ is produced by the Dockerfile frontend build (see .gitignore); a bare checkout has no such directory
THEMES_BUILT = BASE_DIR.joinpath("themes/2024/assets").is_dir()
class SettingsOverrideMixin:
def setUp(self):
@@ -21,6 +24,7 @@ class ThemeAssetTests(SettingsOverrideMixin, unittest.TestCase):
self.assertTrue(assets, f"{theme} 缺少 index JS 资源")
return assets[0].name
@unittest.skipUnless(THEMES_BUILT, "themes/ exists only after the Docker frontend build; skip in bare checkouts")
def test_resolves_assets_from_current_theme(self):
settings.themesSelect = "themes/2023"
theme_2023_asset = resolve_theme_file(
@@ -43,6 +47,7 @@ class ThemeAssetTests(SettingsOverrideMixin, unittest.TestCase):
self.assertEqual(error.exception.status_code, 404)
@unittest.skipUnless(THEMES_BUILT, "themes/ exists only after the Docker frontend build; skip in bare checkouts")
def test_index_keeps_absolute_asset_urls(self):
settings.themesSelect = "themes/2023"
-1
View File
@@ -1,6 +1,5 @@
import asyncio
import hashlib
import time
import unittest
from unittest.mock import patch
+1 -3
View File
@@ -1,6 +1,5 @@
import asyncio
from io import BytesIO
import os
import tempfile
import unittest
from pathlib import Path
@@ -11,13 +10,12 @@ from starlette.datastructures import UploadFile
from tortoise import Tortoise
from apps.base import views
from apps.admin import dependencies as admin_dependencies
from apps.admin.dependencies import create_token, verify_token
from apps.admin.services import LocalFileClass
from apps.base.models import FileCodes, UploadChunk
from apps.base.schemas import CompleteUploadModel, InitChunkUploadModel
from apps.base.utils import get_chunk_file_path_name
from core.settings import data_root, settings
from core.settings import settings
from core.storage import SystemFileStorage
from core.utils import hash_password, verify_password