Files
FileCodeBox/tests/test_s3_oss_compatibility.py
vastsa 381a539c92 fix: resolve OSS chunk cleanup and legacy download issues
Use compatible S3 checksum settings, retry failed cleanup without reusing completed uploads, and include legacy attachment filenames. Correct admin-page URLs in both documentation languages.

Refs #556, #531, #502
2026-10-27 16:08:35 +08:00

315 lines
13 KiB
Python

"""OSS compatibility regressions: real SDK request preparation, no external I/O."""
from datetime import timedelta
from unittest.mock import AsyncMock
from urllib.parse import urlsplit
import aioboto3
import pytest
from botocore.exceptions import ClientError
from fastapi import HTTPException
from apps.base import services, tasks, views
from apps.base.models import FileCodes, StorageReservation, UploadChunk
from core.settings import settings
from core.storage.s3 import S3FileStorage
from core.utils import get_now
@pytest.fixture
def storage(monkeypatch):
monkeypatch.setattr(settings, "user_config", {
**settings.user_config, "file_storage": "s3", "upload_size": 1024,
"chunk_expire_hours": 24,
})
storage = S3FileStorage.__new__(S3FileStorage)
storage.signature_version = "s3v4"
storage.addressing_style = "virtual"
storage.bucket_name = "test-bucket"
# Reserved .invalid domain; before-send hooks stop requests before any I/O.
storage.endpoint_url = "https://oss-cn-hangzhou.example.invalid"
storage.region_name = "oss-cn-hangzhou"
storage.aws_session_token = None
storage.session = aioboto3.Session(
aws_access_key_id="testing", aws_secret_access_key="testing"
)
return storage
def test_checksum_config_preserves_signature_and_addressing(storage):
config = storage._client_config()
assert config.request_checksum_calculation == "when_required"
assert config.response_checksum_validation == "when_required"
assert config.signature_version == "s3v4"
assert config.s3["addressing_style"] == "virtual"
@pytest.mark.asyncio
@pytest.mark.parametrize("operation", ["PutObject", "UploadPart"])
async def test_sdk_upload_has_no_flexible_checksum_trailer(storage, monkeypatch, operation):
monkeypatch.setenv("AWS_REQUEST_CHECKSUM_CALCULATION", "when_supported")
monkeypatch.setenv("AWS_RESPONSE_CHECKSUM_VALIDATION", "when_supported")
captured = []
class RequestCaptured(Exception):
pass
def intercept(request, **kwargs):
captured.append(request)
raise RequestCaptured
# Register on the session so save_chunk's real _client() inherits the hook.
storage.session.events.register(f"before-send.s3.{operation}", intercept)
with pytest.raises(RequestCaptured):
if operation == "PutObject":
await storage.save_chunk("up", 0, b"payload", "hash", "share/file.bin")
else:
async with storage._client() as client:
assert client.meta.config.response_checksum_validation == "when_required"
await client.upload_part(
Bucket=storage.bucket_name, Key="share/file.bin", UploadId="up",
PartNumber=1, Body=b"payload",
)
assert len(captured) == 1
request = captured[0]
headers = {key.lower(): value for key, value in request.headers.items()}
assert "x-amz-trailer" not in headers
assert not any(key.startswith("x-amz-checksum-") for key in headers)
assert "x-amz-sdk-checksum-algorithm" not in headers
assert b"aws-chunked" not in headers.get("content-encoding", b"")
assert b"TRAILER" not in headers.get("x-amz-content-sha256", b"")
assert urlsplit(request.url).hostname == "test-bucket.oss-cn-hangzhou.example.invalid"
assert not urlsplit(request.url).path.startswith("/test-bucket/")
class FakeS3:
"""Only single-object deletion is available; paginate from a bounded snapshot."""
def __init__(self, keys=(), fail_key=None, list_error=False):
self.keys = set(keys)
self.fail_key = fail_key
self.list_error = list_error
self.deleted = []
self.prefixes = []
self.pages = 0
async def __aenter__(self):
return self
async def __aexit__(self, *args):
pass
def get_paginator(self, operation):
assert operation == "list_objects_v2"
return self
async def paginate(self, Bucket, Prefix):
assert Bucket == "test-bucket"
self.prefixes.append(Prefix)
if self.list_error:
raise RuntimeError("list failed")
# Also exercise a page without Contents.
yield {}
keys = sorted(key for key in self.keys if key.startswith(Prefix))
for offset in range(0, len(keys), 1000):
self.pages += 1
yield {"Contents": [{"Key": key} for key in keys[offset:offset + 1000]]}
async def delete_object(self, Bucket, Key):
assert Bucket == "test-bucket"
if Key == self.fail_key:
raise ClientError({"Error": {"Code": "AccessDenied"}}, "DeleteObject")
self.deleted.append(Key)
self.keys.discard(Key)
return {}
def install_client(storage, monkeypatch, fake):
monkeypatch.setattr(storage, "_client", lambda: fake)
monkeypatch.setattr(tasks, "storages", {"s3": lambda: storage})
monkeypatch.setattr(views, "storages", {"s3": lambda: storage})
monkeypatch.setattr(services.FileUploadService, "_storage", lambda: storage)
@pytest.mark.asyncio
async def test_cleanup_paginates_and_keeps_neighboring_upload(storage, monkeypatch):
prefix = "share/chunks/up/"
keys = {f"{prefix}{i}.part" for i in range(1005)}
neighbors = {"share/chunks/up-other/0.part", "share/file.bin"}
fake = FakeS3(keys | neighbors)
install_client(storage, monkeypatch, fake)
await storage.clean_chunks("up", "share/file.bin")
assert fake.prefixes == [prefix]
assert fake.pages == 2
assert set(fake.deleted) == keys
assert fake.keys == neighbors
await storage.clean_chunks("up", "share/file.bin") # Empty/idempotent retry.
assert fake.keys == neighbors
@pytest.mark.asyncio
async def test_cleanup_listing_failure_propagates(storage, monkeypatch):
install_client(storage, monkeypatch, FakeS3(list_error=True))
with pytest.raises(RuntimeError, match="list failed"):
await storage.clean_chunks("up", "share/file.bin")
async def seed_session(upload_id="up"):
now = await get_now()
values = dict(
upload_id=upload_id, chunk_hash="a" * 64, total_chunks=2,
file_size=8, chunk_size=4, file_name="file.bin", save_path="share/file.bin",
created_at=now - timedelta(hours=48),
)
session = await UploadChunk.create(chunk_index=-1, **values)
for index in range(2):
await UploadChunk.create(chunk_index=index, completed=True, **values)
await StorageReservation.create(
token=f"chunk:{upload_id}", size=8, expires_at=now + timedelta(hours=1)
)
return session
async def run_cleanup_round(monkeypatch):
class StopLoop(Exception):
pass
monkeypatch.setattr(tasks, "refresh_settings", AsyncMock())
monkeypatch.setattr(tasks.asyncio, "sleep", AsyncMock(side_effect=StopLoop))
with pytest.raises(StopLoop):
await tasks.clean_incomplete_uploads()
@pytest.mark.asyncio
async def test_partial_failure_keeps_records_then_task_retries(db, storage, monkeypatch):
await seed_session()
await seed_session("other")
keys = {"share/chunks/up/0.part", "share/chunks/up/1.part"}
fake = FakeS3(keys, fail_key="share/chunks/up/1.part")
install_client(storage, monkeypatch, fake)
with pytest.raises(ClientError):
await storage.clean_chunks("up", "share/file.bin")
assert fake.keys == {fake.fail_key}
await run_cleanup_round(monkeypatch)
assert await UploadChunk.filter(upload_id="up").count() == 3
assert await StorageReservation.filter(token="chunk:up").exists()
# Failure in one session must not prevent another session being cleaned.
assert not await UploadChunk.filter(upload_id="other").exists()
fake.fail_key = None
await run_cleanup_round(monkeypatch)
assert not fake.keys
assert not await UploadChunk.filter(upload_id="up").exists()
assert not await StorageReservation.filter(token="chunk:up").exists()
@pytest.mark.asyncio
@pytest.mark.parametrize("cleanup_fails", [False, True])
async def test_completed_share_survives_cleanup_failure(db, storage, monkeypatch, cleanup_fails):
session = await seed_session()
fake = FakeS3(list_error=cleanup_fails)
install_client(storage, monkeypatch, fake)
monkeypatch.setattr(storage, "merge_chunks", AsyncMock(return_value=("share/file.bin", "a" * 64)))
monkeypatch.setattr(services, "reserve_storage", AsyncMock())
monkeypatch.setattr(services, "get_expire_info", AsyncMock(return_value=(
await get_now() + timedelta(days=1), -1, 0, "oss-code",
)))
result = await services.FileUploadService.complete_chunked_upload("up", session, 1, "day")
assert result == {"code": "oss-code", "name": "file.bin"}
assert await FileCodes.filter(code="oss-code").exists()
assert await UploadChunk.filter(upload_id="up").exists() is cleanup_fails
assert not await StorageReservation.filter(token="chunk:up").exists()
if cleanup_fails:
fake.list_error = False
await run_cleanup_round(monkeypatch)
assert not await UploadChunk.filter(upload_id="up").exists()
assert await FileCodes.filter(code="oss-code").exists()
@pytest.mark.asyncio
async def test_oversize_cleanup_failure_keeps_retry_records(db, storage, monkeypatch):
session = await seed_session()
install_client(storage, monkeypatch, FakeS3(list_error=True))
monkeypatch.setattr(services, "reserve_storage", AsyncMock())
settings.user_config = {**settings.user_config, "upload_size": 1}
with pytest.raises(HTTPException) as error:
await services.FileUploadService.complete_chunked_upload("up", session, 1, "day")
assert error.value.status_code == 403
assert await UploadChunk.filter(upload_id="up").count() == 3
assert not await FileCodes.all().exists()
@pytest.mark.asyncio
async def test_cancel_cleanup_failure_keeps_retry_records(db, storage, monkeypatch):
await seed_session()
fake = FakeS3(list_error=True)
install_client(storage, monkeypatch, fake)
with pytest.raises(HTTPException) as error:
await views.cancel_upload("up")
assert error.value.status_code == 503
assert await UploadChunk.filter(upload_id="up").count() == 3
assert await StorageReservation.filter(token="chunk:up").exists()
fake.list_error = False
await views.cancel_upload("up")
assert not await UploadChunk.filter(upload_id="up").exists()
assert not await StorageReservation.filter(token="chunk:up").exists()
@pytest.mark.asyncio
@pytest.mark.parametrize("delete_share", [False, True])
async def test_published_session_cannot_be_reused(db, storage, monkeypatch, delete_share):
session = await seed_session()
install_client(storage, monkeypatch, FakeS3(list_error=True))
merge = AsyncMock(return_value=("share/file.bin", "a" * 64))
save_chunk = AsyncMock()
reserve = AsyncMock()
monkeypatch.setattr(storage, "merge_chunks", merge)
monkeypatch.setattr(storage, "save_chunk", save_chunk)
monkeypatch.setattr(services, "reserve_storage", reserve)
monkeypatch.setattr(views, "reserve_storage", reserve)
monkeypatch.setattr(services, "get_expire_info", AsyncMock(return_value=(
await get_now() + timedelta(days=1), -1, 0, "oss-code",
)))
result = await services.FileUploadService.complete_chunked_upload("up", session, 1, "day")
assert result["code"] == "oss-code"
persisted_session = await UploadChunk.get(upload_id="up", chunk_index=-1)
assert persisted_session.completed is True
assert await FileCodes.filter(upload_id="up").count() == 1
if delete_share:
# The terminal marker must survive expiration/deletion of the share itself.
await FileCodes.filter(upload_id="up").delete()
with pytest.raises(HTTPException) as error:
await services.FileUploadService.complete_chunked_upload(
"up", persisted_session, 1, "day"
)
assert error.value.status_code == 409
with pytest.raises(HTTPException) as error:
await views.complete_upload(
"up", views.CompleteUploadModel(expire_value=1, expire_style="day"), "127.0.0.1"
)
assert error.value.status_code == 409
# Even when a cleaned fragment no longer has a row, it cannot be uploaded again.
await UploadChunk.filter(upload_id="up", chunk_index=0).delete()
chunk = AsyncMock()
with pytest.raises(HTTPException) as error:
await views.upload_chunk("up", 0, chunk)
assert error.value.status_code == 409
chunk.read.assert_not_awaited()
save_chunk.assert_not_awaited()
merge.assert_awaited_once()
reserve.assert_awaited_once()
assert await FileCodes.filter(upload_id="up").count() == (0 if delete_share else 1)
# The -1 cleanup marker is not a file fragment or progress credit.
status = await views.get_upload_status("up")
assert status.detail["uploaded_chunks"] == [1]
assert status.detail["progress"] == 50
# Initializing the same file must create a new upload, not revive the old ID.
initialized = await views.init_chunk_upload(views.InitChunkUploadModel(
file_name="file.bin", file_size=8, chunk_size=4, file_hash="a" * 64,
))
assert initialized.detail["upload_id"] != "up"
assert initialized.detail["uploaded_chunks"] == []
assert (await UploadChunk.get(upload_id="up", chunk_index=-1)).completed is True