mirror of
https://github.com/vastsa/FileCodeBox.git
synced 2026-11-20 13:29:56 +08:00
Use compatible S3 checksum settings, retry failed cleanup without reusing completed uploads, and include legacy attachment filenames. Correct admin-page URLs in both documentation languages. Refs #556, #531, #502
315 lines
13 KiB
Python
315 lines
13 KiB
Python
"""OSS compatibility regressions: real SDK request preparation, no external I/O."""
|
|
from datetime import timedelta
|
|
from unittest.mock import AsyncMock
|
|
from urllib.parse import urlsplit
|
|
|
|
import aioboto3
|
|
import pytest
|
|
from botocore.exceptions import ClientError
|
|
from fastapi import HTTPException
|
|
|
|
from apps.base import services, tasks, views
|
|
from apps.base.models import FileCodes, StorageReservation, UploadChunk
|
|
from core.settings import settings
|
|
from core.storage.s3 import S3FileStorage
|
|
from core.utils import get_now
|
|
|
|
|
|
@pytest.fixture
|
|
def storage(monkeypatch):
|
|
monkeypatch.setattr(settings, "user_config", {
|
|
**settings.user_config, "file_storage": "s3", "upload_size": 1024,
|
|
"chunk_expire_hours": 24,
|
|
})
|
|
storage = S3FileStorage.__new__(S3FileStorage)
|
|
storage.signature_version = "s3v4"
|
|
storage.addressing_style = "virtual"
|
|
storage.bucket_name = "test-bucket"
|
|
# Reserved .invalid domain; before-send hooks stop requests before any I/O.
|
|
storage.endpoint_url = "https://oss-cn-hangzhou.example.invalid"
|
|
storage.region_name = "oss-cn-hangzhou"
|
|
storage.aws_session_token = None
|
|
storage.session = aioboto3.Session(
|
|
aws_access_key_id="testing", aws_secret_access_key="testing"
|
|
)
|
|
return storage
|
|
|
|
|
|
def test_checksum_config_preserves_signature_and_addressing(storage):
|
|
config = storage._client_config()
|
|
assert config.request_checksum_calculation == "when_required"
|
|
assert config.response_checksum_validation == "when_required"
|
|
assert config.signature_version == "s3v4"
|
|
assert config.s3["addressing_style"] == "virtual"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("operation", ["PutObject", "UploadPart"])
|
|
async def test_sdk_upload_has_no_flexible_checksum_trailer(storage, monkeypatch, operation):
|
|
monkeypatch.setenv("AWS_REQUEST_CHECKSUM_CALCULATION", "when_supported")
|
|
monkeypatch.setenv("AWS_RESPONSE_CHECKSUM_VALIDATION", "when_supported")
|
|
captured = []
|
|
|
|
class RequestCaptured(Exception):
|
|
pass
|
|
|
|
def intercept(request, **kwargs):
|
|
captured.append(request)
|
|
raise RequestCaptured
|
|
|
|
# Register on the session so save_chunk's real _client() inherits the hook.
|
|
storage.session.events.register(f"before-send.s3.{operation}", intercept)
|
|
with pytest.raises(RequestCaptured):
|
|
if operation == "PutObject":
|
|
await storage.save_chunk("up", 0, b"payload", "hash", "share/file.bin")
|
|
else:
|
|
async with storage._client() as client:
|
|
assert client.meta.config.response_checksum_validation == "when_required"
|
|
await client.upload_part(
|
|
Bucket=storage.bucket_name, Key="share/file.bin", UploadId="up",
|
|
PartNumber=1, Body=b"payload",
|
|
)
|
|
assert len(captured) == 1
|
|
request = captured[0]
|
|
headers = {key.lower(): value for key, value in request.headers.items()}
|
|
assert "x-amz-trailer" not in headers
|
|
assert not any(key.startswith("x-amz-checksum-") for key in headers)
|
|
assert "x-amz-sdk-checksum-algorithm" not in headers
|
|
assert b"aws-chunked" not in headers.get("content-encoding", b"")
|
|
assert b"TRAILER" not in headers.get("x-amz-content-sha256", b"")
|
|
assert urlsplit(request.url).hostname == "test-bucket.oss-cn-hangzhou.example.invalid"
|
|
assert not urlsplit(request.url).path.startswith("/test-bucket/")
|
|
|
|
|
|
class FakeS3:
|
|
"""Only single-object deletion is available; paginate from a bounded snapshot."""
|
|
|
|
def __init__(self, keys=(), fail_key=None, list_error=False):
|
|
self.keys = set(keys)
|
|
self.fail_key = fail_key
|
|
self.list_error = list_error
|
|
self.deleted = []
|
|
self.prefixes = []
|
|
self.pages = 0
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, *args):
|
|
pass
|
|
|
|
def get_paginator(self, operation):
|
|
assert operation == "list_objects_v2"
|
|
return self
|
|
|
|
async def paginate(self, Bucket, Prefix):
|
|
assert Bucket == "test-bucket"
|
|
self.prefixes.append(Prefix)
|
|
if self.list_error:
|
|
raise RuntimeError("list failed")
|
|
# Also exercise a page without Contents.
|
|
yield {}
|
|
keys = sorted(key for key in self.keys if key.startswith(Prefix))
|
|
for offset in range(0, len(keys), 1000):
|
|
self.pages += 1
|
|
yield {"Contents": [{"Key": key} for key in keys[offset:offset + 1000]]}
|
|
|
|
async def delete_object(self, Bucket, Key):
|
|
assert Bucket == "test-bucket"
|
|
if Key == self.fail_key:
|
|
raise ClientError({"Error": {"Code": "AccessDenied"}}, "DeleteObject")
|
|
self.deleted.append(Key)
|
|
self.keys.discard(Key)
|
|
return {}
|
|
|
|
|
|
def install_client(storage, monkeypatch, fake):
|
|
monkeypatch.setattr(storage, "_client", lambda: fake)
|
|
monkeypatch.setattr(tasks, "storages", {"s3": lambda: storage})
|
|
monkeypatch.setattr(views, "storages", {"s3": lambda: storage})
|
|
monkeypatch.setattr(services.FileUploadService, "_storage", lambda: storage)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cleanup_paginates_and_keeps_neighboring_upload(storage, monkeypatch):
|
|
prefix = "share/chunks/up/"
|
|
keys = {f"{prefix}{i}.part" for i in range(1005)}
|
|
neighbors = {"share/chunks/up-other/0.part", "share/file.bin"}
|
|
fake = FakeS3(keys | neighbors)
|
|
install_client(storage, monkeypatch, fake)
|
|
await storage.clean_chunks("up", "share/file.bin")
|
|
assert fake.prefixes == [prefix]
|
|
assert fake.pages == 2
|
|
assert set(fake.deleted) == keys
|
|
assert fake.keys == neighbors
|
|
await storage.clean_chunks("up", "share/file.bin") # Empty/idempotent retry.
|
|
assert fake.keys == neighbors
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cleanup_listing_failure_propagates(storage, monkeypatch):
|
|
install_client(storage, monkeypatch, FakeS3(list_error=True))
|
|
with pytest.raises(RuntimeError, match="list failed"):
|
|
await storage.clean_chunks("up", "share/file.bin")
|
|
|
|
|
|
async def seed_session(upload_id="up"):
|
|
now = await get_now()
|
|
values = dict(
|
|
upload_id=upload_id, chunk_hash="a" * 64, total_chunks=2,
|
|
file_size=8, chunk_size=4, file_name="file.bin", save_path="share/file.bin",
|
|
created_at=now - timedelta(hours=48),
|
|
)
|
|
session = await UploadChunk.create(chunk_index=-1, **values)
|
|
for index in range(2):
|
|
await UploadChunk.create(chunk_index=index, completed=True, **values)
|
|
await StorageReservation.create(
|
|
token=f"chunk:{upload_id}", size=8, expires_at=now + timedelta(hours=1)
|
|
)
|
|
return session
|
|
|
|
|
|
async def run_cleanup_round(monkeypatch):
|
|
class StopLoop(Exception):
|
|
pass
|
|
|
|
monkeypatch.setattr(tasks, "refresh_settings", AsyncMock())
|
|
monkeypatch.setattr(tasks.asyncio, "sleep", AsyncMock(side_effect=StopLoop))
|
|
with pytest.raises(StopLoop):
|
|
await tasks.clean_incomplete_uploads()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_partial_failure_keeps_records_then_task_retries(db, storage, monkeypatch):
|
|
await seed_session()
|
|
await seed_session("other")
|
|
keys = {"share/chunks/up/0.part", "share/chunks/up/1.part"}
|
|
fake = FakeS3(keys, fail_key="share/chunks/up/1.part")
|
|
install_client(storage, monkeypatch, fake)
|
|
with pytest.raises(ClientError):
|
|
await storage.clean_chunks("up", "share/file.bin")
|
|
assert fake.keys == {fake.fail_key}
|
|
await run_cleanup_round(monkeypatch)
|
|
assert await UploadChunk.filter(upload_id="up").count() == 3
|
|
assert await StorageReservation.filter(token="chunk:up").exists()
|
|
# Failure in one session must not prevent another session being cleaned.
|
|
assert not await UploadChunk.filter(upload_id="other").exists()
|
|
fake.fail_key = None
|
|
await run_cleanup_round(monkeypatch)
|
|
assert not fake.keys
|
|
assert not await UploadChunk.filter(upload_id="up").exists()
|
|
assert not await StorageReservation.filter(token="chunk:up").exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("cleanup_fails", [False, True])
|
|
async def test_completed_share_survives_cleanup_failure(db, storage, monkeypatch, cleanup_fails):
|
|
session = await seed_session()
|
|
fake = FakeS3(list_error=cleanup_fails)
|
|
install_client(storage, monkeypatch, fake)
|
|
monkeypatch.setattr(storage, "merge_chunks", AsyncMock(return_value=("share/file.bin", "a" * 64)))
|
|
monkeypatch.setattr(services, "reserve_storage", AsyncMock())
|
|
monkeypatch.setattr(services, "get_expire_info", AsyncMock(return_value=(
|
|
await get_now() + timedelta(days=1), -1, 0, "oss-code",
|
|
)))
|
|
result = await services.FileUploadService.complete_chunked_upload("up", session, 1, "day")
|
|
assert result == {"code": "oss-code", "name": "file.bin"}
|
|
assert await FileCodes.filter(code="oss-code").exists()
|
|
assert await UploadChunk.filter(upload_id="up").exists() is cleanup_fails
|
|
assert not await StorageReservation.filter(token="chunk:up").exists()
|
|
if cleanup_fails:
|
|
fake.list_error = False
|
|
await run_cleanup_round(monkeypatch)
|
|
assert not await UploadChunk.filter(upload_id="up").exists()
|
|
assert await FileCodes.filter(code="oss-code").exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_oversize_cleanup_failure_keeps_retry_records(db, storage, monkeypatch):
|
|
session = await seed_session()
|
|
install_client(storage, monkeypatch, FakeS3(list_error=True))
|
|
monkeypatch.setattr(services, "reserve_storage", AsyncMock())
|
|
settings.user_config = {**settings.user_config, "upload_size": 1}
|
|
with pytest.raises(HTTPException) as error:
|
|
await services.FileUploadService.complete_chunked_upload("up", session, 1, "day")
|
|
assert error.value.status_code == 403
|
|
assert await UploadChunk.filter(upload_id="up").count() == 3
|
|
assert not await FileCodes.all().exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cancel_cleanup_failure_keeps_retry_records(db, storage, monkeypatch):
|
|
await seed_session()
|
|
fake = FakeS3(list_error=True)
|
|
install_client(storage, monkeypatch, fake)
|
|
with pytest.raises(HTTPException) as error:
|
|
await views.cancel_upload("up")
|
|
assert error.value.status_code == 503
|
|
assert await UploadChunk.filter(upload_id="up").count() == 3
|
|
assert await StorageReservation.filter(token="chunk:up").exists()
|
|
fake.list_error = False
|
|
await views.cancel_upload("up")
|
|
assert not await UploadChunk.filter(upload_id="up").exists()
|
|
assert not await StorageReservation.filter(token="chunk:up").exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("delete_share", [False, True])
|
|
async def test_published_session_cannot_be_reused(db, storage, monkeypatch, delete_share):
|
|
session = await seed_session()
|
|
install_client(storage, monkeypatch, FakeS3(list_error=True))
|
|
merge = AsyncMock(return_value=("share/file.bin", "a" * 64))
|
|
save_chunk = AsyncMock()
|
|
reserve = AsyncMock()
|
|
monkeypatch.setattr(storage, "merge_chunks", merge)
|
|
monkeypatch.setattr(storage, "save_chunk", save_chunk)
|
|
monkeypatch.setattr(services, "reserve_storage", reserve)
|
|
monkeypatch.setattr(views, "reserve_storage", reserve)
|
|
monkeypatch.setattr(services, "get_expire_info", AsyncMock(return_value=(
|
|
await get_now() + timedelta(days=1), -1, 0, "oss-code",
|
|
)))
|
|
result = await services.FileUploadService.complete_chunked_upload("up", session, 1, "day")
|
|
assert result["code"] == "oss-code"
|
|
persisted_session = await UploadChunk.get(upload_id="up", chunk_index=-1)
|
|
assert persisted_session.completed is True
|
|
assert await FileCodes.filter(upload_id="up").count() == 1
|
|
if delete_share:
|
|
# The terminal marker must survive expiration/deletion of the share itself.
|
|
await FileCodes.filter(upload_id="up").delete()
|
|
|
|
with pytest.raises(HTTPException) as error:
|
|
await services.FileUploadService.complete_chunked_upload(
|
|
"up", persisted_session, 1, "day"
|
|
)
|
|
assert error.value.status_code == 409
|
|
with pytest.raises(HTTPException) as error:
|
|
await views.complete_upload(
|
|
"up", views.CompleteUploadModel(expire_value=1, expire_style="day"), "127.0.0.1"
|
|
)
|
|
assert error.value.status_code == 409
|
|
|
|
# Even when a cleaned fragment no longer has a row, it cannot be uploaded again.
|
|
await UploadChunk.filter(upload_id="up", chunk_index=0).delete()
|
|
chunk = AsyncMock()
|
|
with pytest.raises(HTTPException) as error:
|
|
await views.upload_chunk("up", 0, chunk)
|
|
assert error.value.status_code == 409
|
|
chunk.read.assert_not_awaited()
|
|
save_chunk.assert_not_awaited()
|
|
merge.assert_awaited_once()
|
|
reserve.assert_awaited_once()
|
|
assert await FileCodes.filter(upload_id="up").count() == (0 if delete_share else 1)
|
|
|
|
# The -1 cleanup marker is not a file fragment or progress credit.
|
|
status = await views.get_upload_status("up")
|
|
assert status.detail["uploaded_chunks"] == [1]
|
|
assert status.detail["progress"] == 50
|
|
|
|
# Initializing the same file must create a new upload, not revive the old ID.
|
|
initialized = await views.init_chunk_upload(views.InitChunkUploadModel(
|
|
file_name="file.bin", file_size=8, chunk_size=4, file_hash="a" * 64,
|
|
))
|
|
assert initialized.detail["upload_id"] != "up"
|
|
assert initialized.detail["uploaded_chunks"] == []
|
|
assert (await UploadChunk.get(upload_id="up", chunk_index=-1)).completed is True
|