fix: #461 support s3 path style addressing

This commit is contained in:
Lan committed 2026-06-05 11:32:31 +08:00
1 parent aa6904a12c
commit 3fe136536e
3 files changed
+61 -53

No files matched your search

+1
View File
@@ -26,6 +26,7 @@ DEFAULT_CONFIG = {
"s3_region_name": "auto",
"s3_signature_version": "s3v2",
"s3_hostname": "",
"s3_addressing_style": "auto",
"s3_proxy": 0,
"max_save_seconds": 0,
"aws_session_token": "",
+27 -53
View File
@@ -267,6 +267,7 @@ class S3FileStorage(FileStorageInterface):
self.signature_version = settings.s3_signature_version
self.endpoint_url = settings.s3_endpoint_url or f"https://{self.s3_hostname}"
self.aws_session_token = settings.aws_session_token
self.addressing_style = str(settings.s3_addressing_style or "auto").lower()
self.proxy = settings.s3_proxy
self.session = aioboto3.Session(
aws_access_key_id=self.access_key_id,
@@ -278,14 +279,26 @@ class S3FileStorage(FileStorageInterface):
# 如果提供了 s3_endpoint_url,则优先使用它
self.endpoint_url = settings.s3_endpoint_url
def _client_config(self) -> Config:
config = {"signature_version": self.signature_version}
s3_config = {}
if self.addressing_style in {"path", "virtual", "auto"}:
s3_config["addressing_style"] = self.addressing_style
if s3_config:
config["s3"] = s3_config
return Config(**config)
def _client(self):
return self.session.client(
"s3",
endpoint_url=self.endpoint_url,
aws_session_token=self.aws_session_token,
region_name=self.region_name,
config=self._client_config(),
)
async def save_file(self, file: UploadFile, save_path: str):
async with self.session.client(
"s3",
endpoint_url=self.endpoint_url,
aws_session_token=self.aws_session_token,
region_name=self.region_name,
config=Config(signature_version=self.signature_version),
) as s3:
async with self._client() as s3:
# 使用 upload_fileobj 流式上传,避免将整个文件加载到内存
await s3.upload_fileobj(
file.file,
@@ -295,12 +308,7 @@ class S3FileStorage(FileStorageInterface):
)
async def delete_file(self, file_code: FileCodes):
async with self.session.client(
"s3",
endpoint_url=self.endpoint_url,
region_name=self.region_name,
config=Config(signature_version=self.signature_version),
) as s3:
async with self._client() as s3:
await s3.delete_object(
Bucket=self.bucket_name, Key=await file_code.get_file_path()
)
@@ -310,12 +318,7 @@ class S3FileStorage(FileStorageInterface):
filename = file_code.prefix + file_code.suffix
content_length = None # 初始化为 None,表示未知大小
async with self.session.client(
"s3",
endpoint_url=self.endpoint_url,
region_name=self.region_name,
config=Config(signature_version=self.signature_version),
) as s3:
async with self._client() as s3:
# 尝试获取文件大小(HEAD请求)
try:
head_response = await s3.head_object(
@@ -384,12 +387,7 @@ class S3FileStorage(FileStorageInterface):
if self.proxy:
return await get_file_url(file_code.code)
else:
async with self.session.client(
"s3",
endpoint_url=self.endpoint_url,
region_name=self.region_name,
config=Config(signature_version=self.signature_version),
) as s3:
async with self._client() as s3:
result = await s3.generate_presigned_url(
"get_object",
Params={
@@ -406,13 +404,7 @@ class S3FileStorage(FileStorageInterface):
注意:这里不使用 S3 原生的 multipart upload,而是将每个分片作为独立对象存储
"""
chunk_key = str(Path(save_path).parent / "chunks" / upload_id / f"{chunk_index}.part")
async with self.session.client(
's3',
endpoint_url=self.endpoint_url,
aws_session_token=self.aws_session_token,
region_name=self.region_name,
config=Config(signature_version=self.signature_version),
) as s3:
async with self._client() as s3:
# 将分片作为独立对象上传
await s3.put_object(
Bucket=self.bucket_name,
@@ -432,13 +424,7 @@ class S3FileStorage(FileStorageInterface):
file_sha256 = hashlib.sha256()
chunk_dir = str(Path(save_path).parent / "chunks" / upload_id)
async with self.session.client(
's3',
endpoint_url=self.endpoint_url,
aws_session_token=self.aws_session_token,
region_name=self.region_name,
config=Config(signature_version=self.signature_version),
) as s3:
async with self._client() as s3:
# 创建 multipart upload
mpu = await s3.create_multipart_upload(
Bucket=self.bucket_name,
@@ -512,13 +498,7 @@ class S3FileStorage(FileStorageInterface):
:param save_path: 文件保存路径
"""
chunk_dir = str(Path(save_path).parent / "chunks" / upload_id)
async with self.session.client(
's3',
endpoint_url=self.endpoint_url,
aws_session_token=self.aws_session_token,
region_name=self.region_name,
config=Config(signature_version=self.signature_version),
) as s3:
async with self._client() as s3:
try:
# 列出并删除所有分片对象
paginator = s3.get_paginator('list_objects_v2')
@@ -540,13 +520,7 @@ class S3FileStorage(FileStorageInterface):
:param expires_in: URL过期时间(秒),默认15分钟
:return: 预签名PUT URL
"""
async with self.session.client(
"s3",
endpoint_url=self.endpoint_url,
aws_session_token=self.aws_session_token,
region_name=self.region_name,
config=Config(signature_version=self.signature_version),
) as s3:
async with self._client() as s3:
return await s3.generate_presigned_url(
"put_object",
Params={
+33
View File
@@ -0,0 +1,33 @@
import unittest
from core.settings import settings
from core.storage import S3FileStorage
class SettingsOverrideMixin:
def setUp(self):
self._original_user_config = dict(settings.user_config)
def tearDown(self):
settings.user_config = self._original_user_config
class S3StorageConfigTests(SettingsOverrideMixin, unittest.TestCase):
def test_client_config_uses_path_style_when_configured(self):
storage = S3FileStorage.__new__(S3FileStorage)
storage.signature_version = "s3v4"
storage.addressing_style = "path"
config = storage._client_config()
self.assertEqual(config.signature_version, "s3v4")
self.assertEqual(config.s3["addressing_style"], "path")
def test_client_config_ignores_invalid_addressing_style(self):
storage = S3FileStorage.__new__(S3FileStorage)
storage.signature_version = "s3v4"
storage.addressing_style = "invalid"
config = storage._client_config()
self.assertIsNone(config.s3)