feat: add local fallback for S3 writes
This commit is contained in:
1
Cargo.lock
generated
1
Cargo.lock
generated
@@ -2169,6 +2169,7 @@ dependencies = [
|
|||||||
"axum",
|
"axum",
|
||||||
"axum-extra",
|
"axum-extra",
|
||||||
"base64 0.22.1",
|
"base64 0.22.1",
|
||||||
|
"bytes",
|
||||||
"chrono",
|
"chrono",
|
||||||
"dotenvy",
|
"dotenvy",
|
||||||
"hex",
|
"hex",
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ time = "0.3"
|
|||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
thiserror = "1"
|
thiserror = "1"
|
||||||
|
bytes = "1"
|
||||||
|
|
||||||
tracing = "0.1"
|
tracing = "0.1"
|
||||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
|
|||||||
@@ -731,6 +731,8 @@ Authorization: Bearer <admin_token>
|
|||||||
|
|
||||||
凭据加密保存且不通过 API 回传。测试接口执行 Bucket 检查、内部临时对象读写删和公网预签名下载;激活接口会再次测试并原子切换活动端点。活动端点不能直接编辑或删除。
|
凭据加密保存且不通过 API 回传。测试接口执行 Bucket 检查、内部临时对象读写删和公网预签名下载;激活接口会再次测试并原子切换活动端点。活动端点不能直接编辑或删除。
|
||||||
|
|
||||||
|
列表响应中的 `local_object_count` 与 `local_stored_bytes` 统计当前实际位于应用服务器的成品。存在活动 S3 时,新对象仍优先写入 S3;若本次 S3 写入失败,则自动回退本地并按实际后端完成下载和到期清理。
|
||||||
|
|
||||||
### 11.7 兑换码管理
|
### 11.7 兑换码管理
|
||||||
|
|
||||||
```http
|
```http
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ flowchart LR
|
|||||||
4. 批处理原图只在 118 的共享 `uploads/orig` 临时保存,Worker 完成或失败后删除,不上传 S3。
|
4. 批处理原图只在 118 的共享 `uploads/orig` 临时保存,Worker 完成或失败后删除,不上传 S3。
|
||||||
5. 批量 ZIP 在 118 临时生成,64 MiB 以上自动使用 S3 分片上传,上传后删除临时目录。
|
5. 批量 ZIP 在 118 临时生成,64 MiB 以上自动使用 S3 分片上传,上传后删除临时目录。
|
||||||
6. 每个成品记录写入时的端点 ID。以后切换端点不会使旧文件失联;旧端点凭据会保留到关联对象清空。
|
6. 每个成品记录写入时的端点 ID。以后切换端点不会使旧文件失联;旧端点凭据会保留到关联对象清空。
|
||||||
|
7. 活动 S3 仍是首选后端;连接、凭据或上传失败时,本次成品自动写入 118 本地卷并记录 `storage_backend=local`。S3 恢复后,后续新对象会再次优先写入 S3。
|
||||||
|
|
||||||
`/api/v1/compress/direct` 保留原有“响应体直接返回图片”的 API 语义,避免破坏现有调用方;网站下载、历史记录下载、普通压缩结果和批量 ZIP 均走 S3。
|
`/api/v1/compress/direct` 保留原有“响应体直接返回图片”的 API 语义,避免破坏现有调用方;网站下载、历史记录下载、普通压缩结果和批量 ZIP 均走 S3。
|
||||||
|
|
||||||
@@ -126,6 +127,7 @@ aws --endpoint-url http://127.0.0.1:3900 \
|
|||||||
- 未授权用户无法取得签名 URL;任务过期后应用下载接口返回 404。
|
- 未授权用户无法取得签名 URL;任务过期后应用下载接口返回 404。
|
||||||
- 单文件与批任务数据库均记录正确的 `storage_endpoint_id` 和对象键。
|
- 单文件与批任务数据库均记录正确的 `storage_endpoint_id` 和对象键。
|
||||||
- 切换到第二端点后,新对象进入第二端点,第一端点历史对象仍可下载。
|
- 切换到第二端点后,新对象进入第二端点,第一端点历史对象仍可下载。
|
||||||
- S3 停止时同步请求返回 503,批任务保留临时原图并重试,不静默写回本地。
|
- S3 停止时同步请求和批任务均完成本地兜底,数据库记录 `storage_backend=local`,下载接口直接从应用服务器返回文件。
|
||||||
|
- S3 恢复后下一次新对象重新记录为 `storage_backend=s3`,下载接口恢复 `307` 到签名 URL。
|
||||||
- 删除 S3 对象失败时,过期任务数据库记录保留并在下一轮重试。
|
- 删除 S3 对象失败时,过期任务数据库记录保留并在下一轮重试。
|
||||||
- 119 磁盘 70%/85% 告警、容器重启、NTP 时间同步和证书续期均验证通过。
|
- 119 磁盘 70%/85% 告警、容器重启、NTP 时间同步和证书续期均验证通过。
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ const auth = useAuthStore()
|
|||||||
const router = useRouter()
|
const router = useRouter()
|
||||||
|
|
||||||
const isLoggedIn = computed(() => auth.isLoggedIn)
|
const isLoggedIn = computed(() => auth.isLoggedIn)
|
||||||
|
const consoleRoute = computed(() => (auth.user?.role === 'admin' ? '/admin' : '/dashboard'))
|
||||||
|
const consoleLabel = computed(() => (auth.user?.role === 'admin' ? '管理后台' : '控制台'))
|
||||||
|
|
||||||
function logout() {
|
function logout() {
|
||||||
auth.logout()
|
auth.logout()
|
||||||
@@ -35,10 +37,10 @@ function logout() {
|
|||||||
<div class="flex items-center gap-3">
|
<div class="flex items-center gap-3">
|
||||||
<template v-if="isLoggedIn">
|
<template v-if="isLoggedIn">
|
||||||
<RouterLink
|
<RouterLink
|
||||||
to="/dashboard"
|
:to="consoleRoute"
|
||||||
class="rounded-md border border-slate-200 bg-white px-3 py-1.5 text-sm text-slate-700 hover:bg-slate-50"
|
class="rounded-md border border-slate-200 bg-white px-3 py-1.5 text-sm text-slate-700 hover:bg-slate-50"
|
||||||
>
|
>
|
||||||
控制台
|
{{ consoleLabel }}
|
||||||
</RouterLink>
|
</RouterLink>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ export function createAppRouter(pinia: Pinia) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if ((to.name === 'login' || to.name === 'register') && auth.isLoggedIn) {
|
if ((to.name === 'login' || to.name === 'register') && auth.isLoggedIn) {
|
||||||
return { name: 'dashboard' }
|
return { name: auth.user?.role === 'admin' ? 'admin' : 'dashboard' }
|
||||||
}
|
}
|
||||||
|
|
||||||
if (to.meta?.requiresAdmin && auth.user?.role !== 'admin') {
|
if (to.meta?.requiresAdmin && auth.user?.role !== 'admin') {
|
||||||
|
|||||||
@@ -22,8 +22,10 @@ async function submit() {
|
|||||||
const resp = await login(email.value.trim(), password.value)
|
const resp = await login(email.value.trim(), password.value)
|
||||||
auth.setAuth(resp.token, resp.user)
|
auth.setAuth(resp.token, resp.user)
|
||||||
|
|
||||||
const redirect = typeof route.query.redirect === 'string' ? route.query.redirect : '/dashboard'
|
const defaultRoute = resp.user.role === 'admin' ? '/admin' : '/dashboard'
|
||||||
await router.push(redirect)
|
const requestedRoute = typeof route.query.redirect === 'string' ? route.query.redirect : null
|
||||||
|
const redirect = requestedRoute?.startsWith('/') && !requestedRoute.startsWith('//') ? requestedRoute : defaultRoute
|
||||||
|
await router.replace(redirect)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err instanceof ApiError) {
|
if (err instanceof ApiError) {
|
||||||
error.value = `[${err.code}] ${err.message}`
|
error.value = `[${err.code}] ${err.message}`
|
||||||
|
|||||||
@@ -20,6 +20,8 @@ const loading = ref(true)
|
|||||||
const error = ref<string | null>(null)
|
const error = ref<string | null>(null)
|
||||||
const message = ref<string | null>(null)
|
const message = ref<string | null>(null)
|
||||||
const activeBackend = ref<'local' | 's3'>('local')
|
const activeBackend = ref<'local' | 's3'>('local')
|
||||||
|
const localObjectCount = ref(0)
|
||||||
|
const localStoredBytes = ref(0)
|
||||||
const endpoints = ref<AdminStorageEndpoint[]>([])
|
const endpoints = ref<AdminStorageEndpoint[]>([])
|
||||||
const editingId = ref<string | null>(null)
|
const editingId = ref<string | null>(null)
|
||||||
const formOpen = ref(false)
|
const formOpen = ref(false)
|
||||||
@@ -59,6 +61,8 @@ async function loadEndpoints(clearError = true) {
|
|||||||
const response = await listStorageEndpoints(auth.token)
|
const response = await listStorageEndpoints(auth.token)
|
||||||
endpoints.value = response.endpoints
|
endpoints.value = response.endpoints
|
||||||
activeBackend.value = response.active_backend
|
activeBackend.value = response.active_backend
|
||||||
|
localObjectCount.value = response.local_object_count
|
||||||
|
localStoredBytes.value = response.local_stored_bytes
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
error.value = errorText(err, '加载存储配置失败')
|
error.value = errorText(err, '加载存储配置失败')
|
||||||
} finally {
|
} finally {
|
||||||
@@ -207,7 +211,7 @@ onMounted(loadEndpoints)
|
|||||||
<p class="text-xs font-semibold uppercase tracking-[0.22em] text-cyan-300">Storage control plane</p>
|
<p class="text-xs font-semibold uppercase tracking-[0.22em] text-cyan-300">Storage control plane</p>
|
||||||
<h2 class="mt-3 text-2xl font-semibold">对象存储</h2>
|
<h2 class="mt-3 text-2xl font-semibold">对象存储</h2>
|
||||||
<p class="mt-2 max-w-2xl text-sm leading-6 text-slate-300">
|
<p class="mt-2 max-w-2xl text-sm leading-6 text-slate-300">
|
||||||
应用完成鉴权后返回短期签名地址,图片与批量 ZIP 的下载流量直接由高带宽 S3 节点承担。每个对象固定绑定写入端点,后续切换或扩容不会影响旧文件。
|
新对象优先写入活动 S3;连接或写入失败时自动保存到应用服务器本地。每个对象固定记录实际写入后端,恢复 S3 后的新文件会自动重新走对象存储。
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
<div class="rounded-xl border border-white/10 bg-white/5 p-4">
|
<div class="rounded-xl border border-white/10 bg-white/5 p-4">
|
||||||
@@ -217,12 +221,16 @@ onMounted(loadEndpoints)
|
|||||||
class="rounded-full px-3 py-1 text-xs font-semibold"
|
class="rounded-full px-3 py-1 text-xs font-semibold"
|
||||||
:class="activeBackend === 's3' ? 'bg-emerald-400/15 text-emerald-300' : 'bg-amber-400/15 text-amber-200'"
|
:class="activeBackend === 's3' ? 'bg-emerald-400/15 text-emerald-300' : 'bg-amber-400/15 text-amber-200'"
|
||||||
>
|
>
|
||||||
{{ activeBackend === 's3' ? 'S3 已启用' : '本地回退' }}
|
{{ activeBackend === 's3' ? 'S3 优先' : '仅本地' }}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<p class="mt-3 text-xs leading-5 text-slate-400">
|
<p class="mt-3 text-xs leading-5 text-slate-400">
|
||||||
推荐内部 Endpoint 走两台服务器之间的 WireGuard 地址;公网 Endpoint 使用带 TLS 的下载域名。签名默认 5 分钟有效。
|
S3 故障只影响当次写入,下载与到期清理会按对象记录的真实后端执行。签名默认 5 分钟有效。
|
||||||
</p>
|
</p>
|
||||||
|
<div class="mt-3 flex items-center justify-between border-t border-white/10 pt-3 text-xs">
|
||||||
|
<span class="text-slate-400">本地对象</span>
|
||||||
|
<span class="font-medium text-slate-200">{{ localObjectCount }} 个 · {{ formatBytes(localStoredBytes) }}</span>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
@@ -238,7 +246,7 @@ onMounted(loadEndpoints)
|
|||||||
<div class="flex flex-wrap items-center justify-between gap-3">
|
<div class="flex flex-wrap items-center justify-between gap-3">
|
||||||
<div>
|
<div>
|
||||||
<h3 class="font-semibold text-slate-900">存储端点</h3>
|
<h3 class="font-semibold text-slate-900">存储端点</h3>
|
||||||
<p class="mt-1 text-sm text-slate-500">同一时间只有一个端点接收新对象,停用端点继续服务其历史对象。</p>
|
<p class="mt-1 text-sm text-slate-500">同一时间只有一个首选 S3 端点;写入失败自动回退本地,停用端点继续服务其历史对象。</p>
|
||||||
</div>
|
</div>
|
||||||
<button
|
<button
|
||||||
class="rounded-lg bg-slate-900 px-4 py-2 text-sm font-medium text-white hover:bg-slate-800"
|
class="rounded-lg bg-slate-900 px-4 py-2 text-sm font-medium text-white hover:bg-slate-800"
|
||||||
|
|||||||
@@ -669,6 +669,8 @@ export interface AdminStorageEndpoint {
|
|||||||
|
|
||||||
export interface AdminStorageEndpointsResponse {
|
export interface AdminStorageEndpointsResponse {
|
||||||
active_backend: 'local' | 's3'
|
active_backend: 'local' | 's3'
|
||||||
|
local_object_count: number
|
||||||
|
local_stored_bytes: number
|
||||||
endpoints: AdminStorageEndpoint[]
|
endpoints: AdminStorageEndpoint[]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -60,6 +60,8 @@ struct StorageEndpointView {
|
|||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
struct StorageEndpointsResponse {
|
struct StorageEndpointsResponse {
|
||||||
active_backend: String,
|
active_backend: String,
|
||||||
|
local_object_count: i64,
|
||||||
|
local_stored_bytes: i64,
|
||||||
endpoints: Vec<StorageEndpointView>,
|
endpoints: Vec<StorageEndpointView>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -109,6 +111,7 @@ async fn list_storage_endpoints(
|
|||||||
} else {
|
} else {
|
||||||
"local"
|
"local"
|
||||||
};
|
};
|
||||||
|
let (local_object_count, local_stored_bytes) = local_usage(&state).await?;
|
||||||
let mut views = Vec::with_capacity(endpoints.len());
|
let mut views = Vec::with_capacity(endpoints.len());
|
||||||
for endpoint in endpoints {
|
for endpoint in endpoints {
|
||||||
views.push(endpoint_view(&state, endpoint).await?);
|
views.push(endpoint_view(&state, endpoint).await?);
|
||||||
@@ -118,6 +121,8 @@ async fn list_storage_endpoints(
|
|||||||
success: true,
|
success: true,
|
||||||
data: StorageEndpointsResponse {
|
data: StorageEndpointsResponse {
|
||||||
active_backend: active_backend.to_string(),
|
active_backend: active_backend.to_string(),
|
||||||
|
local_object_count,
|
||||||
|
local_stored_bytes,
|
||||||
endpoints: views,
|
endpoints: views,
|
||||||
},
|
},
|
||||||
}))
|
}))
|
||||||
@@ -536,6 +541,28 @@ async fn endpoint_usage(state: &AppState, endpoint_id: Uuid) -> Result<(i64, i64
|
|||||||
.map_err(|err| AppError::new(ErrorCode::Internal, "统计存储使用量失败").with_source(err))
|
.map_err(|err| AppError::new(ErrorCode::Internal, "统计存储使用量失败").with_source(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn local_usage(state: &AppState) -> Result<(i64, i64), AppError> {
|
||||||
|
sqlx::query_as::<_, (i64, i64)>(
|
||||||
|
r#"
|
||||||
|
SELECT
|
||||||
|
(SELECT COUNT(*) FROM task_files
|
||||||
|
WHERE status = 'completed'
|
||||||
|
AND storage_backend = 'local'
|
||||||
|
AND COALESCE(storage_key, storage_path) IS NOT NULL)
|
||||||
|
+ (SELECT COUNT(*) FROM tasks
|
||||||
|
WHERE zip_storage_backend = 'local'
|
||||||
|
AND zip_storage_key IS NOT NULL) AS object_count,
|
||||||
|
COALESCE((SELECT SUM(compressed_size)::BIGINT FROM task_files
|
||||||
|
WHERE status = 'completed' AND storage_backend = 'local'), 0::BIGINT)
|
||||||
|
+ COALESCE((SELECT SUM(zip_size)::BIGINT FROM tasks
|
||||||
|
WHERE zip_storage_backend = 'local'), 0::BIGINT) AS stored_bytes
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.fetch_one(&state.db)
|
||||||
|
.await
|
||||||
|
.map_err(|err| AppError::new(ErrorCode::Internal, "统计本地存储使用量失败").with_source(err))
|
||||||
|
}
|
||||||
|
|
||||||
async fn record_test_result(
|
async fn record_test_result(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
endpoint_id: Uuid,
|
endpoint_id: Uuid,
|
||||||
|
|||||||
@@ -3,12 +3,14 @@ use crate::services::settings;
|
|||||||
use crate::state::AppState;
|
use crate::state::AppState;
|
||||||
|
|
||||||
use aws_sdk_s3::config::{
|
use aws_sdk_s3::config::{
|
||||||
BehaviorVersion, Credentials, Region, RequestChecksumCalculation, ResponseChecksumValidation,
|
retry::RetryConfig, timeout::TimeoutConfig, BehaviorVersion, Credentials, Region,
|
||||||
|
RequestChecksumCalculation, ResponseChecksumValidation,
|
||||||
};
|
};
|
||||||
use aws_sdk_s3::presigning::PresigningConfig;
|
use aws_sdk_s3::presigning::PresigningConfig;
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||||
use aws_sdk_s3::Client;
|
use aws_sdk_s3::Client;
|
||||||
|
use bytes::Bytes;
|
||||||
use chrono::{DateTime, Datelike, Utc};
|
use chrono::{DateTime, Datelike, Utc};
|
||||||
use sqlx::FromRow;
|
use sqlx::FromRow;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
@@ -151,28 +153,50 @@ pub async fn store_bytes(
|
|||||||
bytes: Vec<u8>,
|
bytes: Vec<u8>,
|
||||||
content_type: &str,
|
content_type: &str,
|
||||||
) -> Result<StoredObject, AppError> {
|
) -> Result<StoredObject, AppError> {
|
||||||
|
let bytes = Bytes::from(bytes);
|
||||||
if let Some(endpoint) = active_endpoint(state).await? {
|
if let Some(endpoint) = active_endpoint(state).await? {
|
||||||
let client = client_for(state, &endpoint, EndpointKind::Internal)?;
|
match store_bytes_s3(state, &endpoint, key, bytes.clone(), content_type).await {
|
||||||
let size = bytes.len() as u64;
|
Ok(stored) => return Ok(stored),
|
||||||
let output = client
|
Err(err) => log_local_fallback(&endpoint, key, &err),
|
||||||
.put_object()
|
}
|
||||||
.bucket(&endpoint.bucket)
|
|
||||||
.key(key)
|
|
||||||
.content_type(content_type)
|
|
||||||
.body(ByteStream::from(bytes))
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(|err| storage_error("上传 S3 对象失败", err))?;
|
|
||||||
|
|
||||||
return Ok(StoredObject {
|
|
||||||
backend: "s3".to_string(),
|
|
||||||
endpoint_id: Some(endpoint.id),
|
|
||||||
key: key.to_string(),
|
|
||||||
etag: output.e_tag().map(ToOwned::to_owned),
|
|
||||||
size,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
store_bytes_local(state, key, bytes.as_ref()).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn store_bytes_s3(
|
||||||
|
state: &AppState,
|
||||||
|
endpoint: &StorageEndpoint,
|
||||||
|
key: &str,
|
||||||
|
bytes: Bytes,
|
||||||
|
content_type: &str,
|
||||||
|
) -> Result<StoredObject, AppError> {
|
||||||
|
let client = client_for(state, endpoint, EndpointKind::Internal)?;
|
||||||
|
let size = bytes.len() as u64;
|
||||||
|
let output = client
|
||||||
|
.put_object()
|
||||||
|
.bucket(&endpoint.bucket)
|
||||||
|
.key(key)
|
||||||
|
.content_type(content_type)
|
||||||
|
.body(ByteStream::from(bytes))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|err| storage_error("上传 S3 对象失败", err))?;
|
||||||
|
|
||||||
|
Ok(StoredObject {
|
||||||
|
backend: "s3".to_string(),
|
||||||
|
endpoint_id: Some(endpoint.id),
|
||||||
|
key: key.to_string(),
|
||||||
|
etag: output.e_tag().map(ToOwned::to_owned),
|
||||||
|
size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn store_bytes_local(
|
||||||
|
state: &AppState,
|
||||||
|
key: &str,
|
||||||
|
bytes: &[u8],
|
||||||
|
) -> Result<StoredObject, AppError> {
|
||||||
let path = local_path(state, key)?;
|
let path = local_path(state, key)?;
|
||||||
if let Some(parent) = path.parent() {
|
if let Some(parent) = path.parent() {
|
||||||
tokio::fs::create_dir_all(parent).await.map_err(|err| {
|
tokio::fs::create_dir_all(parent).await.map_err(|err| {
|
||||||
@@ -203,35 +227,58 @@ pub async fn store_file(
|
|||||||
})?;
|
})?;
|
||||||
|
|
||||||
if let Some(endpoint) = active_endpoint(state).await? {
|
if let Some(endpoint) = active_endpoint(state).await? {
|
||||||
let client = client_for(state, &endpoint, EndpointKind::Internal)?;
|
match store_file_s3(state, &endpoint, key, path, content_type, metadata.len()).await {
|
||||||
let etag = if metadata.len() >= MULTIPART_THRESHOLD {
|
Ok(stored) => return Ok(stored),
|
||||||
multipart_upload(&client, &endpoint.bucket, key, path, content_type).await?
|
Err(err) => log_local_fallback(&endpoint, key, &err),
|
||||||
} else {
|
}
|
||||||
let body = ByteStream::from_path(path).await.map_err(|err| {
|
|
||||||
AppError::new(ErrorCode::StorageUnavailable, "读取待上传文件失败").with_source(err)
|
|
||||||
})?;
|
|
||||||
client
|
|
||||||
.put_object()
|
|
||||||
.bucket(&endpoint.bucket)
|
|
||||||
.key(key)
|
|
||||||
.content_type(content_type)
|
|
||||||
.body(body)
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(|err| storage_error("上传 S3 对象失败", err))?
|
|
||||||
.e_tag()
|
|
||||||
.map(ToOwned::to_owned)
|
|
||||||
};
|
|
||||||
|
|
||||||
return Ok(StoredObject {
|
|
||||||
backend: "s3".to_string(),
|
|
||||||
endpoint_id: Some(endpoint.id),
|
|
||||||
key: key.to_string(),
|
|
||||||
etag,
|
|
||||||
size: metadata.len(),
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
store_file_local(state, key, path, metadata.len()).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn store_file_s3(
|
||||||
|
state: &AppState,
|
||||||
|
endpoint: &StorageEndpoint,
|
||||||
|
key: &str,
|
||||||
|
path: &Path,
|
||||||
|
content_type: &str,
|
||||||
|
size: u64,
|
||||||
|
) -> Result<StoredObject, AppError> {
|
||||||
|
let client = client_for(state, endpoint, EndpointKind::Internal)?;
|
||||||
|
let etag = if size >= MULTIPART_THRESHOLD {
|
||||||
|
multipart_upload(&client, &endpoint.bucket, key, path, content_type).await?
|
||||||
|
} else {
|
||||||
|
let body = ByteStream::from_path(path).await.map_err(|err| {
|
||||||
|
AppError::new(ErrorCode::StorageUnavailable, "读取待上传文件失败").with_source(err)
|
||||||
|
})?;
|
||||||
|
client
|
||||||
|
.put_object()
|
||||||
|
.bucket(&endpoint.bucket)
|
||||||
|
.key(key)
|
||||||
|
.content_type(content_type)
|
||||||
|
.body(body)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|err| storage_error("上传 S3 对象失败", err))?
|
||||||
|
.e_tag()
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(StoredObject {
|
||||||
|
backend: "s3".to_string(),
|
||||||
|
endpoint_id: Some(endpoint.id),
|
||||||
|
key: key.to_string(),
|
||||||
|
etag,
|
||||||
|
size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn store_file_local(
|
||||||
|
state: &AppState,
|
||||||
|
key: &str,
|
||||||
|
path: &Path,
|
||||||
|
size: u64,
|
||||||
|
) -> Result<StoredObject, AppError> {
|
||||||
let destination = local_path(state, key)?;
|
let destination = local_path(state, key)?;
|
||||||
if let Some(parent) = destination.parent() {
|
if let Some(parent) = destination.parent() {
|
||||||
tokio::fs::create_dir_all(parent).await.map_err(|err| {
|
tokio::fs::create_dir_all(parent).await.map_err(|err| {
|
||||||
@@ -247,10 +294,21 @@ pub async fn store_file(
|
|||||||
endpoint_id: None,
|
endpoint_id: None,
|
||||||
key: destination.to_string_lossy().to_string(),
|
key: destination.to_string_lossy().to_string(),
|
||||||
etag: None,
|
etag: None,
|
||||||
size: metadata.len(),
|
size,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn log_local_fallback(endpoint: &StorageEndpoint, key: &str, err: &AppError) {
|
||||||
|
tracing::warn!(
|
||||||
|
storage_endpoint_id = %endpoint.id,
|
||||||
|
storage_endpoint = %endpoint.name,
|
||||||
|
object_key = %key,
|
||||||
|
error = %err,
|
||||||
|
fallback_backend = "local",
|
||||||
|
"S3 write failed; storing object on local disk"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn read_bytes(state: &AppState, object: &ObjectLocator) -> Result<Vec<u8>, AppError> {
|
pub async fn read_bytes(state: &AppState, object: &ObjectLocator) -> Result<Vec<u8>, AppError> {
|
||||||
if object.backend == "local" {
|
if object.backend == "local" {
|
||||||
return tokio::fs::read(&object.key).await.map_err(|err| {
|
return tokio::fs::read(&object.key).await.map_err(|err| {
|
||||||
@@ -509,6 +567,12 @@ fn build_client(
|
|||||||
.region(Region::new(region.to_string()))
|
.region(Region::new(region.to_string()))
|
||||||
.endpoint_url(endpoint_url)
|
.endpoint_url(endpoint_url)
|
||||||
.force_path_style(force_path_style)
|
.force_path_style(force_path_style)
|
||||||
|
.retry_config(RetryConfig::standard().with_max_attempts(2))
|
||||||
|
.timeout_config(
|
||||||
|
TimeoutConfig::builder()
|
||||||
|
.connect_timeout(Duration::from_secs(3))
|
||||||
|
.build(),
|
||||||
|
)
|
||||||
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
|
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
|
||||||
.response_checksum_validation(ResponseChecksumValidation::WhenRequired)
|
.response_checksum_validation(ResponseChecksumValidation::WhenRequired)
|
||||||
.build();
|
.build();
|
||||||
|
|||||||
Reference in New Issue
Block a user