feat: add local fallback for S3 writes

This commit is contained in:
237899745
2026-07-25 16:29:05 +08:00
parent 63744c6d2f
commit 0fe3d4ce8e
11 changed files with 169 additions and 58 deletions

1
Cargo.lock generated
View File

@@ -2169,6 +2169,7 @@ dependencies = [
"axum",
"axum-extra",
"base64 0.22.1",
"bytes",
"chrono",
"dotenvy",
"hex",

View File

@@ -16,6 +16,7 @@ time = "0.3"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
thiserror = "1"
bytes = "1"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }

View File

@@ -731,6 +731,8 @@ Authorization: Bearer <admin_token>
凭据加密保存且不通过 API 回传。测试接口执行 Bucket 检查、内部临时对象读写删和公网预签名下载;激活接口会再次测试并原子切换活动端点。活动端点不能直接编辑或删除。
列表响应中的 `local_object_count``local_stored_bytes` 统计当前实际位于应用服务器的成品。存在活动 S3 时,新对象仍优先写入 S3若本次 S3 写入失败,则自动回退本地并按实际后端完成下载和到期清理。
### 11.7 兑换码管理
```http

View File

@@ -39,6 +39,7 @@ flowchart LR
4. 批处理原图只在 118 的共享 `uploads/orig` 临时保存Worker 完成或失败后删除,不上传 S3。
5. 批量 ZIP 在 118 临时生成64 MiB 以上自动使用 S3 分片上传,上传后删除临时目录。
6. 每个成品记录写入时的端点 ID。以后切换端点不会使旧文件失联旧端点凭据会保留到关联对象清空。
7. 活动 S3 仍是首选后端;连接、凭据或上传失败时,本次成品自动写入 118 本地卷并记录 `storage_backend=local`。S3 恢复后,后续新对象会再次优先写入 S3。
`/api/v1/compress/direct` 保留原有“响应体直接返回图片”的 API 语义,避免破坏现有调用方;网站下载、历史记录下载、普通压缩结果和批量 ZIP 均走 S3。
@@ -126,6 +127,7 @@ aws --endpoint-url http://127.0.0.1:3900 \
- 未授权用户无法取得签名 URL任务过期后应用下载接口返回 404。
- 单文件与批任务数据库均记录正确的 `storage_endpoint_id` 和对象键。
- 切换到第二端点后,新对象进入第二端点,第一端点历史对象仍可下载。
- S3 停止时同步请求返回 503批任务保留临时原图并重试不静默写回本地
- S3 停止时同步请求和批任务均完成本地兜底,数据库记录 `storage_backend=local`,下载接口直接从应用服务器返回文件
- S3 恢复后下一次新对象重新记录为 `storage_backend=s3`,下载接口恢复 `307` 到签名 URL。
- 删除 S3 对象失败时,过期任务数据库记录保留并在下一轮重试。
- 119 磁盘 70%/85% 告警、容器重启、NTP 时间同步和证书续期均验证通过。

View File

@@ -8,6 +8,8 @@ const auth = useAuthStore()
const router = useRouter()
const isLoggedIn = computed(() => auth.isLoggedIn)
const consoleRoute = computed(() => (auth.user?.role === 'admin' ? '/admin' : '/dashboard'))
const consoleLabel = computed(() => (auth.user?.role === 'admin' ? '管理后台' : '控制台'))
function logout() {
auth.logout()
@@ -35,10 +37,10 @@ function logout() {
<div class="flex items-center gap-3">
<template v-if="isLoggedIn">
<RouterLink
to="/dashboard"
:to="consoleRoute"
class="rounded-md border border-slate-200 bg-white px-3 py-1.5 text-sm text-slate-700 hover:bg-slate-50"
>
控制台
{{ consoleLabel }}
</RouterLink>
<button
type="button"

View File

@@ -81,7 +81,7 @@ export function createAppRouter(pinia: Pinia) {
}
if ((to.name === 'login' || to.name === 'register') && auth.isLoggedIn) {
return { name: 'dashboard' }
return { name: auth.user?.role === 'admin' ? 'admin' : 'dashboard' }
}
if (to.meta?.requiresAdmin && auth.user?.role !== 'admin') {

View File

@@ -22,8 +22,10 @@ async function submit() {
const resp = await login(email.value.trim(), password.value)
auth.setAuth(resp.token, resp.user)
const redirect = typeof route.query.redirect === 'string' ? route.query.redirect : '/dashboard'
await router.push(redirect)
const defaultRoute = resp.user.role === 'admin' ? '/admin' : '/dashboard'
const requestedRoute = typeof route.query.redirect === 'string' ? route.query.redirect : null
const redirect = requestedRoute?.startsWith('/') && !requestedRoute.startsWith('//') ? requestedRoute : defaultRoute
await router.replace(redirect)
} catch (err) {
if (err instanceof ApiError) {
error.value = `[${err.code}] ${err.message}`

View File

@@ -20,6 +20,8 @@ const loading = ref(true)
const error = ref<string | null>(null)
const message = ref<string | null>(null)
const activeBackend = ref<'local' | 's3'>('local')
const localObjectCount = ref(0)
const localStoredBytes = ref(0)
const endpoints = ref<AdminStorageEndpoint[]>([])
const editingId = ref<string | null>(null)
const formOpen = ref(false)
@@ -59,6 +61,8 @@ async function loadEndpoints(clearError = true) {
const response = await listStorageEndpoints(auth.token)
endpoints.value = response.endpoints
activeBackend.value = response.active_backend
localObjectCount.value = response.local_object_count
localStoredBytes.value = response.local_stored_bytes
} catch (err) {
error.value = errorText(err, '加载存储配置失败')
} finally {
@@ -207,7 +211,7 @@ onMounted(loadEndpoints)
<p class="text-xs font-semibold uppercase tracking-[0.22em] text-cyan-300">Storage control plane</p>
<h2 class="mt-3 text-2xl font-semibold">对象存储</h2>
<p class="mt-2 max-w-2xl text-sm leading-6 text-slate-300">
应用完成鉴权后返回短期签名地址图片与批量 ZIP 的下载流量直接由高带宽 S3 节点承担每个对象固定绑定写入端点后续切换或扩容不会影响旧文件
新对象优先写入活动 S3连接或写入失败时自动保存到应用服务器本地每个对象固定记录实际写入后端恢复 S3 后的新文件会自动重新走对象存储
</p>
</div>
<div class="rounded-xl border border-white/10 bg-white/5 p-4">
@@ -217,12 +221,16 @@ onMounted(loadEndpoints)
class="rounded-full px-3 py-1 text-xs font-semibold"
:class="activeBackend === 's3' ? 'bg-emerald-400/15 text-emerald-300' : 'bg-amber-400/15 text-amber-200'"
>
{{ activeBackend === 's3' ? 'S3 已启用' : '本地回退' }}
{{ activeBackend === 's3' ? 'S3 优先' : '本地' }}
</span>
</div>
<p class="mt-3 text-xs leading-5 text-slate-400">
推荐内部 Endpoint 走两台服务器之间的 WireGuard 地址公网 Endpoint 使用带 TLS 的下载域名签名默认 5 分钟有效
S3 故障只影响当次写入下载与到期清理会按对象记录的真实后端执行签名默认 5 分钟有效
</p>
<div class="mt-3 flex items-center justify-between border-t border-white/10 pt-3 text-xs">
<span class="text-slate-400">本地对象</span>
<span class="font-medium text-slate-200">{{ localObjectCount }} · {{ formatBytes(localStoredBytes) }}</span>
</div>
</div>
</div>
</section>
@@ -238,7 +246,7 @@ onMounted(loadEndpoints)
<div class="flex flex-wrap items-center justify-between gap-3">
<div>
<h3 class="font-semibold text-slate-900">存储端点</h3>
<p class="mt-1 text-sm text-slate-500">同一时间只有一个端点接收新对象停用端点继续服务其历史对象</p>
<p class="mt-1 text-sm text-slate-500">同一时间只有一个首选 S3 端点写入失败自动回退本地停用端点继续服务其历史对象</p>
</div>
<button
class="rounded-lg bg-slate-900 px-4 py-2 text-sm font-medium text-white hover:bg-slate-800"

View File

@@ -669,6 +669,8 @@ export interface AdminStorageEndpoint {
export interface AdminStorageEndpointsResponse {
active_backend: 'local' | 's3'
local_object_count: number
local_stored_bytes: number
endpoints: AdminStorageEndpoint[]
}

View File

@@ -60,6 +60,8 @@ struct StorageEndpointView {
#[derive(Debug, Serialize)]
struct StorageEndpointsResponse {
active_backend: String,
local_object_count: i64,
local_stored_bytes: i64,
endpoints: Vec<StorageEndpointView>,
}
@@ -109,6 +111,7 @@ async fn list_storage_endpoints(
} else {
"local"
};
let (local_object_count, local_stored_bytes) = local_usage(&state).await?;
let mut views = Vec::with_capacity(endpoints.len());
for endpoint in endpoints {
views.push(endpoint_view(&state, endpoint).await?);
@@ -118,6 +121,8 @@ async fn list_storage_endpoints(
success: true,
data: StorageEndpointsResponse {
active_backend: active_backend.to_string(),
local_object_count,
local_stored_bytes,
endpoints: views,
},
}))
@@ -536,6 +541,28 @@ async fn endpoint_usage(state: &AppState, endpoint_id: Uuid) -> Result<(i64, i64
.map_err(|err| AppError::new(ErrorCode::Internal, "统计存储使用量失败").with_source(err))
}
async fn local_usage(state: &AppState) -> Result<(i64, i64), AppError> {
sqlx::query_as::<_, (i64, i64)>(
r#"
SELECT
(SELECT COUNT(*) FROM task_files
WHERE status = 'completed'
AND storage_backend = 'local'
AND COALESCE(storage_key, storage_path) IS NOT NULL)
+ (SELECT COUNT(*) FROM tasks
WHERE zip_storage_backend = 'local'
AND zip_storage_key IS NOT NULL) AS object_count,
COALESCE((SELECT SUM(compressed_size)::BIGINT FROM task_files
WHERE status = 'completed' AND storage_backend = 'local'), 0::BIGINT)
+ COALESCE((SELECT SUM(zip_size)::BIGINT FROM tasks
WHERE zip_storage_backend = 'local'), 0::BIGINT) AS stored_bytes
"#,
)
.fetch_one(&state.db)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "统计本地存储使用量失败").with_source(err))
}
async fn record_test_result(
state: &AppState,
endpoint_id: Uuid,

View File

@@ -3,12 +3,14 @@ use crate::services::settings;
use crate::state::AppState;
use aws_sdk_s3::config::{
BehaviorVersion, Credentials, Region, RequestChecksumCalculation, ResponseChecksumValidation,
retry::RetryConfig, timeout::TimeoutConfig, BehaviorVersion, Credentials, Region,
RequestChecksumCalculation, ResponseChecksumValidation,
};
use aws_sdk_s3::presigning::PresigningConfig;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use aws_sdk_s3::Client;
use bytes::Bytes;
use chrono::{DateTime, Datelike, Utc};
use sqlx::FromRow;
use std::path::{Path, PathBuf};
@@ -151,8 +153,25 @@ pub async fn store_bytes(
bytes: Vec<u8>,
content_type: &str,
) -> Result<StoredObject, AppError> {
let bytes = Bytes::from(bytes);
if let Some(endpoint) = active_endpoint(state).await? {
let client = client_for(state, &endpoint, EndpointKind::Internal)?;
match store_bytes_s3(state, &endpoint, key, bytes.clone(), content_type).await {
Ok(stored) => return Ok(stored),
Err(err) => log_local_fallback(&endpoint, key, &err),
}
}
store_bytes_local(state, key, bytes.as_ref()).await
}
async fn store_bytes_s3(
state: &AppState,
endpoint: &StorageEndpoint,
key: &str,
bytes: Bytes,
content_type: &str,
) -> Result<StoredObject, AppError> {
let client = client_for(state, endpoint, EndpointKind::Internal)?;
let size = bytes.len() as u64;
let output = client
.put_object()
@@ -164,15 +183,20 @@ pub async fn store_bytes(
.await
.map_err(|err| storage_error("上传 S3 对象失败", err))?;
return Ok(StoredObject {
Ok(StoredObject {
backend: "s3".to_string(),
endpoint_id: Some(endpoint.id),
key: key.to_string(),
etag: output.e_tag().map(ToOwned::to_owned),
size,
});
})
}
async fn store_bytes_local(
state: &AppState,
key: &str,
bytes: &[u8],
) -> Result<StoredObject, AppError> {
let path = local_path(state, key)?;
if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent).await.map_err(|err| {
@@ -203,8 +227,25 @@ pub async fn store_file(
})?;
if let Some(endpoint) = active_endpoint(state).await? {
let client = client_for(state, &endpoint, EndpointKind::Internal)?;
let etag = if metadata.len() >= MULTIPART_THRESHOLD {
match store_file_s3(state, &endpoint, key, path, content_type, metadata.len()).await {
Ok(stored) => return Ok(stored),
Err(err) => log_local_fallback(&endpoint, key, &err),
}
}
store_file_local(state, key, path, metadata.len()).await
}
async fn store_file_s3(
state: &AppState,
endpoint: &StorageEndpoint,
key: &str,
path: &Path,
content_type: &str,
size: u64,
) -> Result<StoredObject, AppError> {
let client = client_for(state, endpoint, EndpointKind::Internal)?;
let etag = if size >= MULTIPART_THRESHOLD {
multipart_upload(&client, &endpoint.bucket, key, path, content_type).await?
} else {
let body = ByteStream::from_path(path).await.map_err(|err| {
@@ -223,15 +264,21 @@ pub async fn store_file(
.map(ToOwned::to_owned)
};
return Ok(StoredObject {
Ok(StoredObject {
backend: "s3".to_string(),
endpoint_id: Some(endpoint.id),
key: key.to_string(),
etag,
size: metadata.len(),
});
size,
})
}
async fn store_file_local(
state: &AppState,
key: &str,
path: &Path,
size: u64,
) -> Result<StoredObject, AppError> {
let destination = local_path(state, key)?;
if let Some(parent) = destination.parent() {
tokio::fs::create_dir_all(parent).await.map_err(|err| {
@@ -247,10 +294,21 @@ pub async fn store_file(
endpoint_id: None,
key: destination.to_string_lossy().to_string(),
etag: None,
size: metadata.len(),
size,
})
}
fn log_local_fallback(endpoint: &StorageEndpoint, key: &str, err: &AppError) {
tracing::warn!(
storage_endpoint_id = %endpoint.id,
storage_endpoint = %endpoint.name,
object_key = %key,
error = %err,
fallback_backend = "local",
"S3 write failed; storing object on local disk"
);
}
pub async fn read_bytes(state: &AppState, object: &ObjectLocator) -> Result<Vec<u8>, AppError> {
if object.backend == "local" {
return tokio::fs::read(&object.key).await.map_err(|err| {
@@ -509,6 +567,12 @@ fn build_client(
.region(Region::new(region.to_string()))
.endpoint_url(endpoint_url)
.force_path_style(force_path_style)
.retry_config(RetryConfig::standard().with_max_attempts(2))
.timeout_config(
TimeoutConfig::builder()
.connect_timeout(Duration::from_secs(3))
.build(),
)
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
.response_checksum_validation(ResponseChecksumValidation::WhenRequired)
.build();