feat: add configurable S3 object storage
This commit is contained in:
237899745
@@ -40,7 +40,7 @@ pub fn router() -> Router<AppState> {
|
||||
.route("/admin/config", put(update_config))
|
||||
}
|
||||
|
||||
async fn require_admin(
|
||||
pub(super) async fn require_admin(
|
||||
state: &AppState,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
headers: &HeaderMap,
|
||||
|
||||
761
src/api/admin_storage.rs
Normal file
761
src/api/admin_storage.rs
Normal file
@@ -0,0 +1,761 @@
|
||||
use crate::api::admin::require_admin;
|
||||
use crate::api::context;
|
||||
use crate::api::envelope::Envelope;
|
||||
use crate::error::{AppError, ErrorCode};
|
||||
use crate::services::{settings, storage};
|
||||
use crate::state::AppState;
|
||||
|
||||
use axum::extract::{ConnectInfo, Path, State};
|
||||
use axum::http::HeaderMap;
|
||||
use axum::routing::{get, post, put};
|
||||
use axum::{Json, Router};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use url::Url;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub fn router() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route(
|
||||
"/admin/storage/endpoints",
|
||||
get(list_storage_endpoints).post(create_storage_endpoint),
|
||||
)
|
||||
.route(
|
||||
"/admin/storage/endpoints/{endpoint_id}",
|
||||
put(update_storage_endpoint).delete(delete_storage_endpoint),
|
||||
)
|
||||
.route(
|
||||
"/admin/storage/endpoints/{endpoint_id}/test",
|
||||
post(test_storage_endpoint),
|
||||
)
|
||||
.route(
|
||||
"/admin/storage/endpoints/{endpoint_id}/activate",
|
||||
post(activate_storage_endpoint),
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct StorageEndpointView {
|
||||
id: Uuid,
|
||||
name: String,
|
||||
internal_endpoint: String,
|
||||
public_endpoint: String,
|
||||
bucket: String,
|
||||
region: String,
|
||||
access_key_hint: String,
|
||||
credentials_configured: bool,
|
||||
force_path_style: bool,
|
||||
presign_ttl_seconds: i32,
|
||||
is_active: bool,
|
||||
last_test_at: Option<DateTime<Utc>>,
|
||||
last_test_ok: Option<bool>,
|
||||
last_test_error: Option<String>,
|
||||
object_count: i64,
|
||||
stored_bytes: i64,
|
||||
created_at: DateTime<Utc>,
|
||||
updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct StorageEndpointsResponse {
|
||||
active_backend: String,
|
||||
endpoints: Vec<StorageEndpointView>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CreateStorageEndpointRequest {
|
||||
name: String,
|
||||
internal_endpoint: String,
|
||||
public_endpoint: String,
|
||||
bucket: String,
|
||||
region: Option<String>,
|
||||
access_key: String,
|
||||
secret_key: String,
|
||||
force_path_style: Option<bool>,
|
||||
presign_ttl_seconds: Option<i32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct UpdateStorageEndpointRequest {
|
||||
name: Option<String>,
|
||||
internal_endpoint: Option<String>,
|
||||
public_endpoint: Option<String>,
|
||||
bucket: Option<String>,
|
||||
region: Option<String>,
|
||||
access_key: Option<String>,
|
||||
secret_key: Option<String>,
|
||||
force_path_style: Option<bool>,
|
||||
presign_ttl_seconds: Option<i32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct StorageActionResponse {
|
||||
message: String,
|
||||
endpoint: StorageEndpointView,
|
||||
}
|
||||
|
||||
async fn list_storage_endpoints(
|
||||
State(state): State<AppState>,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
ConnectInfo(addr): ConnectInfo<SocketAddr>,
|
||||
headers: HeaderMap,
|
||||
) -> Result<Json<Envelope<StorageEndpointsResponse>>, AppError> {
|
||||
let ip = context::client_ip(&headers, addr.ip());
|
||||
let (_jar, _admin_id) = require_admin(&state, jar, &headers, ip).await?;
|
||||
let endpoints = storage::list_endpoints(&state).await?;
|
||||
let active_backend = if endpoints.iter().any(|endpoint| endpoint.is_active) {
|
||||
"s3"
|
||||
} else {
|
||||
"local"
|
||||
};
|
||||
let mut views = Vec::with_capacity(endpoints.len());
|
||||
for endpoint in endpoints {
|
||||
views.push(endpoint_view(&state, endpoint).await?);
|
||||
}
|
||||
|
||||
Ok(Json(Envelope {
|
||||
success: true,
|
||||
data: StorageEndpointsResponse {
|
||||
active_backend: active_backend.to_string(),
|
||||
endpoints: views,
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
async fn create_storage_endpoint(
|
||||
State(state): State<AppState>,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
ConnectInfo(addr): ConnectInfo<SocketAddr>,
|
||||
headers: HeaderMap,
|
||||
Json(req): Json<CreateStorageEndpointRequest>,
|
||||
) -> Result<Json<Envelope<StorageEndpointView>>, AppError> {
|
||||
let ip = context::client_ip(&headers, addr.ip());
|
||||
let (_jar, admin_id) = require_admin(&state, jar, &headers, ip).await?;
|
||||
|
||||
let name = validate_name(&req.name)?;
|
||||
ensure_name_available(&state, &name, None).await?;
|
||||
let internal_endpoint = validate_endpoint_url(&req.internal_endpoint, "内部 Endpoint")?;
|
||||
let public_endpoint = validate_endpoint_url(&req.public_endpoint, "公网 Endpoint")?;
|
||||
let bucket = validate_bucket(&req.bucket)?;
|
||||
let region = validate_region(req.region.as_deref().unwrap_or("garage"))?;
|
||||
let access_key = validate_credential(&req.access_key, "Access Key")?;
|
||||
let secret_key = validate_credential(&req.secret_key, "Secret Key")?;
|
||||
let presign_ttl_seconds = validate_presign_ttl(req.presign_ttl_seconds.unwrap_or(300))?;
|
||||
|
||||
let endpoint = sqlx::query_as::<_, storage::StorageEndpoint>(
|
||||
r#"
|
||||
INSERT INTO storage_endpoints (
|
||||
name, internal_endpoint, public_endpoint, bucket, region,
|
||||
access_key_encrypted, secret_key_encrypted, access_key_hint,
|
||||
force_path_style, presign_ttl_seconds, updated_by
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||
RETURNING id, name, internal_endpoint, public_endpoint, bucket, region,
|
||||
access_key_encrypted, secret_key_encrypted, access_key_hint,
|
||||
force_path_style, presign_ttl_seconds, is_active,
|
||||
last_test_at, last_test_ok, last_test_error,
|
||||
created_at, updated_at, deleted_at
|
||||
"#,
|
||||
)
|
||||
.bind(&name)
|
||||
.bind(&internal_endpoint)
|
||||
.bind(&public_endpoint)
|
||||
.bind(&bucket)
|
||||
.bind(®ion)
|
||||
.bind(settings::encrypt_secret(&state, &access_key)?)
|
||||
.bind(settings::encrypt_secret(&state, &secret_key)?)
|
||||
.bind(access_key_hint(&access_key))
|
||||
.bind(req.force_path_style.unwrap_or(true))
|
||||
.bind(presign_ttl_seconds)
|
||||
.bind(admin_id)
|
||||
.fetch_one(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "创建存储端点失败").with_source(err))?;
|
||||
|
||||
audit_storage_action(
|
||||
&state,
|
||||
admin_id,
|
||||
ip,
|
||||
"storage_endpoint.create",
|
||||
endpoint.id,
|
||||
serde_json::json!({ "name": endpoint.name }),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(Json(Envelope {
|
||||
success: true,
|
||||
data: endpoint_view(&state, endpoint).await?,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn update_storage_endpoint(
|
||||
State(state): State<AppState>,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
ConnectInfo(addr): ConnectInfo<SocketAddr>,
|
||||
headers: HeaderMap,
|
||||
Path(endpoint_id): Path<Uuid>,
|
||||
Json(req): Json<UpdateStorageEndpointRequest>,
|
||||
) -> Result<Json<Envelope<StorageEndpointView>>, AppError> {
|
||||
let ip = context::client_ip(&headers, addr.ip());
|
||||
let (_jar, admin_id) = require_admin(&state, jar, &headers, ip).await?;
|
||||
let existing = storage::get_endpoint(&state, endpoint_id).await?;
|
||||
ensure_configurable(&existing)?;
|
||||
if existing.is_active {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"活动端点不能直接编辑，请先新增并启用替代端点",
|
||||
));
|
||||
}
|
||||
let (object_count, _) = endpoint_usage(&state, endpoint_id).await?;
|
||||
if object_count > 0 {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"端点仍有关联对象，不能修改；请新增替代端点并等待旧对象过期",
|
||||
));
|
||||
}
|
||||
|
||||
let name = match req.name.as_deref() {
|
||||
Some(value) => validate_name(value)?,
|
||||
None => existing.name.clone(),
|
||||
};
|
||||
ensure_name_available(&state, &name, Some(endpoint_id)).await?;
|
||||
let internal_endpoint = match req.internal_endpoint.as_deref() {
|
||||
Some(value) => validate_endpoint_url(value, "内部 Endpoint")?,
|
||||
None => existing.internal_endpoint.clone(),
|
||||
};
|
||||
let public_endpoint = match req.public_endpoint.as_deref() {
|
||||
Some(value) => validate_endpoint_url(value, "公网 Endpoint")?,
|
||||
None => existing.public_endpoint.clone(),
|
||||
};
|
||||
let bucket = match req.bucket.as_deref() {
|
||||
Some(value) => validate_bucket(value)?,
|
||||
None => existing.bucket.clone(),
|
||||
};
|
||||
let region = match req.region.as_deref() {
|
||||
Some(value) => validate_region(value)?,
|
||||
None => existing.region.clone(),
|
||||
};
|
||||
let access_key_encrypted = match req.access_key.as_deref() {
|
||||
Some(value) if !value.trim().is_empty() => Some(settings::encrypt_secret(
|
||||
&state,
|
||||
&validate_credential(value, "Access Key")?,
|
||||
)?),
|
||||
_ => None,
|
||||
};
|
||||
let secret_key_encrypted = match req.secret_key.as_deref() {
|
||||
Some(value) if !value.trim().is_empty() => Some(settings::encrypt_secret(
|
||||
&state,
|
||||
&validate_credential(value, "Secret Key")?,
|
||||
)?),
|
||||
_ => None,
|
||||
};
|
||||
let access_key_hint_value = req
|
||||
.access_key
|
||||
.as_deref()
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.map(access_key_hint);
|
||||
let presign_ttl_seconds = validate_presign_ttl(
|
||||
req.presign_ttl_seconds
|
||||
.unwrap_or(existing.presign_ttl_seconds),
|
||||
)?;
|
||||
|
||||
let endpoint = sqlx::query_as::<_, storage::StorageEndpoint>(
|
||||
r#"
|
||||
UPDATE storage_endpoints
|
||||
SET name = $2,
|
||||
internal_endpoint = $3,
|
||||
public_endpoint = $4,
|
||||
bucket = $5,
|
||||
region = $6,
|
||||
access_key_encrypted = COALESCE($7, access_key_encrypted),
|
||||
secret_key_encrypted = COALESCE($8, secret_key_encrypted),
|
||||
access_key_hint = COALESCE($9, access_key_hint),
|
||||
force_path_style = $10,
|
||||
presign_ttl_seconds = $11,
|
||||
is_active = false,
|
||||
last_test_at = NULL,
|
||||
last_test_ok = NULL,
|
||||
last_test_error = NULL,
|
||||
updated_at = NOW(),
|
||||
updated_by = $12
|
||||
WHERE id = $1
|
||||
RETURNING id, name, internal_endpoint, public_endpoint, bucket, region,
|
||||
access_key_encrypted, secret_key_encrypted, access_key_hint,
|
||||
force_path_style, presign_ttl_seconds, is_active,
|
||||
last_test_at, last_test_ok, last_test_error,
|
||||
created_at, updated_at, deleted_at
|
||||
"#,
|
||||
)
|
||||
.bind(endpoint_id)
|
||||
.bind(&name)
|
||||
.bind(&internal_endpoint)
|
||||
.bind(&public_endpoint)
|
||||
.bind(&bucket)
|
||||
.bind(®ion)
|
||||
.bind(access_key_encrypted)
|
||||
.bind(secret_key_encrypted)
|
||||
.bind(access_key_hint_value)
|
||||
.bind(req.force_path_style.unwrap_or(existing.force_path_style))
|
||||
.bind(presign_ttl_seconds)
|
||||
.bind(admin_id)
|
||||
.fetch_one(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "更新存储端点失败").with_source(err))?;
|
||||
|
||||
audit_storage_action(
|
||||
&state,
|
||||
admin_id,
|
||||
ip,
|
||||
"storage_endpoint.update",
|
||||
endpoint.id,
|
||||
serde_json::json!({ "name": endpoint.name, "requires_reactivation": true }),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(Json(Envelope {
|
||||
success: true,
|
||||
data: endpoint_view(&state, endpoint).await?,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn test_storage_endpoint(
|
||||
State(state): State<AppState>,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
ConnectInfo(addr): ConnectInfo<SocketAddr>,
|
||||
headers: HeaderMap,
|
||||
Path(endpoint_id): Path<Uuid>,
|
||||
) -> Result<Json<Envelope<StorageActionResponse>>, AppError> {
|
||||
let ip = context::client_ip(&headers, addr.ip());
|
||||
let (_jar, admin_id) = require_admin(&state, jar, &headers, ip).await?;
|
||||
let endpoint = storage::get_endpoint(&state, endpoint_id).await?;
|
||||
ensure_configurable(&endpoint)?;
|
||||
if let Err(err) = storage::test_endpoint(&state, &endpoint).await {
|
||||
let detail = storage_test_message(&err);
|
||||
record_test_result(&state, endpoint_id, false, Some(&detail), admin_id).await?;
|
||||
audit_storage_action(
|
||||
&state,
|
||||
admin_id,
|
||||
ip,
|
||||
"storage_endpoint.test_failed",
|
||||
endpoint_id,
|
||||
serde_json::json!({ "message": detail }),
|
||||
)
|
||||
.await;
|
||||
return Err(AppError::new(ErrorCode::StorageUnavailable, detail));
|
||||
}
|
||||
|
||||
record_test_result(&state, endpoint_id, true, None, admin_id).await?;
|
||||
let endpoint = storage::get_endpoint(&state, endpoint_id).await?;
|
||||
ensure_configurable(&endpoint)?;
|
||||
audit_storage_action(
|
||||
&state,
|
||||
admin_id,
|
||||
ip,
|
||||
"storage_endpoint.test_succeeded",
|
||||
endpoint_id,
|
||||
serde_json::json!({}),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(Json(Envelope {
|
||||
success: true,
|
||||
data: StorageActionResponse {
|
||||
message: "S3 内部读写删与公网签名下载测试通过".to_string(),
|
||||
endpoint: endpoint_view(&state, endpoint).await?,
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
async fn activate_storage_endpoint(
|
||||
State(state): State<AppState>,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
ConnectInfo(addr): ConnectInfo<SocketAddr>,
|
||||
headers: HeaderMap,
|
||||
Path(endpoint_id): Path<Uuid>,
|
||||
) -> Result<Json<Envelope<StorageActionResponse>>, AppError> {
|
||||
let ip = context::client_ip(&headers, addr.ip());
|
||||
let (_jar, admin_id) = require_admin(&state, jar, &headers, ip).await?;
|
||||
let endpoint = storage::get_endpoint(&state, endpoint_id).await?;
|
||||
ensure_configurable(&endpoint)?;
|
||||
let tested_config_updated_at = endpoint.updated_at;
|
||||
|
||||
if let Err(err) = storage::test_endpoint(&state, &endpoint).await {
|
||||
let detail = storage_test_message(&err);
|
||||
record_test_result(&state, endpoint_id, false, Some(&detail), admin_id).await?;
|
||||
return Err(AppError::new(ErrorCode::StorageUnavailable, detail));
|
||||
}
|
||||
|
||||
let mut tx = state.db.begin().await.map_err(|err| {
|
||||
AppError::new(ErrorCode::Internal, "开启存储切换事务失败").with_source(err)
|
||||
})?;
|
||||
sqlx::query(
|
||||
"UPDATE storage_endpoints SET is_active = false, updated_at = NOW() WHERE is_active = true",
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "停用旧存储端点失败").with_source(err))?;
|
||||
let activated = sqlx::query(
|
||||
r#"
|
||||
UPDATE storage_endpoints
|
||||
SET is_active = true,
|
||||
last_test_at = NOW(),
|
||||
last_test_ok = true,
|
||||
last_test_error = NULL,
|
||||
updated_at = NOW(),
|
||||
updated_by = $2
|
||||
WHERE id = $1 AND updated_at = $3 AND deleted_at IS NULL
|
||||
"#,
|
||||
)
|
||||
.bind(endpoint_id)
|
||||
.bind(admin_id)
|
||||
.bind(tested_config_updated_at)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "启用存储端点失败").with_source(err))?;
|
||||
if activated.rows_affected() == 0 {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"端点配置在测试期间发生变化，请重新测试",
|
||||
));
|
||||
}
|
||||
tx.commit().await.map_err(|err| {
|
||||
AppError::new(ErrorCode::Internal, "提交存储切换事务失败").with_source(err)
|
||||
})?;
|
||||
|
||||
let endpoint = storage::get_endpoint(&state, endpoint_id).await?;
|
||||
audit_storage_action(
|
||||
&state,
|
||||
admin_id,
|
||||
ip,
|
||||
"storage_endpoint.activate",
|
||||
endpoint_id,
|
||||
serde_json::json!({ "name": endpoint.name }),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(Json(Envelope {
|
||||
success: true,
|
||||
data: StorageActionResponse {
|
||||
message: "S3 端点已启用，新生成文件将写入该端点".to_string(),
|
||||
endpoint: endpoint_view(&state, endpoint).await?,
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
async fn delete_storage_endpoint(
|
||||
State(state): State<AppState>,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
ConnectInfo(addr): ConnectInfo<SocketAddr>,
|
||||
headers: HeaderMap,
|
||||
Path(endpoint_id): Path<Uuid>,
|
||||
) -> Result<Json<Envelope<serde_json::Value>>, AppError> {
|
||||
let ip = context::client_ip(&headers, addr.ip());
|
||||
let (_jar, admin_id) = require_admin(&state, jar, &headers, ip).await?;
|
||||
let endpoint = storage::get_endpoint(&state, endpoint_id).await?;
|
||||
ensure_configurable(&endpoint)?;
|
||||
if endpoint.is_active {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"活动端点不能删除，请先启用其他端点",
|
||||
));
|
||||
}
|
||||
|
||||
let (object_count, _): (i64, i64) = endpoint_usage(&state, endpoint_id).await?;
|
||||
if object_count > 0 {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
format!("端点仍关联 {object_count} 个对象，需等待对象过期清理后再删除"),
|
||||
));
|
||||
}
|
||||
|
||||
sqlx::query(
|
||||
"UPDATE storage_endpoints SET is_active = false, deleted_at = NOW(), updated_at = NOW(), updated_by = $2 WHERE id = $1",
|
||||
)
|
||||
.bind(endpoint_id)
|
||||
.bind(admin_id)
|
||||
.execute(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "归档存储端点失败").with_source(err))?;
|
||||
audit_storage_action(
|
||||
&state,
|
||||
admin_id,
|
||||
ip,
|
||||
"storage_endpoint.delete",
|
||||
endpoint_id,
|
||||
serde_json::json!({ "name": endpoint.name }),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(Json(Envelope {
|
||||
success: true,
|
||||
data: serde_json::json!({ "message": "存储端点已删除" }),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn endpoint_view(
|
||||
state: &AppState,
|
||||
endpoint: storage::StorageEndpoint,
|
||||
) -> Result<StorageEndpointView, AppError> {
|
||||
let (object_count, stored_bytes) = endpoint_usage(state, endpoint.id).await?;
|
||||
Ok(StorageEndpointView {
|
||||
id: endpoint.id,
|
||||
name: endpoint.name,
|
||||
internal_endpoint: endpoint.internal_endpoint,
|
||||
public_endpoint: endpoint.public_endpoint,
|
||||
bucket: endpoint.bucket,
|
||||
region: endpoint.region,
|
||||
access_key_hint: endpoint.access_key_hint,
|
||||
credentials_configured: !endpoint.access_key_encrypted.is_empty()
|
||||
&& !endpoint.secret_key_encrypted.is_empty(),
|
||||
force_path_style: endpoint.force_path_style,
|
||||
presign_ttl_seconds: endpoint.presign_ttl_seconds,
|
||||
is_active: endpoint.is_active,
|
||||
last_test_at: endpoint.last_test_at,
|
||||
last_test_ok: endpoint.last_test_ok,
|
||||
last_test_error: endpoint.last_test_error,
|
||||
object_count,
|
||||
stored_bytes,
|
||||
created_at: endpoint.created_at,
|
||||
updated_at: endpoint.updated_at,
|
||||
})
|
||||
}
|
||||
|
||||
async fn endpoint_usage(state: &AppState, endpoint_id: Uuid) -> Result<(i64, i64), AppError> {
|
||||
sqlx::query_as::<_, (i64, i64)>(
|
||||
r#"
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM task_files WHERE storage_endpoint_id = $1)
|
||||
+ (SELECT COUNT(*) FROM tasks WHERE zip_storage_endpoint_id = $1) AS object_count,
|
||||
COALESCE((SELECT SUM(compressed_size)::BIGINT FROM task_files WHERE storage_endpoint_id = $1), 0::BIGINT)
|
||||
+ COALESCE((SELECT SUM(zip_size)::BIGINT FROM tasks WHERE zip_storage_endpoint_id = $1), 0::BIGINT) AS stored_bytes
|
||||
"#,
|
||||
)
|
||||
.bind(endpoint_id)
|
||||
.fetch_one(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "统计存储使用量失败").with_source(err))
|
||||
}
|
||||
|
||||
async fn record_test_result(
|
||||
state: &AppState,
|
||||
endpoint_id: Uuid,
|
||||
ok: bool,
|
||||
error: Option<&str>,
|
||||
admin_id: Uuid,
|
||||
) -> Result<(), AppError> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE storage_endpoints
|
||||
SET last_test_at = NOW(),
|
||||
last_test_ok = $2,
|
||||
last_test_error = $3,
|
||||
updated_at = NOW(),
|
||||
updated_by = $4
|
||||
WHERE id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(endpoint_id)
|
||||
.bind(ok)
|
||||
.bind(error)
|
||||
.bind(admin_id)
|
||||
.execute(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "记录存储测试结果失败").with_source(err))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn ensure_name_available(
|
||||
state: &AppState,
|
||||
name: &str,
|
||||
except_id: Option<Uuid>,
|
||||
) -> Result<(), AppError> {
|
||||
let exists: bool = sqlx::query_scalar(
|
||||
"SELECT EXISTS(SELECT 1 FROM storage_endpoints WHERE deleted_at IS NULL AND lower(name) = lower($1) AND ($2::uuid IS NULL OR id <> $2))",
|
||||
)
|
||||
.bind(name)
|
||||
.bind(except_id)
|
||||
.fetch_one(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "校验存储端点名称失败").with_source(err))?;
|
||||
if exists {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"存储端点名称已存在",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_name(value: &str) -> Result<String, AppError> {
|
||||
let value = value.trim();
|
||||
if value.is_empty() || value.chars().count() > 100 {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"端点名称长度必须为 1-100 个字符",
|
||||
));
|
||||
}
|
||||
Ok(value.to_string())
|
||||
}
|
||||
|
||||
fn ensure_configurable(endpoint: &storage::StorageEndpoint) -> Result<(), AppError> {
|
||||
if endpoint.deleted_at.is_some() {
|
||||
return Err(AppError::new(ErrorCode::NotFound, "存储端点不存在"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_endpoint_url(value: &str, label: &str) -> Result<String, AppError> {
|
||||
let value = value.trim().trim_end_matches('/');
|
||||
let parsed = Url::parse(value).map_err(|err| {
|
||||
AppError::new(ErrorCode::InvalidRequest, format!("{label} 格式错误")).with_source(err)
|
||||
})?;
|
||||
if !matches!(parsed.scheme(), "http" | "https")
|
||||
|| !parsed.username().is_empty()
|
||||
|| parsed.password().is_some()
|
||||
|| parsed.query().is_some()
|
||||
|| parsed.fragment().is_some()
|
||||
|| parsed.host_str().is_none()
|
||||
{
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
format!("{label} 必须是无凭据、无查询参数的 HTTP(S) 地址"),
|
||||
));
|
||||
}
|
||||
Ok(value.to_string())
|
||||
}
|
||||
|
||||
fn validate_bucket(value: &str) -> Result<String, AppError> {
|
||||
let value = value.trim();
|
||||
let valid = (3..=63).contains(&value.len())
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || matches!(ch, b'.' | b'-'))
|
||||
&& value
|
||||
.as_bytes()
|
||||
.first()
|
||||
.is_some_and(u8::is_ascii_alphanumeric)
|
||||
&& value
|
||||
.as_bytes()
|
||||
.last()
|
||||
.is_some_and(u8::is_ascii_alphanumeric)
|
||||
&& !value.contains("..")
|
||||
&& !value.contains(".-")
|
||||
&& !value.contains("-.");
|
||||
if !valid {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"Bucket 名称不符合 S3 命名规则",
|
||||
));
|
||||
}
|
||||
Ok(value.to_string())
|
||||
}
|
||||
|
||||
fn validate_region(value: &str) -> Result<String, AppError> {
|
||||
let value = value.trim();
|
||||
if value.is_empty()
|
||||
|| value.len() > 100
|
||||
|| !value
|
||||
.bytes()
|
||||
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, b'-' | b'_'))
|
||||
{
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"Region 格式不正确",
|
||||
));
|
||||
}
|
||||
Ok(value.to_string())
|
||||
}
|
||||
|
||||
fn validate_credential(value: &str, label: &str) -> Result<String, AppError> {
|
||||
let value = value.trim();
|
||||
if value.len() < 3 || value.len() > 512 || value.chars().any(char::is_control) {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
format!("{label} 长度或格式不正确"),
|
||||
));
|
||||
}
|
||||
Ok(value.to_string())
|
||||
}
|
||||
|
||||
fn validate_presign_ttl(value: i32) -> Result<i32, AppError> {
|
||||
if !(60..=3600).contains(&value) {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"签名有效期必须为 60-3600 秒",
|
||||
));
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
fn access_key_hint(value: &str) -> String {
|
||||
let chars = value.chars().collect::<Vec<_>>();
|
||||
if chars.len() <= 8 {
|
||||
return "********".to_string();
|
||||
}
|
||||
let start = chars.iter().take(4).collect::<String>();
|
||||
let end = chars.iter().rev().take(4).rev().collect::<String>();
|
||||
format!("{start}...{end}")
|
||||
}
|
||||
|
||||
fn storage_test_message(error: &AppError) -> String {
|
||||
let Some(source) = error.source.as_deref() else {
|
||||
return error.message.clone();
|
||||
};
|
||||
let source = source.replace(['\r', '\n'], " ");
|
||||
let mut detail = format!("{}：{}", error.message, source);
|
||||
if detail.len() > 500 {
|
||||
let mut end = 500;
|
||||
while !detail.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
detail.truncate(end);
|
||||
}
|
||||
detail
|
||||
}
|
||||
|
||||
async fn audit_storage_action(
|
||||
state: &AppState,
|
||||
admin_id: Uuid,
|
||||
ip: IpAddr,
|
||||
action: &str,
|
||||
endpoint_id: Uuid,
|
||||
details: serde_json::Value,
|
||||
) {
|
||||
let _ = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO audit_logs (user_id, action, resource_type, resource_id, details, ip_address)
|
||||
VALUES ($1, $2, 'storage_endpoint', $3, $4, $5::inet)
|
||||
"#,
|
||||
)
|
||||
.bind(admin_id)
|
||||
.bind(action)
|
||||
.bind(endpoint_id)
|
||||
.bind(details)
|
||||
.bind(ip.to_string())
|
||||
.execute(&state.db)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn endpoint_url_rejects_embedded_credentials() {
|
||||
assert!(validate_endpoint_url("https://user:pass@example.com", "Endpoint").is_err());
|
||||
assert!(validate_endpoint_url("https://files.example.com/", "Endpoint").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_validation_matches_s3_basics() {
|
||||
assert!(validate_bucket("imageforge-results").is_ok());
|
||||
assert!(validate_bucket("Bad_Bucket").is_err());
|
||||
assert!(validate_bucket("ab").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credentials_are_masked() {
|
||||
assert_eq!(access_key_hint("ABCD12345678WXYZ"), "ABCD...WXYZ");
|
||||
assert_eq!(access_key_hint("short"), "********");
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ use crate::services::compress;
|
||||
use crate::services::compress::{CompressionLevel, ImageFmt};
|
||||
use crate::services::idempotency;
|
||||
use crate::services::quota;
|
||||
use crate::services::storage;
|
||||
use crate::state::AppState;
|
||||
|
||||
use axum::extract::{ConnectInfo, Multipart, State};
|
||||
@@ -19,7 +20,6 @@ use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use sqlx::FromRow;
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub fn router() -> Router<AppState> {
|
||||
@@ -300,35 +300,31 @@ async fn compress_json(
|
||||
&& req.max_height.is_none();
|
||||
let charge_units = !skip_charge && compressed_size < original_size;
|
||||
|
||||
if charge_units {
|
||||
if let QuotaContext::Anonymous { session_id, ip } = "a_ctx {
|
||||
quota::consume_anonymous_units(&state, session_id, *ip, 1).await?;
|
||||
}
|
||||
}
|
||||
|
||||
if !state.config.storage_type.eq_ignore_ascii_case("local") {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::StorageUnavailable,
|
||||
"当前仅支持本地存储（STORAGE_TYPE=local）",
|
||||
));
|
||||
}
|
||||
|
||||
tokio::fs::create_dir_all(&state.config.storage_path)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "创建存储目录失败").with_source(err)
|
||||
})?;
|
||||
|
||||
let task_id = Uuid::new_v4();
|
||||
let file_id = Uuid::new_v4();
|
||||
let file_path = format!(
|
||||
"{}/{}.{}",
|
||||
state.config.storage_path,
|
||||
file_id,
|
||||
format_out.extension()
|
||||
);
|
||||
let retention_hours = retention.num_hours();
|
||||
let object_key =
|
||||
storage::result_key(retention_hours, task_id, file_id, format_out.extension());
|
||||
let stored =
|
||||
storage::store_bytes(&state, &object_key, compressed, format_out.content_type())
|
||||
.await?;
|
||||
|
||||
write_file(&file_path, &compressed).await?;
|
||||
if charge_units {
|
||||
if let QuotaContext::Anonymous { session_id, ip } = "a_ctx {
|
||||
if let Err(err) = quota::consume_anonymous_units(&state, session_id, *ip, 1).await {
|
||||
let _ = storage::delete_object(
|
||||
&state,
|
||||
&storage::ObjectLocator {
|
||||
backend: stored.backend.clone(),
|
||||
endpoint_id: stored.endpoint_id,
|
||||
key: stored.key.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let expires_at = Utc::now() + retention;
|
||||
|
||||
@@ -338,7 +334,7 @@ async fn compress_json(
|
||||
ip,
|
||||
task_id,
|
||||
file_id,
|
||||
&file_path,
|
||||
&stored,
|
||||
&req.file_name,
|
||||
req.max_width,
|
||||
req.max_height,
|
||||
@@ -350,12 +346,21 @@ async fn compress_json(
|
||||
compressed_size,
|
||||
saved_percent,
|
||||
expires_at,
|
||||
retention_hours,
|
||||
"a_ctx,
|
||||
charge_units,
|
||||
)
|
||||
.await
|
||||
{
|
||||
let _ = tokio::fs::remove_file(&file_path).await;
|
||||
let _ = storage::delete_object(
|
||||
&state,
|
||||
&storage::ObjectLocator {
|
||||
backend: stored.backend.clone(),
|
||||
endpoint_id: stored.endpoint_id,
|
||||
key: stored.key.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
@@ -671,27 +676,18 @@ async fn compress_direct(
|
||||
&& req.max_height.is_none();
|
||||
let charge_units = !skip_charge && compressed_size < original_size;
|
||||
|
||||
if !state.config.storage_type.eq_ignore_ascii_case("local") {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::StorageUnavailable,
|
||||
"当前仅支持本地存储（STORAGE_TYPE=local）",
|
||||
));
|
||||
}
|
||||
tokio::fs::create_dir_all(&state.config.storage_path)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "创建存储目录失败").with_source(err)
|
||||
})?;
|
||||
|
||||
let task_id = Uuid::new_v4();
|
||||
let file_id = Uuid::new_v4();
|
||||
let file_path = format!(
|
||||
"{}/{}.{}",
|
||||
state.config.storage_path,
|
||||
file_id,
|
||||
format_out.extension()
|
||||
);
|
||||
write_file(&file_path, &compressed).await?;
|
||||
let retention_hours = retention.num_hours();
|
||||
let object_key =
|
||||
storage::result_key(retention_hours, task_id, file_id, format_out.extension());
|
||||
let stored = storage::store_bytes(
|
||||
&state,
|
||||
&object_key,
|
||||
compressed.clone(),
|
||||
format_out.content_type(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let expires_at = Utc::now() + retention;
|
||||
|
||||
@@ -701,7 +697,7 @@ async fn compress_direct(
|
||||
ip,
|
||||
task_id,
|
||||
file_id,
|
||||
&file_path,
|
||||
&stored,
|
||||
&req.file_name,
|
||||
req.max_width,
|
||||
req.max_height,
|
||||
@@ -713,12 +709,21 @@ async fn compress_direct(
|
||||
compressed_size,
|
||||
saved_percent,
|
||||
expires_at,
|
||||
retention_hours,
|
||||
"a_ctx,
|
||||
charge_units,
|
||||
)
|
||||
.await
|
||||
{
|
||||
let _ = tokio::fs::remove_file(&file_path).await;
|
||||
let _ = storage::delete_object(
|
||||
&state,
|
||||
&storage::ObjectLocator {
|
||||
backend: stored.backend.clone(),
|
||||
endpoint_id: stored.endpoint_id,
|
||||
key: stored.key.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
@@ -800,19 +805,11 @@ async fn compress_direct(
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_file(path: &str, bytes: &[u8]) -> Result<(), AppError> {
|
||||
let mut file = tokio::fs::File::create(path).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "写入文件失败").with_source(err)
|
||||
})?;
|
||||
file.write_all(bytes).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "写入文件失败").with_source(err)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, FromRow)]
|
||||
struct DirectReplayRow {
|
||||
storage_path: Option<String>,
|
||||
storage_backend: String,
|
||||
storage_endpoint_id: Option<Uuid>,
|
||||
storage_key: Option<String>,
|
||||
output_format: String,
|
||||
file_status: String,
|
||||
expires_at: DateTime<Utc>,
|
||||
@@ -828,7 +825,9 @@ async fn load_direct_replay_bytes(
|
||||
sqlx::query_as::<_, DirectReplayRow>(
|
||||
r#"
|
||||
SELECT
|
||||
f.storage_path,
|
||||
f.storage_backend,
|
||||
f.storage_endpoint_id,
|
||||
COALESCE(f.storage_key, f.storage_path) AS storage_key,
|
||||
f.output_format,
|
||||
f.status::text AS file_status,
|
||||
t.expires_at
|
||||
@@ -846,7 +845,9 @@ async fn load_direct_replay_bytes(
|
||||
sqlx::query_as::<_, DirectReplayRow>(
|
||||
r#"
|
||||
SELECT
|
||||
f.storage_path,
|
||||
f.storage_backend,
|
||||
f.storage_endpoint_id,
|
||||
COALESCE(f.storage_key, f.storage_path) AS storage_key,
|
||||
f.output_format,
|
||||
f.status::text AS file_status,
|
||||
t.expires_at
|
||||
@@ -873,13 +874,18 @@ async fn load_direct_replay_bytes(
|
||||
if row.file_status != "completed" {
|
||||
return Err(AppError::new(ErrorCode::NotFound, "文件不存在"));
|
||||
}
|
||||
let Some(path) = row.storage_path else {
|
||||
let Some(key) = row.storage_key else {
|
||||
return Err(AppError::new(ErrorCode::NotFound, "文件不存在"));
|
||||
};
|
||||
|
||||
let bytes = tokio::fs::read(&path).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "读取文件失败").with_source(err)
|
||||
})?;
|
||||
let bytes = storage::read_bytes(
|
||||
state,
|
||||
&storage::ObjectLocator {
|
||||
backend: row.storage_backend,
|
||||
endpoint_id: row.storage_endpoint_id,
|
||||
key,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let fmt = compress::parse_output_format(&row.output_format)?;
|
||||
Ok((bytes, fmt))
|
||||
@@ -1082,7 +1088,7 @@ async fn record_task_and_metering(
|
||||
client_ip: IpAddr,
|
||||
task_id: Uuid,
|
||||
file_id: Uuid,
|
||||
file_path: &str,
|
||||
stored: &storage::StoredObject,
|
||||
original_name: &str,
|
||||
max_width: Option<u32>,
|
||||
max_height: Option<u32>,
|
||||
@@ -1094,6 +1100,7 @@ async fn record_task_and_metering(
|
||||
compressed_size: u64,
|
||||
saved_percent: f64,
|
||||
expires_at: DateTime<Utc>,
|
||||
retention_hours: i64,
|
||||
quota_ctx: &QuotaContext,
|
||||
charge_units: bool,
|
||||
) -> Result<(), AppError> {
|
||||
@@ -1123,13 +1130,13 @@ async fn record_task_and_metering(
|
||||
compression_rate,
|
||||
total_files, completed_files, failed_files,
|
||||
total_original_size, total_compressed_size,
|
||||
started_at, completed_at, expires_at
|
||||
started_at, completed_at, expires_at, retention_hours
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5::inet, $6::task_source, 'completed',
|
||||
$7::compression_level, $8, $9, $10, $11, $12,
|
||||
1, 1, 0,
|
||||
$13, $14,
|
||||
NOW(), NOW(), $15
|
||||
NOW(), NOW(), $15, $16
|
||||
)
|
||||
"#,
|
||||
)
|
||||
@@ -1148,6 +1155,7 @@ async fn record_task_and_metering(
|
||||
.bind(original_size as i64)
|
||||
.bind(compressed_size as i64)
|
||||
.bind(expires_at)
|
||||
.bind(retention_hours as i32)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "创建任务失败").with_source(err))?;
|
||||
@@ -1158,12 +1166,13 @@ async fn record_task_and_metering(
|
||||
id, task_id,
|
||||
original_name, original_format, output_format,
|
||||
original_size, compressed_size, saved_percent,
|
||||
storage_path, status, completed_at
|
||||
storage_path, storage_backend, storage_endpoint_id, storage_key, storage_etag,
|
||||
status, completed_at
|
||||
) VALUES (
|
||||
$1, $2,
|
||||
$3, $4, $5,
|
||||
$6, $7, $8,
|
||||
$9, 'completed', NOW()
|
||||
$9, $10, $11, $12, $13, 'completed', NOW()
|
||||
)
|
||||
"#,
|
||||
)
|
||||
@@ -1175,7 +1184,15 @@ async fn record_task_and_metering(
|
||||
.bind(original_size as i64)
|
||||
.bind(compressed_size as i64)
|
||||
.bind(saved_percent)
|
||||
.bind(file_path)
|
||||
.bind(if stored.backend == "local" {
|
||||
Some(stored.key.as_str())
|
||||
} else {
|
||||
None
|
||||
})
|
||||
.bind(&stored.backend)
|
||||
.bind(stored.endpoint_id)
|
||||
.bind(&stored.key)
|
||||
.bind(&stored.etag)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "创建文件记录失败").with_source(err))?;
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
use crate::api::context;
|
||||
use crate::error::{AppError, ErrorCode};
|
||||
use crate::services::storage;
|
||||
use crate::state::AppState;
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::extract::{ConnectInfo, Path, State};
|
||||
use axum::http::{header, HeaderMap};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::response::{IntoResponse, Redirect, Response};
|
||||
use axum::routing::get;
|
||||
use axum::Router;
|
||||
use chrono::{DateTime, Utc};
|
||||
@@ -25,7 +26,9 @@ pub fn router() -> Router<AppState> {
|
||||
|
||||
#[derive(Debug, FromRow)]
|
||||
struct DownloadRow {
|
||||
storage_path: Option<String>,
|
||||
storage_backend: String,
|
||||
storage_endpoint_id: Option<Uuid>,
|
||||
storage_key: Option<String>,
|
||||
output_format: String,
|
||||
original_name: String,
|
||||
file_status: String,
|
||||
@@ -47,7 +50,9 @@ async fn download_file(
|
||||
let row = sqlx::query_as::<_, DownloadRow>(
|
||||
r#"
|
||||
SELECT
|
||||
f.storage_path,
|
||||
f.storage_backend,
|
||||
f.storage_endpoint_id,
|
||||
COALESCE(f.storage_key, f.storage_path) AS storage_key,
|
||||
f.output_format,
|
||||
f.original_name,
|
||||
f.status::text AS file_status,
|
||||
@@ -75,32 +80,51 @@ async fn download_file(
|
||||
|
||||
authorize_download(&principal, &row)?;
|
||||
|
||||
let Some(path) = &row.storage_path else {
|
||||
let Some(key) = row.storage_key else {
|
||||
return Err(AppError::new(ErrorCode::NotFound, "文件不存在"));
|
||||
};
|
||||
let object = storage::ObjectLocator {
|
||||
backend: row.storage_backend,
|
||||
endpoint_id: row.storage_endpoint_id,
|
||||
key,
|
||||
};
|
||||
respond_object(
|
||||
&state,
|
||||
jar,
|
||||
&object,
|
||||
&output_file_name(&row.original_name, &row.output_format),
|
||||
content_type(&row.output_format),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
let file = tokio::fs::File::open(path).await.map_err(|err| {
|
||||
async fn respond_object(
|
||||
state: &AppState,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
object: &storage::ObjectLocator,
|
||||
file_name: &str,
|
||||
content_type_value: &str,
|
||||
) -> Result<(axum_extra::extract::cookie::CookieJar, Response), AppError> {
|
||||
if let Some(url) =
|
||||
storage::presign_download(state, object, file_name, content_type_value).await?
|
||||
{
|
||||
return Ok((jar, Redirect::temporary(&url).into_response()));
|
||||
}
|
||||
|
||||
let file = tokio::fs::File::open(&object.key).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "读取文件失败").with_source(err)
|
||||
})?;
|
||||
let content_length = file.metadata().await.ok().map(|metadata| metadata.len());
|
||||
let body = Body::from_stream(ReaderStream::new(file));
|
||||
|
||||
let mut resp_headers = HeaderMap::new();
|
||||
resp_headers.insert(
|
||||
header::CONTENT_TYPE,
|
||||
content_type(&row.output_format).parse().unwrap(),
|
||||
);
|
||||
resp_headers.insert(
|
||||
header::CONTENT_DISPOSITION,
|
||||
content_disposition(&row.original_name)?,
|
||||
);
|
||||
resp_headers.insert(header::CONTENT_TYPE, content_type_value.parse().unwrap());
|
||||
resp_headers.insert(header::CONTENT_DISPOSITION, content_disposition(file_name)?);
|
||||
if let Some(content_length) = content_length {
|
||||
resp_headers.insert(
|
||||
header::CONTENT_LENGTH,
|
||||
content_length.to_string().parse().unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
Ok((jar, (resp_headers, body).into_response()))
|
||||
}
|
||||
|
||||
@@ -126,10 +150,36 @@ fn content_type(format: &str) -> &'static str {
|
||||
"jpeg" | "jpg" => "image/jpeg",
|
||||
"webp" => "image/webp",
|
||||
"avif" => "image/avif",
|
||||
"gif" => "image/gif",
|
||||
"bmp" => "image/bmp",
|
||||
"tif" | "tiff" => "image/tiff",
|
||||
"ico" => "image/x-icon",
|
||||
_ => "application/octet-stream",
|
||||
}
|
||||
}
|
||||
|
||||
fn output_file_name(original_name: &str, output_format: &str) -> String {
|
||||
let sanitized = sanitize_filename(original_name);
|
||||
let base = sanitized
|
||||
.rsplit_once('.')
|
||||
.map(|(value, _)| value)
|
||||
.unwrap_or(&sanitized)
|
||||
.trim_end_matches('.');
|
||||
let base = if base.is_empty() { "download" } else { base };
|
||||
let extension = match output_format.trim().to_ascii_lowercase().as_str() {
|
||||
"jpeg" | "jpg" => "jpg",
|
||||
"png" => "png",
|
||||
"webp" => "webp",
|
||||
"avif" => "avif",
|
||||
"gif" => "gif",
|
||||
"bmp" => "bmp",
|
||||
"tif" | "tiff" => "tiff",
|
||||
"ico" => "ico",
|
||||
_ => "bin",
|
||||
};
|
||||
format!("{base}.{extension}")
|
||||
}
|
||||
|
||||
fn sanitize_filename(name: &str) -> String {
|
||||
let mut out = name.trim().to_string();
|
||||
if out.is_empty() {
|
||||
@@ -177,11 +227,17 @@ struct TaskZipRow {
|
||||
status: String,
|
||||
completed_at: Option<DateTime<Utc>>,
|
||||
expires_at: DateTime<Utc>,
|
||||
retention_hours: i32,
|
||||
zip_storage_backend: Option<String>,
|
||||
zip_storage_endpoint_id: Option<Uuid>,
|
||||
zip_storage_key: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, FromRow)]
|
||||
struct TaskZipFileRow {
|
||||
storage_path: Option<String>,
|
||||
storage_backend: String,
|
||||
storage_endpoint_id: Option<Uuid>,
|
||||
storage_key: Option<String>,
|
||||
original_name: String,
|
||||
output_format: String,
|
||||
}
|
||||
@@ -203,7 +259,11 @@ async fn download_task_zip(
|
||||
session_id,
|
||||
status::text AS status,
|
||||
completed_at,
|
||||
expires_at
|
||||
expires_at,
|
||||
retention_hours,
|
||||
zip_storage_backend,
|
||||
zip_storage_endpoint_id,
|
||||
zip_storage_key
|
||||
FROM tasks
|
||||
WHERE id = $1
|
||||
"#,
|
||||
@@ -235,26 +295,29 @@ async fn download_task_zip(
|
||||
}
|
||||
}
|
||||
|
||||
if !state.config.storage_type.eq_ignore_ascii_case("local") {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::StorageUnavailable,
|
||||
"当前仅支持本地存储（STORAGE_TYPE=local）",
|
||||
));
|
||||
}
|
||||
|
||||
let zip_dir = format!("{}/zips", state.config.storage_path);
|
||||
tokio::fs::create_dir_all(&zip_dir).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "创建存储目录失败").with_source(err)
|
||||
})?;
|
||||
let zip_path = PathBuf::from(format!("{zip_dir}/{task_id}.zip"));
|
||||
|
||||
if tokio::fs::try_exists(&zip_path).await.unwrap_or(false) {
|
||||
return stream_zip(jar, zip_path, task_id).await;
|
||||
if let (Some(backend), Some(key)) = (
|
||||
task.zip_storage_backend.clone(),
|
||||
task.zip_storage_key.clone(),
|
||||
) {
|
||||
return respond_object(
|
||||
&state,
|
||||
jar,
|
||||
&storage::ObjectLocator {
|
||||
backend,
|
||||
endpoint_id: task.zip_storage_endpoint_id,
|
||||
key,
|
||||
},
|
||||
&format!("task_{task_id}.zip"),
|
||||
"application/zip",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let rows = sqlx::query_as::<_, TaskZipFileRow>(
|
||||
r#"
|
||||
SELECT storage_path, original_name, output_format
|
||||
SELECT storage_backend, storage_endpoint_id,
|
||||
COALESCE(storage_key, storage_path) AS storage_key,
|
||||
original_name, output_format
|
||||
FROM task_files
|
||||
WHERE task_id = $1 AND status = 'completed'
|
||||
ORDER BY created_at ASC
|
||||
@@ -269,53 +332,92 @@ async fn download_task_zip(
|
||||
return Err(AppError::new(ErrorCode::NotFound, "没有可打包的文件"));
|
||||
}
|
||||
|
||||
let mut used_names: HashMap<String, usize> = HashMap::new();
|
||||
let mut entries: Vec<(String, String)> = Vec::new();
|
||||
for row in rows {
|
||||
let Some(path) = row.storage_path else {
|
||||
continue;
|
||||
};
|
||||
let name = build_zip_entry_name(&row.original_name, &row.output_format, &mut used_names);
|
||||
entries.push((name, path));
|
||||
}
|
||||
if entries.is_empty() {
|
||||
return Err(AppError::new(ErrorCode::NotFound, "没有可打包的文件"));
|
||||
}
|
||||
|
||||
let zip_path_cloned = zip_path.clone();
|
||||
let task_id_str = task_id.to_string();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
generate_zip_file(&zip_path_cloned, &task_id_str, &entries)
|
||||
})
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "生成 ZIP 失败").with_source(err))?
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "生成 ZIP 失败").with_source(err))?;
|
||||
|
||||
stream_zip(jar, zip_path, task_id).await
|
||||
}
|
||||
|
||||
async fn stream_zip(
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
zip_path: PathBuf,
|
||||
task_id: Uuid,
|
||||
) -> Result<(axum_extra::extract::cookie::CookieJar, Response), AppError> {
|
||||
let file = tokio::fs::File::open(&zip_path).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "读取 ZIP 失败").with_source(err)
|
||||
let temp_dir = PathBuf::from(format!(
|
||||
"{}/tmp/zips/{task_id}-{}",
|
||||
state.config.storage_path,
|
||||
Uuid::new_v4()
|
||||
));
|
||||
tokio::fs::create_dir_all(&temp_dir).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "创建 ZIP 临时目录失败").with_source(err)
|
||||
})?;
|
||||
let zip_path = temp_dir.join(format!("task_{task_id}.zip"));
|
||||
|
||||
let stream = ReaderStream::new(file);
|
||||
let body = Body::from_stream(stream);
|
||||
let build_result: Result<storage::StoredObject, AppError> = async {
|
||||
let mut used_names: HashMap<String, usize> = HashMap::new();
|
||||
let mut entries: Vec<(String, String)> = Vec::new();
|
||||
for (index, row) in rows.into_iter().enumerate() {
|
||||
let Some(key) = row.storage_key else {
|
||||
continue;
|
||||
};
|
||||
let path = temp_dir.join(format!("entry-{index}"));
|
||||
storage::download_to_file(
|
||||
&state,
|
||||
&storage::ObjectLocator {
|
||||
backend: row.storage_backend,
|
||||
endpoint_id: row.storage_endpoint_id,
|
||||
key,
|
||||
},
|
||||
&path,
|
||||
)
|
||||
.await?;
|
||||
let name =
|
||||
build_zip_entry_name(&row.original_name, &row.output_format, &mut used_names);
|
||||
entries.push((name, path.to_string_lossy().to_string()));
|
||||
}
|
||||
if entries.is_empty() {
|
||||
return Err(AppError::new(ErrorCode::NotFound, "没有可打包的文件"));
|
||||
}
|
||||
|
||||
let mut resp_headers = HeaderMap::new();
|
||||
resp_headers.insert(header::CONTENT_TYPE, "application/zip".parse().unwrap());
|
||||
resp_headers.insert(
|
||||
header::CONTENT_DISPOSITION,
|
||||
format!("attachment; filename=\"task_{task_id}.zip\"")
|
||||
.parse()
|
||||
.unwrap(),
|
||||
);
|
||||
let zip_path_cloned = zip_path.clone();
|
||||
let task_id_str = task_id.to_string();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
generate_zip_file(&zip_path_cloned, &task_id_str, &entries)
|
||||
})
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "生成 ZIP 失败").with_source(err))?
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "生成 ZIP 失败").with_source(err))?;
|
||||
|
||||
Ok((jar, (resp_headers, body).into_response()))
|
||||
let object_key = storage::archive_key(task.retention_hours as i64, task_id);
|
||||
storage::store_file(&state, &object_key, &zip_path, "application/zip").await
|
||||
}
|
||||
.await;
|
||||
|
||||
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
|
||||
let stored = build_result?;
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE tasks
|
||||
SET zip_storage_backend = $2,
|
||||
zip_storage_endpoint_id = $3,
|
||||
zip_storage_key = $4,
|
||||
zip_storage_etag = $5,
|
||||
zip_size = $6
|
||||
WHERE id = $1 AND zip_storage_key IS NULL
|
||||
"#,
|
||||
)
|
||||
.bind(task_id)
|
||||
.bind(&stored.backend)
|
||||
.bind(stored.endpoint_id)
|
||||
.bind(&stored.key)
|
||||
.bind(&stored.etag)
|
||||
.bind(stored.size as i64)
|
||||
.execute(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "记录 ZIP 对象失败").with_source(err))?;
|
||||
|
||||
respond_object(
|
||||
&state,
|
||||
jar,
|
||||
&storage::ObjectLocator {
|
||||
backend: stored.backend,
|
||||
endpoint_id: stored.endpoint_id,
|
||||
key: stored.key,
|
||||
},
|
||||
&format!("task_{task_id}.zip"),
|
||||
"application/zip",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn build_zip_entry_name(
|
||||
@@ -333,6 +435,10 @@ fn build_zip_entry_name(
|
||||
"png" => "png",
|
||||
"webp" => "webp",
|
||||
"avif" => "avif",
|
||||
"gif" => "gif",
|
||||
"bmp" => "bmp",
|
||||
"tif" | "tiff" => "tiff",
|
||||
"ico" => "ico",
|
||||
_ => "bin",
|
||||
};
|
||||
|
||||
@@ -411,4 +517,10 @@ mod tests {
|
||||
assert_eq!(sanitized, "a".repeat(119));
|
||||
assert!(sanitized.is_char_boundary(sanitized.len()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_file_name_matches_converted_format() {
|
||||
assert_eq!(output_file_name("photo.png", "webp"), "photo.webp");
|
||||
assert_eq!(output_file_name("没有扩展名", "jpeg"), "没有扩展名.jpg");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
mod admin;
|
||||
mod admin_storage;
|
||||
mod auth;
|
||||
mod billing;
|
||||
mod compress;
|
||||
@@ -62,5 +63,6 @@ fn v1_router() -> Router<AppState> {
|
||||
.merge(webhooks::router())
|
||||
.merge(user::router())
|
||||
.merge(admin::router())
|
||||
.merge(admin_storage::router())
|
||||
.fallback(response::not_found)
|
||||
}
|
||||
|
||||
100
src/api/tasks.rs
100
src/api/tasks.rs
@@ -6,6 +6,7 @@ use crate::services::billing::{BillingContext, Plan};
|
||||
use crate::services::compress;
|
||||
use crate::services::compress::{CompressionLevel, ImageFmt};
|
||||
use crate::services::idempotency;
|
||||
use crate::services::storage;
|
||||
use crate::state::AppState;
|
||||
|
||||
use axum::extract::{ConnectInfo, Multipart, Path, State};
|
||||
@@ -72,13 +73,6 @@ async fn create_batch_task(
|
||||
let ip = context::client_ip(&headers, addr.ip());
|
||||
let (jar, principal) = context::authenticate(&state, jar, &headers, ip).await?;
|
||||
|
||||
if !state.config.storage_type.eq_ignore_ascii_case("local") {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::StorageUnavailable,
|
||||
"当前仅支持本地存储（STORAGE_TYPE=local）",
|
||||
));
|
||||
}
|
||||
|
||||
let idempotency_key = headers
|
||||
.get("idempotency-key")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
@@ -217,13 +211,8 @@ async fn create_batch_task(
|
||||
}
|
||||
}?;
|
||||
|
||||
tokio::fs::create_dir_all(&state.config.storage_path)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "创建存储目录失败").with_source(err)
|
||||
})?;
|
||||
|
||||
let expires_at = Utc::now() + retention;
|
||||
let retention_hours = retention.num_hours();
|
||||
let (user_id, session_id, api_key_id) = match &task_owner {
|
||||
TaskOwner::Anonymous { session_id } => (None, Some(session_id.clone()), None),
|
||||
TaskOwner::User { user_id } => (Some(*user_id), None, None),
|
||||
@@ -248,13 +237,13 @@ async fn create_batch_task(
|
||||
compression_rate,
|
||||
total_files, completed_files, failed_files,
|
||||
total_original_size, total_compressed_size,
|
||||
expires_at
|
||||
expires_at, retention_hours
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5::inet, $6::task_source, 'pending',
|
||||
$7::compression_level, $8, $9, $10, $11, $12,
|
||||
$13, 0, 0,
|
||||
$14, 0,
|
||||
$15
|
||||
$15, $16
|
||||
)
|
||||
"#,
|
||||
)
|
||||
@@ -273,6 +262,7 @@ async fn create_batch_task(
|
||||
.bind(files.len() as i32)
|
||||
.bind(total_original_size)
|
||||
.bind(expires_at)
|
||||
.bind(retention_hours as i32)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "创建任务失败").with_source(err))?;
|
||||
@@ -284,7 +274,7 @@ async fn create_batch_task(
|
||||
id, task_id,
|
||||
original_name, original_format, output_format,
|
||||
original_size,
|
||||
storage_path, status
|
||||
input_path, status
|
||||
) VALUES (
|
||||
$1, $2,
|
||||
$3, $4, $5,
|
||||
@@ -990,24 +980,61 @@ async fn delete_task(
|
||||
));
|
||||
}
|
||||
|
||||
let paths: Vec<Option<String>> =
|
||||
sqlx::query_scalar("SELECT storage_path FROM task_files WHERE task_id = $1")
|
||||
.bind(task_id)
|
||||
.fetch_all(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "查询文件失败").with_source(err))?;
|
||||
let files = sqlx::query_as::<_, TaskStorageRow>(
|
||||
r#"
|
||||
SELECT storage_backend, storage_endpoint_id,
|
||||
COALESCE(storage_key, storage_path) AS storage_key,
|
||||
input_path
|
||||
FROM task_files
|
||||
WHERE task_id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(task_id)
|
||||
.fetch_all(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "查询文件失败").with_source(err))?;
|
||||
|
||||
for p in paths.into_iter().flatten() {
|
||||
let _ = tokio::fs::remove_file(p).await;
|
||||
for file in files {
|
||||
if let Some(key) = file.storage_key {
|
||||
storage::delete_object(
|
||||
&state,
|
||||
&storage::ObjectLocator {
|
||||
backend: file.storage_backend,
|
||||
endpoint_id: file.storage_endpoint_id,
|
||||
key,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
if let Some(input_path) = file.input_path {
|
||||
let _ = tokio::fs::remove_file(input_path).await;
|
||||
}
|
||||
}
|
||||
|
||||
if state.config.storage_type.eq_ignore_ascii_case("local") {
|
||||
let zip_path = format!("{}/zips/{task_id}.zip", state.config.storage_path);
|
||||
let _ = tokio::fs::remove_file(zip_path).await;
|
||||
let orig_dir = format!("{}/orig/{task_id}", state.config.storage_path);
|
||||
let _ = tokio::fs::remove_dir_all(orig_dir).await;
|
||||
let zip = sqlx::query_as::<_, TaskZipStorageRow>(
|
||||
"SELECT zip_storage_backend, zip_storage_endpoint_id, zip_storage_key FROM tasks WHERE id = $1",
|
||||
)
|
||||
.bind(task_id)
|
||||
.fetch_one(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "查询 ZIP 文件失败").with_source(err))?;
|
||||
if let (Some(backend), Some(key)) = (zip.zip_storage_backend, zip.zip_storage_key) {
|
||||
storage::delete_object(
|
||||
&state,
|
||||
&storage::ObjectLocator {
|
||||
backend,
|
||||
endpoint_id: zip.zip_storage_endpoint_id,
|
||||
key,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
let legacy_zip_path = format!("{}/zips/{task_id}.zip", state.config.storage_path);
|
||||
let _ = tokio::fs::remove_file(legacy_zip_path).await;
|
||||
let orig_dir = format!("{}/orig/{task_id}", state.config.storage_path);
|
||||
let _ = tokio::fs::remove_dir_all(orig_dir).await;
|
||||
|
||||
let deleted = sqlx::query("DELETE FROM tasks WHERE id = $1")
|
||||
.bind(task_id)
|
||||
.execute(&state.db)
|
||||
@@ -1027,6 +1054,21 @@ async fn delete_task(
|
||||
))
|
||||
}
|
||||
|
||||
#[derive(Debug, FromRow)]
|
||||
struct TaskStorageRow {
|
||||
storage_backend: String,
|
||||
storage_endpoint_id: Option<Uuid>,
|
||||
storage_key: Option<String>,
|
||||
input_path: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, FromRow)]
|
||||
struct TaskZipStorageRow {
|
||||
zip_storage_backend: Option<String>,
|
||||
zip_storage_endpoint_id: Option<Uuid>,
|
||||
zip_storage_key: Option<String>,
|
||||
}
|
||||
|
||||
fn authorize_task(
|
||||
principal: &context::Principal,
|
||||
user_id: Option<Uuid>,
|
||||
|
||||
@@ -500,7 +500,7 @@ async fn list_history(
|
||||
status: String,
|
||||
output_format: String,
|
||||
error_message: Option<String>,
|
||||
storage_path: Option<String>,
|
||||
has_storage: bool,
|
||||
}
|
||||
|
||||
let now = Utc::now();
|
||||
@@ -517,7 +517,7 @@ async fn list_history(
|
||||
status::text AS status,
|
||||
output_format,
|
||||
error_message,
|
||||
storage_path
|
||||
COALESCE(storage_key, storage_path) IS NOT NULL AS has_storage
|
||||
FROM task_files
|
||||
WHERE task_id = $1
|
||||
ORDER BY created_at ASC
|
||||
@@ -540,7 +540,7 @@ async fn list_history(
|
||||
output_format: file.output_format,
|
||||
error_message: file.error_message,
|
||||
download_url: if file.status == "completed"
|
||||
&& file.storage_path.is_some()
|
||||
&& file.has_storage
|
||||
&& task.expires_at > now
|
||||
{
|
||||
Some(format!("/downloads/{}", file.id))
|
||||
|
||||
@@ -25,7 +25,6 @@ pub struct Config {
|
||||
pub stripe_secret_key: Option<String>,
|
||||
pub stripe_webhook_secret: Option<String>,
|
||||
|
||||
pub storage_type: String,
|
||||
pub storage_path: String,
|
||||
|
||||
pub allow_anonymous_upload: bool,
|
||||
@@ -95,13 +94,6 @@ impl Config {
|
||||
let stripe_secret_key = env_string("STRIPE_SECRET_KEY");
|
||||
let stripe_webhook_secret = env_string("STRIPE_WEBHOOK_SECRET");
|
||||
|
||||
let storage_type = env_string("STORAGE_TYPE").unwrap_or_else(|| "local".to_string());
|
||||
if !storage_type.eq_ignore_ascii_case("local") {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"STORAGE_TYPE 目前仅支持 local",
|
||||
));
|
||||
}
|
||||
let storage_path = env_string("STORAGE_PATH").unwrap_or_else(|| "./uploads".to_string());
|
||||
|
||||
let allow_anonymous_upload = env_bool("ALLOW_ANONYMOUS_UPLOAD").unwrap_or(true);
|
||||
@@ -141,7 +133,6 @@ impl Config {
|
||||
api_key_pepper,
|
||||
stripe_secret_key,
|
||||
stripe_webhook_secret,
|
||||
storage_type,
|
||||
storage_path,
|
||||
allow_anonymous_upload,
|
||||
anon_max_file_size_mb,
|
||||
|
||||
@@ -5,3 +5,4 @@ pub mod idempotency;
|
||||
pub mod mail;
|
||||
pub mod quota;
|
||||
pub mod settings;
|
||||
pub mod storage;
|
||||
|
||||
718
src/services/storage.rs
Normal file
718
src/services/storage.rs
Normal file
@@ -0,0 +1,718 @@
|
||||
use crate::error::{AppError, ErrorCode};
|
||||
use crate::services::settings;
|
||||
use crate::state::AppState;
|
||||
|
||||
use aws_sdk_s3::config::{
|
||||
BehaviorVersion, Credentials, Region, RequestChecksumCalculation, ResponseChecksumValidation,
|
||||
};
|
||||
use aws_sdk_s3::presigning::PresigningConfig;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use aws_sdk_s3::Client;
|
||||
use chrono::{DateTime, Datelike, Utc};
|
||||
use sqlx::FromRow;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use uuid::Uuid;
|
||||
|
||||
const MULTIPART_THRESHOLD: u64 = 64 * 1024 * 1024;
|
||||
const MULTIPART_PART_SIZE: usize = 16 * 1024 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, FromRow)]
|
||||
pub struct StorageEndpoint {
|
||||
pub id: Uuid,
|
||||
pub name: String,
|
||||
pub internal_endpoint: String,
|
||||
pub public_endpoint: String,
|
||||
pub bucket: String,
|
||||
pub region: String,
|
||||
pub access_key_encrypted: String,
|
||||
pub secret_key_encrypted: String,
|
||||
pub access_key_hint: String,
|
||||
pub force_path_style: bool,
|
||||
pub presign_ttl_seconds: i32,
|
||||
pub is_active: bool,
|
||||
pub last_test_at: Option<DateTime<Utc>>,
|
||||
pub last_test_ok: Option<bool>,
|
||||
pub last_test_error: Option<String>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
pub deleted_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct StoredObject {
|
||||
pub backend: String,
|
||||
pub endpoint_id: Option<Uuid>,
|
||||
pub key: String,
|
||||
pub etag: Option<String>,
|
||||
pub size: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ObjectLocator {
|
||||
pub backend: String,
|
||||
pub endpoint_id: Option<Uuid>,
|
||||
pub key: String,
|
||||
}
|
||||
|
||||
pub async fn list_endpoints(state: &AppState) -> Result<Vec<StorageEndpoint>, AppError> {
|
||||
sqlx::query_as::<_, StorageEndpoint>(
|
||||
r#"
|
||||
SELECT id, name, internal_endpoint, public_endpoint, bucket, region,
|
||||
access_key_encrypted, secret_key_encrypted, access_key_hint,
|
||||
force_path_style, presign_ttl_seconds, is_active,
|
||||
last_test_at, last_test_ok, last_test_error,
|
||||
created_at, updated_at, deleted_at
|
||||
FROM storage_endpoints
|
||||
WHERE deleted_at IS NULL
|
||||
ORDER BY is_active DESC, updated_at DESC
|
||||
"#,
|
||||
)
|
||||
.fetch_all(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "查询存储端点失败").with_source(err))
|
||||
}
|
||||
|
||||
pub async fn get_endpoint(
|
||||
state: &AppState,
|
||||
endpoint_id: Uuid,
|
||||
) -> Result<StorageEndpoint, AppError> {
|
||||
sqlx::query_as::<_, StorageEndpoint>(
|
||||
r#"
|
||||
SELECT id, name, internal_endpoint, public_endpoint, bucket, region,
|
||||
access_key_encrypted, secret_key_encrypted, access_key_hint,
|
||||
force_path_style, presign_ttl_seconds, is_active,
|
||||
last_test_at, last_test_ok, last_test_error,
|
||||
created_at, updated_at, deleted_at
|
||||
FROM storage_endpoints
|
||||
WHERE id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(endpoint_id)
|
||||
.fetch_optional(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "查询存储端点失败").with_source(err))?
|
||||
.ok_or_else(|| AppError::new(ErrorCode::NotFound, "存储端点不存在"))
|
||||
}
|
||||
|
||||
pub async fn active_endpoint(state: &AppState) -> Result<Option<StorageEndpoint>, AppError> {
|
||||
sqlx::query_as::<_, StorageEndpoint>(
|
||||
r#"
|
||||
SELECT id, name, internal_endpoint, public_endpoint, bucket, region,
|
||||
access_key_encrypted, secret_key_encrypted, access_key_hint,
|
||||
force_path_style, presign_ttl_seconds, is_active,
|
||||
last_test_at, last_test_ok, last_test_error,
|
||||
created_at, updated_at, deleted_at
|
||||
FROM storage_endpoints
|
||||
WHERE is_active = true AND deleted_at IS NULL
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.fetch_optional(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "查询活动存储端点失败").with_source(err))
|
||||
}
|
||||
|
||||
pub fn result_key(retention_hours: i64, task_id: Uuid, file_id: Uuid, extension: &str) -> String {
|
||||
let now = Utc::now();
|
||||
format!(
|
||||
"results/{}/{:04}/{:02}/{task_id}/{file_id}.{}",
|
||||
retention_prefix(retention_hours),
|
||||
now.year(),
|
||||
now.month(),
|
||||
extension.trim_start_matches('.')
|
||||
)
|
||||
}
|
||||
|
||||
pub fn archive_key(retention_hours: i64, task_id: Uuid) -> String {
|
||||
let now = Utc::now();
|
||||
format!(
|
||||
"archives/{}/{:04}/{:02}/{task_id}.zip",
|
||||
retention_prefix(retention_hours),
|
||||
now.year(),
|
||||
now.month()
|
||||
)
|
||||
}
|
||||
|
||||
fn retention_prefix(hours: i64) -> String {
|
||||
match hours {
|
||||
0..=24 => "1d".to_string(),
|
||||
25..=168 => "7d".to_string(),
|
||||
169..=360 => "15d".to_string(),
|
||||
value => format!("custom-{}h", value.max(1)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn store_bytes(
|
||||
state: &AppState,
|
||||
key: &str,
|
||||
bytes: Vec<u8>,
|
||||
content_type: &str,
|
||||
) -> Result<StoredObject, AppError> {
|
||||
if let Some(endpoint) = active_endpoint(state).await? {
|
||||
let client = client_for(state, &endpoint, EndpointKind::Internal)?;
|
||||
let size = bytes.len() as u64;
|
||||
let output = client
|
||||
.put_object()
|
||||
.bucket(&endpoint.bucket)
|
||||
.key(key)
|
||||
.content_type(content_type)
|
||||
.body(ByteStream::from(bytes))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| storage_error("上传 S3 对象失败", err))?;
|
||||
|
||||
return Ok(StoredObject {
|
||||
backend: "s3".to_string(),
|
||||
endpoint_id: Some(endpoint.id),
|
||||
key: key.to_string(),
|
||||
etag: output.e_tag().map(ToOwned::to_owned),
|
||||
size,
|
||||
});
|
||||
}
|
||||
|
||||
let path = local_path(state, key)?;
|
||||
if let Some(parent) = path.parent() {
|
||||
tokio::fs::create_dir_all(parent).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "创建本地存储目录失败").with_source(err)
|
||||
})?;
|
||||
}
|
||||
tokio::fs::write(&path, &bytes).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "写入本地存储失败").with_source(err)
|
||||
})?;
|
||||
|
||||
Ok(StoredObject {
|
||||
backend: "local".to_string(),
|
||||
endpoint_id: None,
|
||||
key: path.to_string_lossy().to_string(),
|
||||
etag: None,
|
||||
size: bytes.len() as u64,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn store_file(
|
||||
state: &AppState,
|
||||
key: &str,
|
||||
path: &Path,
|
||||
content_type: &str,
|
||||
) -> Result<StoredObject, AppError> {
|
||||
let metadata = tokio::fs::metadata(path).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "读取待上传文件失败").with_source(err)
|
||||
})?;
|
||||
|
||||
if let Some(endpoint) = active_endpoint(state).await? {
|
||||
let client = client_for(state, &endpoint, EndpointKind::Internal)?;
|
||||
let etag = if metadata.len() >= MULTIPART_THRESHOLD {
|
||||
multipart_upload(&client, &endpoint.bucket, key, path, content_type).await?
|
||||
} else {
|
||||
let body = ByteStream::from_path(path).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "读取待上传文件失败").with_source(err)
|
||||
})?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&endpoint.bucket)
|
||||
.key(key)
|
||||
.content_type(content_type)
|
||||
.body(body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| storage_error("上传 S3 对象失败", err))?
|
||||
.e_tag()
|
||||
.map(ToOwned::to_owned)
|
||||
};
|
||||
|
||||
return Ok(StoredObject {
|
||||
backend: "s3".to_string(),
|
||||
endpoint_id: Some(endpoint.id),
|
||||
key: key.to_string(),
|
||||
etag,
|
||||
size: metadata.len(),
|
||||
});
|
||||
}
|
||||
|
||||
let destination = local_path(state, key)?;
|
||||
if let Some(parent) = destination.parent() {
|
||||
tokio::fs::create_dir_all(parent).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "创建本地存储目录失败").with_source(err)
|
||||
})?;
|
||||
}
|
||||
tokio::fs::copy(path, &destination).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "保存本地文件失败").with_source(err)
|
||||
})?;
|
||||
|
||||
Ok(StoredObject {
|
||||
backend: "local".to_string(),
|
||||
endpoint_id: None,
|
||||
key: destination.to_string_lossy().to_string(),
|
||||
etag: None,
|
||||
size: metadata.len(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn read_bytes(state: &AppState, object: &ObjectLocator) -> Result<Vec<u8>, AppError> {
|
||||
if object.backend == "local" {
|
||||
return tokio::fs::read(&object.key).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "读取本地文件失败").with_source(err)
|
||||
});
|
||||
}
|
||||
|
||||
let endpoint = endpoint_for_object(state, object).await?;
|
||||
let client = client_for(state, &endpoint, EndpointKind::Internal)?;
|
||||
let response = client
|
||||
.get_object()
|
||||
.bucket(&endpoint.bucket)
|
||||
.key(&object.key)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| storage_error("读取 S3 对象失败", err))?;
|
||||
let bytes = response
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.map_err(|err| storage_error("接收 S3 对象失败", err))?;
|
||||
Ok(bytes.into_bytes().to_vec())
|
||||
}
|
||||
|
||||
pub async fn download_to_file(
|
||||
state: &AppState,
|
||||
object: &ObjectLocator,
|
||||
destination: &Path,
|
||||
) -> Result<(), AppError> {
|
||||
if object.backend == "local" {
|
||||
tokio::fs::copy(&object.key, destination)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "复制本地文件失败").with_source(err)
|
||||
})?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let endpoint = endpoint_for_object(state, object).await?;
|
||||
let client = client_for(state, &endpoint, EndpointKind::Internal)?;
|
||||
let response = client
|
||||
.get_object()
|
||||
.bucket(&endpoint.bucket)
|
||||
.key(&object.key)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| storage_error("读取 S3 对象失败", err))?;
|
||||
let mut source = response.body.into_async_read();
|
||||
let mut output = tokio::fs::File::create(destination).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "创建临时文件失败").with_source(err)
|
||||
})?;
|
||||
tokio::io::copy(&mut source, &mut output)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "保存临时文件失败").with_source(err)
|
||||
})?;
|
||||
output.flush().await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "刷新临时文件失败").with_source(err)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn presign_download(
|
||||
state: &AppState,
|
||||
object: &ObjectLocator,
|
||||
file_name: &str,
|
||||
content_type: &str,
|
||||
) -> Result<Option<String>, AppError> {
|
||||
if object.backend == "local" {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let endpoint = endpoint_for_object(state, object).await?;
|
||||
let client = client_for(state, &endpoint, EndpointKind::Public)?;
|
||||
let ttl = Duration::from_secs(endpoint.presign_ttl_seconds as u64);
|
||||
let config = PresigningConfig::expires_in(ttl).map_err(|err| {
|
||||
AppError::new(ErrorCode::Internal, "生成下载签名配置失败").with_source(err)
|
||||
})?;
|
||||
let disposition = content_disposition(file_name);
|
||||
let request = client
|
||||
.get_object()
|
||||
.bucket(&endpoint.bucket)
|
||||
.key(&object.key)
|
||||
.response_content_disposition(disposition)
|
||||
.response_content_type(content_type)
|
||||
.presigned(config)
|
||||
.await
|
||||
.map_err(|err| storage_error("生成 S3 下载地址失败", err))?;
|
||||
Ok(Some(request.uri().to_string()))
|
||||
}
|
||||
|
||||
pub async fn delete_object(state: &AppState, object: &ObjectLocator) -> Result<(), AppError> {
|
||||
if object.backend == "local" {
|
||||
match tokio::fs::remove_file(&object.key).await {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(err) => {
|
||||
return Err(
|
||||
AppError::new(ErrorCode::StorageUnavailable, "删除本地文件失败")
|
||||
.with_source(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let endpoint = endpoint_for_object(state, object).await?;
|
||||
let client = client_for(state, &endpoint, EndpointKind::Internal)?;
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(&endpoint.bucket)
|
||||
.key(&object.key)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| storage_error("删除 S3 对象失败", err))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn test_endpoint(state: &AppState, endpoint: &StorageEndpoint) -> Result<(), AppError> {
|
||||
let client = client_for(state, endpoint, EndpointKind::Internal)?;
|
||||
client
|
||||
.head_bucket()
|
||||
.bucket(&endpoint.bucket)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| storage_error("S3 Bucket 访问失败", err))?;
|
||||
|
||||
let key = format!(".imageforge-health/{}.txt", Uuid::new_v4());
|
||||
let expected = b"imageforge-storage-check".to_vec();
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&endpoint.bucket)
|
||||
.key(&key)
|
||||
.content_type("text/plain")
|
||||
.body(ByteStream::from(expected.clone()))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| storage_error("S3 写入测试失败", err))?;
|
||||
|
||||
let result = async {
|
||||
let response = client
|
||||
.get_object()
|
||||
.bucket(&endpoint.bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| storage_error("S3 读取测试失败", err))?;
|
||||
let actual = response
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.map_err(|err| storage_error("S3 测试对象接收失败", err))?
|
||||
.into_bytes();
|
||||
if actual.as_ref() != expected.as_slice() {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::StorageUnavailable,
|
||||
"S3 读写校验内容不一致",
|
||||
));
|
||||
}
|
||||
|
||||
let public_client = client_for(state, endpoint, EndpointKind::Public)?;
|
||||
let signed = public_client
|
||||
.get_object()
|
||||
.bucket(&endpoint.bucket)
|
||||
.key(&key)
|
||||
.presigned(
|
||||
PresigningConfig::expires_in(Duration::from_secs(60)).map_err(|err| {
|
||||
AppError::new(ErrorCode::Internal, "生成公网下载测试签名失败").with_source(err)
|
||||
})?,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| storage_error("生成公网下载测试地址失败", err))?;
|
||||
let response = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(15))
|
||||
.build()
|
||||
.map_err(|err| storage_error("创建公网下载测试客户端失败", err))?
|
||||
.get(signed.uri().to_string())
|
||||
.header(reqwest::header::USER_AGENT, "ImageForge-Storage-Check/1.0")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| public_test_request_error(endpoint, err))?;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::StorageUnavailable,
|
||||
format!("公网 Endpoint 下载测试返回 HTTP {status}"),
|
||||
));
|
||||
}
|
||||
let public_bytes = response.bytes().await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "读取公网下载测试响应失败")
|
||||
.with_source(format!("response body error: {}", err.is_timeout()))
|
||||
})?;
|
||||
if public_bytes.as_ref() != expected.as_slice() {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::StorageUnavailable,
|
||||
"公网 Endpoint 下载内容校验不一致",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
let delete_result = client
|
||||
.delete_object()
|
||||
.bucket(&endpoint.bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
result?;
|
||||
delete_result.map_err(|err| storage_error("S3 删除测试失败", err))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn client_for(
|
||||
state: &AppState,
|
||||
endpoint: &StorageEndpoint,
|
||||
kind: EndpointKind,
|
||||
) -> Result<Client, AppError> {
|
||||
let access_key =
|
||||
settings::decrypt_secret(state, &endpoint.access_key_encrypted).map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "存储端点凭据不可用").with_source(err)
|
||||
})?;
|
||||
let secret_key =
|
||||
settings::decrypt_secret(state, &endpoint.secret_key_encrypted).map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "存储端点凭据不可用").with_source(err)
|
||||
})?;
|
||||
let endpoint_url = match kind {
|
||||
EndpointKind::Internal => &endpoint.internal_endpoint,
|
||||
EndpointKind::Public => &endpoint.public_endpoint,
|
||||
};
|
||||
Ok(build_client(
|
||||
access_key,
|
||||
secret_key,
|
||||
endpoint_url,
|
||||
&endpoint.region,
|
||||
endpoint.force_path_style,
|
||||
))
|
||||
}
|
||||
|
||||
fn build_client(
|
||||
access_key: String,
|
||||
secret_key: String,
|
||||
endpoint_url: &str,
|
||||
region: &str,
|
||||
force_path_style: bool,
|
||||
) -> Client {
|
||||
let config = aws_sdk_s3::Config::builder()
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.credentials_provider(Credentials::new(
|
||||
access_key,
|
||||
secret_key,
|
||||
None,
|
||||
None,
|
||||
"imageforge-admin",
|
||||
))
|
||||
.region(Region::new(region.to_string()))
|
||||
.endpoint_url(endpoint_url)
|
||||
.force_path_style(force_path_style)
|
||||
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
|
||||
.response_checksum_validation(ResponseChecksumValidation::WhenRequired)
|
||||
.build();
|
||||
Client::from_conf(config)
|
||||
}
|
||||
|
||||
async fn endpoint_for_object(
|
||||
state: &AppState,
|
||||
object: &ObjectLocator,
|
||||
) -> Result<StorageEndpoint, AppError> {
|
||||
let endpoint_id = object
|
||||
.endpoint_id
|
||||
.ok_or_else(|| AppError::new(ErrorCode::StorageUnavailable, "S3 对象缺少存储端点标识"))?;
|
||||
get_endpoint(state, endpoint_id).await
|
||||
}
|
||||
|
||||
fn local_path(state: &AppState, key: &str) -> Result<PathBuf, AppError> {
|
||||
if key.is_empty()
|
||||
|| key.starts_with('/')
|
||||
|| key.starts_with('\\')
|
||||
|| key.split('/').any(|part| part == "..")
|
||||
{
|
||||
return Err(AppError::new(ErrorCode::InvalidRequest, "非法存储对象键"));
|
||||
}
|
||||
Ok(Path::new(&state.config.storage_path).join(key))
|
||||
}
|
||||
|
||||
fn content_disposition(file_name: &str) -> String {
|
||||
let ascii = file_name
|
||||
.chars()
|
||||
.map(|ch| {
|
||||
if ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_') {
|
||||
ch
|
||||
} else {
|
||||
'_'
|
||||
}
|
||||
})
|
||||
.collect::<String>();
|
||||
let ascii = if ascii.is_empty() { "download" } else { &ascii };
|
||||
let encoded =
|
||||
percent_encoding::utf8_percent_encode(file_name, percent_encoding::NON_ALPHANUMERIC);
|
||||
format!("attachment; filename=\"{ascii}\"; filename*=UTF-8''{encoded}")
|
||||
}
|
||||
|
||||
async fn multipart_upload(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
path: &Path,
|
||||
content_type: &str,
|
||||
) -> Result<Option<String>, AppError> {
|
||||
let created = client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.content_type(content_type)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| storage_error("创建 S3 分片上传失败", err))?;
|
||||
let upload_id = created
|
||||
.upload_id()
|
||||
.ok_or_else(|| AppError::new(ErrorCode::StorageUnavailable, "S3 未返回分片上传标识"))?;
|
||||
|
||||
let result = async {
|
||||
let mut file = tokio::fs::File::open(path).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "读取待上传文件失败").with_source(err)
|
||||
})?;
|
||||
let mut parts = Vec::new();
|
||||
let mut part_number = 1;
|
||||
|
||||
loop {
|
||||
let mut buffer = vec![0u8; MULTIPART_PART_SIZE];
|
||||
let mut filled = 0;
|
||||
while filled < buffer.len() {
|
||||
let read = file.read(&mut buffer[filled..]).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "读取上传分片失败")
|
||||
.with_source(err)
|
||||
})?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
filled += read;
|
||||
}
|
||||
if filled == 0 {
|
||||
break;
|
||||
}
|
||||
buffer.truncate(filled);
|
||||
let uploaded = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(buffer))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| storage_error("上传 S3 分片失败", err))?;
|
||||
let part = CompletedPart::builder()
|
||||
.part_number(part_number)
|
||||
.set_e_tag(uploaded.e_tag().map(ToOwned::to_owned))
|
||||
.build();
|
||||
parts.push(part);
|
||||
part_number += 1;
|
||||
}
|
||||
|
||||
let completed = CompletedMultipartUpload::builder()
|
||||
.set_parts(Some(parts))
|
||||
.build();
|
||||
let response = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(completed)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| storage_error("完成 S3 分片上传失败", err))?;
|
||||
Ok(response.e_tag().map(ToOwned::to_owned))
|
||||
}
|
||||
.await;
|
||||
|
||||
if result.is_err() {
|
||||
let _ = client
|
||||
.abort_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await;
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn storage_error(message: &'static str, err: impl std::fmt::Display) -> AppError {
|
||||
AppError::new(ErrorCode::StorageUnavailable, message).with_source(err)
|
||||
}
|
||||
|
||||
fn public_test_request_error(endpoint: &StorageEndpoint, err: reqwest::Error) -> AppError {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "公网 Endpoint 下载测试失败").with_source(format!(
|
||||
"endpoint={}, connect={}, timeout={}",
|
||||
endpoint.public_endpoint,
|
||||
err.is_connect(),
|
||||
err.is_timeout()
|
||||
))
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum EndpointKind {
|
||||
Internal,
|
||||
Public,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn retention_prefixes_match_product_policy() {
|
||||
assert_eq!(retention_prefix(24), "1d");
|
||||
assert_eq!(retention_prefix(168), "7d");
|
||||
assert_eq!(retention_prefix(360), "15d");
|
||||
assert_eq!(retention_prefix(720), "custom-720h");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_keys_are_lifecycle_scoped() {
|
||||
let task_id = Uuid::nil();
|
||||
let file_id = Uuid::from_u128(1);
|
||||
let key = result_key(168, task_id, file_id, "webp");
|
||||
assert!(key.starts_with("results/7d/"));
|
||||
assert!(key.ends_with("/00000000-0000-0000-0000-000000000001.webp"));
|
||||
assert!(archive_key(360, task_id).starts_with("archives/15d/"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_disposition_never_injects_headers() {
|
||||
let value = content_disposition("a\r\nX-Test: yes.png");
|
||||
assert!(!value.contains('\r'));
|
||||
assert!(!value.contains('\n'));
|
||||
assert!(value.contains("filename*=UTF-8''"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn presigning_uses_public_path_style_endpoint() {
|
||||
let client = build_client(
|
||||
"GKTESTACCESSKEY".to_string(),
|
||||
"test-secret-key".to_string(),
|
||||
"https://files.example.com",
|
||||
"garage",
|
||||
true,
|
||||
);
|
||||
let request = client
|
||||
.get_object()
|
||||
.bucket("imageforge-results")
|
||||
.key("results/1d/test.webp")
|
||||
.presigned(PresigningConfig::expires_in(Duration::from_secs(300)).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let uri = request.uri().to_string();
|
||||
assert!(
|
||||
uri.starts_with("https://files.example.com/imageforge-results/results/1d/test.webp?")
|
||||
);
|
||||
assert!(uri.contains("X-Amz-Algorithm=AWS4-HMAC-SHA256"));
|
||||
assert!(uri.contains("X-Amz-Expires=300"));
|
||||
assert!(uri.contains("X-Amz-Signature="));
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ use crate::error::{AppError, ErrorCode};
|
||||
use crate::services::billing;
|
||||
use crate::services::compress;
|
||||
use crate::services::quota;
|
||||
use crate::services::storage;
|
||||
use crate::state::AppState;
|
||||
|
||||
use redis::streams::StreamReadOptions;
|
||||
@@ -148,17 +149,33 @@ struct TaskProcRow {
|
||||
api_key_id: Option<Uuid>,
|
||||
source: String,
|
||||
client_ip: Option<String>,
|
||||
retention_hours: i32,
|
||||
}
|
||||
|
||||
#[derive(Debug, FromRow)]
|
||||
struct TaskFileProcRow {
|
||||
id: Uuid,
|
||||
storage_path: Option<String>,
|
||||
input_path: Option<String>,
|
||||
original_format: String,
|
||||
output_format: String,
|
||||
status: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, FromRow)]
|
||||
struct CleanupFileRow {
|
||||
storage_backend: String,
|
||||
storage_endpoint_id: Option<Uuid>,
|
||||
storage_key: Option<String>,
|
||||
input_path: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, FromRow)]
|
||||
struct CleanupZipRow {
|
||||
zip_storage_backend: Option<String>,
|
||||
zip_storage_endpoint_id: Option<Uuid>,
|
||||
zip_storage_key: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TaskContext {
|
||||
api_key_id: Option<Uuid>,
|
||||
@@ -167,6 +184,7 @@ struct TaskContext {
|
||||
session_id: Option<String>,
|
||||
anon_ip: Option<IpAddr>,
|
||||
is_anonymous: bool,
|
||||
retention_hours: i32,
|
||||
}
|
||||
|
||||
async fn process_task(state: &AppState, task_id: Uuid) -> Result<(), AppError> {
|
||||
@@ -183,7 +201,8 @@ async fn process_task(state: &AppState, task_id: Uuid) -> Result<(), AppError> {
|
||||
session_id,
|
||||
api_key_id,
|
||||
source::text AS source,
|
||||
host(client_ip) AS client_ip
|
||||
host(client_ip) AS client_ip,
|
||||
retention_hours
|
||||
FROM tasks
|
||||
WHERE id = $1
|
||||
"#,
|
||||
@@ -241,7 +260,7 @@ async fn process_task(state: &AppState, task_id: Uuid) -> Result<(), AppError> {
|
||||
r#"
|
||||
SELECT
|
||||
id,
|
||||
storage_path,
|
||||
input_path,
|
||||
original_format,
|
||||
output_format,
|
||||
status::text AS status
|
||||
@@ -273,6 +292,7 @@ async fn process_task(state: &AppState, task_id: Uuid) -> Result<(), AppError> {
|
||||
session_id: task.session_id.clone(),
|
||||
anon_ip,
|
||||
is_anonymous: task.user_id.is_none(),
|
||||
retention_hours: task.retention_hours,
|
||||
};
|
||||
|
||||
let concurrency = state.config.worker_concurrency.max(1) as usize;
|
||||
@@ -391,7 +411,7 @@ async fn process_task_file(
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let Some(input_path) = file.storage_path.clone() else {
|
||||
let Some(input_path) = file.input_path.clone() else {
|
||||
mark_file_failed(&state, task_id, file.id, "原文件不存在").await?;
|
||||
return Ok(());
|
||||
};
|
||||
@@ -462,40 +482,50 @@ async fn process_task_file(
|
||||
&& max_height.is_none();
|
||||
let charge_units = !skip_charge && compressed_size < original_size;
|
||||
|
||||
let object_key = storage::result_key(
|
||||
ctx.retention_hours as i64,
|
||||
task_id,
|
||||
file.id,
|
||||
format_out.extension(),
|
||||
);
|
||||
let stored = match storage::store_bytes(
|
||||
&state,
|
||||
&object_key,
|
||||
compressed,
|
||||
format_out.content_type(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(err) => {
|
||||
reset_file_for_retry(&state, file.id, "对象存储暂时不可用").await?;
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
if ctx.is_anonymous && charge_units {
|
||||
let Some(session_id) = ctx.session_id.as_deref() else {
|
||||
let _ = storage::delete_object(&state, &stored_locator(&stored)).await;
|
||||
mark_file_failed(&state, task_id, file.id, "匿名任务缺少 session_id").await?;
|
||||
let _ = tokio::fs::remove_file(&input_path).await;
|
||||
return Ok(());
|
||||
};
|
||||
let Some(ip) = ctx.anon_ip else {
|
||||
let _ = storage::delete_object(&state, &stored_locator(&stored)).await;
|
||||
mark_file_failed(&state, task_id, file.id, "匿名任务缺少 client_ip").await?;
|
||||
let _ = tokio::fs::remove_file(&input_path).await;
|
||||
return Ok(());
|
||||
};
|
||||
if let Err(err) = quota::consume_anonymous_units(&state, session_id, ip, 1).await {
|
||||
let _ = storage::delete_object(&state, &stored_locator(&stored)).await;
|
||||
mark_file_failed(&state, task_id, file.id, &err.message).await?;
|
||||
let _ = tokio::fs::remove_file(&input_path).await;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
let output_path = format!(
|
||||
"{}/{}.{}",
|
||||
state.config.storage_path,
|
||||
file.id,
|
||||
format_out.extension()
|
||||
);
|
||||
if let Err(err) = tokio::fs::write(&output_path, &compressed).await {
|
||||
mark_file_failed(&state, task_id, file.id, "写入压缩文件失败").await?;
|
||||
let _ = tokio::fs::remove_file(&input_path).await;
|
||||
return Err(
|
||||
AppError::new(ErrorCode::StorageUnavailable, "写入压缩文件失败").with_source(err),
|
||||
);
|
||||
}
|
||||
|
||||
if is_task_cancelled(&state, task_id).await? {
|
||||
let _ = tokio::fs::remove_file(&output_path).await;
|
||||
let _ = storage::delete_object(&state, &stored_locator(&stored)).await;
|
||||
mark_file_failed(&state, task_id, file.id, "已取消").await?;
|
||||
let _ = tokio::fs::remove_file(&input_path).await;
|
||||
return Ok(());
|
||||
@@ -508,7 +538,7 @@ async fn process_task_file(
|
||||
&ctx.source,
|
||||
task_id,
|
||||
file.id,
|
||||
&output_path,
|
||||
&stored,
|
||||
original_size as i64,
|
||||
compressed_size as i64,
|
||||
saved_percent,
|
||||
@@ -518,12 +548,8 @@ async fn process_task_file(
|
||||
)
|
||||
.await
|
||||
{
|
||||
if err.code == ErrorCode::QuotaExceeded {
|
||||
let _ = tokio::fs::remove_file(&output_path).await;
|
||||
mark_file_failed(&state, task_id, file.id, &err.message).await?;
|
||||
} else {
|
||||
mark_file_failed(&state, task_id, file.id, &err.message).await?;
|
||||
}
|
||||
let _ = storage::delete_object(&state, &stored_locator(&stored)).await;
|
||||
mark_file_failed(&state, task_id, file.id, &err.message).await?;
|
||||
let _ = tokio::fs::remove_file(&input_path).await;
|
||||
return Ok(());
|
||||
}
|
||||
@@ -540,7 +566,7 @@ async fn finalize_file(
|
||||
source: &str,
|
||||
task_id: Uuid,
|
||||
task_file_id: Uuid,
|
||||
output_path: &str,
|
||||
stored: &storage::StoredObject,
|
||||
bytes_in: i64,
|
||||
bytes_out: i64,
|
||||
saved_percent: f64,
|
||||
@@ -577,15 +603,28 @@ async fn finalize_file(
|
||||
r#"
|
||||
UPDATE task_files
|
||||
SET storage_path = $2,
|
||||
compressed_size = $3,
|
||||
saved_percent = $4,
|
||||
storage_backend = $3,
|
||||
storage_endpoint_id = $4,
|
||||
storage_key = $5,
|
||||
storage_etag = $6,
|
||||
input_path = NULL,
|
||||
compressed_size = $7,
|
||||
saved_percent = $8,
|
||||
status = 'completed',
|
||||
completed_at = NOW()
|
||||
WHERE id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(task_file_id)
|
||||
.bind(output_path)
|
||||
.bind(if stored.backend == "local" {
|
||||
Some(stored.key.as_str())
|
||||
} else {
|
||||
None
|
||||
})
|
||||
.bind(&stored.backend)
|
||||
.bind(stored.endpoint_id)
|
||||
.bind(&stored.key)
|
||||
.bind(&stored.etag)
|
||||
.bind(bytes_out)
|
||||
.bind(saved_percent)
|
||||
.execute(&mut *tx)
|
||||
@@ -631,6 +670,7 @@ async fn mark_file_failed(
|
||||
SET status = 'failed',
|
||||
error_message = $2,
|
||||
storage_path = NULL,
|
||||
input_path = NULL,
|
||||
completed_at = NOW()
|
||||
WHERE id = $1
|
||||
AND status NOT IN ('completed', 'failed')
|
||||
@@ -663,6 +703,22 @@ async fn mark_file_failed(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn reset_file_for_retry(
|
||||
state: &AppState,
|
||||
task_file_id: Uuid,
|
||||
message: &str,
|
||||
) -> Result<(), AppError> {
|
||||
sqlx::query(
|
||||
"UPDATE task_files SET status = 'pending', error_message = $2 WHERE id = $1 AND status = 'processing'",
|
||||
)
|
||||
.bind(task_file_id)
|
||||
.bind(message)
|
||||
.execute(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "恢复待重试文件失败").with_source(err))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn finalize_task_status(state: &AppState, task_id: Uuid) -> Result<(), AppError> {
|
||||
let row: Option<(i32, i32, i32, String)> = sqlx::query_as(
|
||||
"SELECT total_files, completed_files, failed_files, status::text AS status FROM tasks WHERE id = $1",
|
||||
@@ -677,7 +733,7 @@ async fn finalize_task_status(state: &AppState, task_id: Uuid) -> Result<(), App
|
||||
};
|
||||
if status == "cancelled" {
|
||||
let paths: Vec<Option<String>> = sqlx::query_scalar(
|
||||
"SELECT storage_path FROM task_files WHERE task_id = $1 AND status IN ('pending','processing')",
|
||||
"SELECT input_path FROM task_files WHERE task_id = $1 AND status IN ('pending','processing')",
|
||||
)
|
||||
.bind(task_id)
|
||||
.fetch_all(&state.db)
|
||||
@@ -688,7 +744,7 @@ async fn finalize_task_status(state: &AppState, task_id: Uuid) -> Result<(), App
|
||||
}
|
||||
|
||||
let _ = sqlx::query(
|
||||
"UPDATE task_files SET status = 'failed', error_message = '已取消', storage_path = NULL, completed_at = NOW() WHERE task_id = $1 AND status IN ('pending','processing')",
|
||||
"UPDATE task_files SET status = 'failed', error_message = '已取消', storage_path = NULL, input_path = NULL, completed_at = NOW() WHERE task_id = $1 AND status IN ('pending','processing')",
|
||||
)
|
||||
.bind(task_id)
|
||||
.execute(&state.db)
|
||||
@@ -810,10 +866,47 @@ async fn charge_one_unit(
|
||||
|
||||
async fn maintenance(state: &AppState) -> Result<(), AppError> {
|
||||
cleanup_expired_tasks(state).await?;
|
||||
cleanup_stale_zip_temp(state).await?;
|
||||
cleanup_expired_records(state).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup_stale_zip_temp(state: &AppState) -> Result<(), AppError> {
|
||||
let root = std::path::Path::new(&state.config.storage_path).join("tmp/zips");
|
||||
let mut entries = match tokio::fs::read_dir(&root).await {
|
||||
Ok(entries) => entries,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(err) => {
|
||||
return Err(
|
||||
AppError::new(ErrorCode::StorageUnavailable, "读取 ZIP 临时目录失败")
|
||||
.with_source(err),
|
||||
)
|
||||
}
|
||||
};
|
||||
let cutoff = std::time::SystemTime::now()
|
||||
.checked_sub(std::time::Duration::from_secs(6 * 60 * 60))
|
||||
.unwrap_or(std::time::UNIX_EPOCH);
|
||||
|
||||
while let Some(entry) = entries.next_entry().await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "遍历 ZIP 临时目录失败").with_source(err)
|
||||
})? {
|
||||
let metadata = match entry.metadata().await {
|
||||
Ok(metadata) => metadata,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if !metadata.is_dir()
|
||||
|| metadata
|
||||
.modified()
|
||||
.map(|time| time >= cutoff)
|
||||
.unwrap_or(true)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let _ = tokio::fs::remove_dir_all(entry.path()).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup_expired_records(state: &AppState) -> Result<(), AppError> {
|
||||
let _ = sqlx::query("DELETE FROM idempotency_keys WHERE expires_at < NOW()")
|
||||
.execute(&state.db)
|
||||
@@ -834,6 +927,17 @@ async fn cleanup_expired_records(state: &AppState) -> Result<(), AppError> {
|
||||
.execute(&state.db)
|
||||
.await;
|
||||
|
||||
let _ = sqlx::query(
|
||||
r#"
|
||||
DELETE FROM storage_endpoints e
|
||||
WHERE e.deleted_at < NOW() - INTERVAL '30 days'
|
||||
AND NOT EXISTS (SELECT 1 FROM task_files f WHERE f.storage_endpoint_id = e.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM tasks t WHERE t.zip_storage_endpoint_id = e.id)
|
||||
"#,
|
||||
)
|
||||
.execute(&state.db)
|
||||
.await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -848,33 +952,87 @@ async fn cleanup_expired_tasks(state: &AppState) -> Result<(), AppError> {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if state.config.storage_type.eq_ignore_ascii_case("local") {
|
||||
for task_id in &task_ids {
|
||||
let paths: Vec<Option<String>> =
|
||||
sqlx::query_scalar("SELECT storage_path FROM task_files WHERE task_id = $1")
|
||||
.bind(task_id)
|
||||
.fetch_all(&state.db)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
for p in paths.into_iter().flatten() {
|
||||
let _ = tokio::fs::remove_file(p).await;
|
||||
}
|
||||
|
||||
let zip_path = format!("{}/zips/{task_id}.zip", state.config.storage_path);
|
||||
let _ = tokio::fs::remove_file(zip_path).await;
|
||||
|
||||
let orig_dir = format!("{}/orig/{task_id}", state.config.storage_path);
|
||||
let _ = tokio::fs::remove_dir_all(orig_dir).await;
|
||||
}
|
||||
}
|
||||
|
||||
for task_id in task_ids {
|
||||
let _ = sqlx::query("DELETE FROM tasks WHERE id = $1")
|
||||
.bind(task_id)
|
||||
.execute(&state.db)
|
||||
.await;
|
||||
if let Err(err) = cleanup_expired_task(state, task_id).await {
|
||||
tracing::warn!(task_id = %task_id, error = %err, "expired task cleanup deferred");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup_expired_task(state: &AppState, task_id: Uuid) -> Result<(), AppError> {
|
||||
let files: Vec<CleanupFileRow> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT storage_backend, storage_endpoint_id,
|
||||
COALESCE(storage_key, storage_path) AS storage_key,
|
||||
input_path
|
||||
FROM task_files
|
||||
WHERE task_id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(task_id)
|
||||
.fetch_all(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "查询过期任务文件失败").with_source(err))?;
|
||||
|
||||
for file in files {
|
||||
if let Some(key) = file.storage_key {
|
||||
storage::delete_object(
|
||||
state,
|
||||
&storage::ObjectLocator {
|
||||
backend: file.storage_backend,
|
||||
endpoint_id: file.storage_endpoint_id,
|
||||
key,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
if let Some(input_path) = file.input_path {
|
||||
let _ = tokio::fs::remove_file(input_path).await;
|
||||
}
|
||||
}
|
||||
|
||||
let zip: Option<CleanupZipRow> = sqlx::query_as(
|
||||
"SELECT zip_storage_backend, zip_storage_endpoint_id, zip_storage_key FROM tasks WHERE id = $1",
|
||||
)
|
||||
.bind(task_id)
|
||||
.fetch_optional(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "查询过期 ZIP 失败").with_source(err))?;
|
||||
if let Some(zip) = zip {
|
||||
if let (Some(backend), Some(key)) = (zip.zip_storage_backend, zip.zip_storage_key) {
|
||||
storage::delete_object(
|
||||
state,
|
||||
&storage::ObjectLocator {
|
||||
backend,
|
||||
endpoint_id: zip.zip_storage_endpoint_id,
|
||||
key,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
let legacy_zip_path = format!("{}/zips/{task_id}.zip", state.config.storage_path);
|
||||
let _ = tokio::fs::remove_file(legacy_zip_path).await;
|
||||
let orig_dir = format!("{}/orig/{task_id}", state.config.storage_path);
|
||||
let _ = tokio::fs::remove_dir_all(orig_dir).await;
|
||||
|
||||
sqlx::query("DELETE FROM tasks WHERE id = $1 AND expires_at < NOW()")
|
||||
.bind(task_id)
|
||||
.execute(&state.db)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
AppError::new(ErrorCode::Internal, "删除过期任务记录失败").with_source(err)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn stored_locator(stored: &storage::StoredObject) -> storage::ObjectLocator {
|
||||
storage::ObjectLocator {
|
||||
backend: stored.backend.clone(),
|
||||
endpoint_id: stored.endpoint_id,
|
||||
key: stored.key.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user