feat: add configurable S3 object storage
This commit is contained in:
237899745
@@ -40,7 +40,7 @@ pub fn router() -> Router<AppState> {
|
||||
.route("/admin/config", put(update_config))
|
||||
}
|
||||
|
||||
async fn require_admin(
|
||||
pub(super) async fn require_admin(
|
||||
state: &AppState,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
headers: &HeaderMap,
|
||||
|
||||
761
src/api/admin_storage.rs
Normal file
761
src/api/admin_storage.rs
Normal file
@@ -0,0 +1,761 @@
|
||||
use crate::api::admin::require_admin;
|
||||
use crate::api::context;
|
||||
use crate::api::envelope::Envelope;
|
||||
use crate::error::{AppError, ErrorCode};
|
||||
use crate::services::{settings, storage};
|
||||
use crate::state::AppState;
|
||||
|
||||
use axum::extract::{ConnectInfo, Path, State};
|
||||
use axum::http::HeaderMap;
|
||||
use axum::routing::{get, post, put};
|
||||
use axum::{Json, Router};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use url::Url;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub fn router() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route(
|
||||
"/admin/storage/endpoints",
|
||||
get(list_storage_endpoints).post(create_storage_endpoint),
|
||||
)
|
||||
.route(
|
||||
"/admin/storage/endpoints/{endpoint_id}",
|
||||
put(update_storage_endpoint).delete(delete_storage_endpoint),
|
||||
)
|
||||
.route(
|
||||
"/admin/storage/endpoints/{endpoint_id}/test",
|
||||
post(test_storage_endpoint),
|
||||
)
|
||||
.route(
|
||||
"/admin/storage/endpoints/{endpoint_id}/activate",
|
||||
post(activate_storage_endpoint),
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct StorageEndpointView {
|
||||
id: Uuid,
|
||||
name: String,
|
||||
internal_endpoint: String,
|
||||
public_endpoint: String,
|
||||
bucket: String,
|
||||
region: String,
|
||||
access_key_hint: String,
|
||||
credentials_configured: bool,
|
||||
force_path_style: bool,
|
||||
presign_ttl_seconds: i32,
|
||||
is_active: bool,
|
||||
last_test_at: Option<DateTime<Utc>>,
|
||||
last_test_ok: Option<bool>,
|
||||
last_test_error: Option<String>,
|
||||
object_count: i64,
|
||||
stored_bytes: i64,
|
||||
created_at: DateTime<Utc>,
|
||||
updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct StorageEndpointsResponse {
|
||||
active_backend: String,
|
||||
endpoints: Vec<StorageEndpointView>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CreateStorageEndpointRequest {
|
||||
name: String,
|
||||
internal_endpoint: String,
|
||||
public_endpoint: String,
|
||||
bucket: String,
|
||||
region: Option<String>,
|
||||
access_key: String,
|
||||
secret_key: String,
|
||||
force_path_style: Option<bool>,
|
||||
presign_ttl_seconds: Option<i32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct UpdateStorageEndpointRequest {
|
||||
name: Option<String>,
|
||||
internal_endpoint: Option<String>,
|
||||
public_endpoint: Option<String>,
|
||||
bucket: Option<String>,
|
||||
region: Option<String>,
|
||||
access_key: Option<String>,
|
||||
secret_key: Option<String>,
|
||||
force_path_style: Option<bool>,
|
||||
presign_ttl_seconds: Option<i32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct StorageActionResponse {
|
||||
message: String,
|
||||
endpoint: StorageEndpointView,
|
||||
}
|
||||
|
||||
async fn list_storage_endpoints(
|
||||
State(state): State<AppState>,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
ConnectInfo(addr): ConnectInfo<SocketAddr>,
|
||||
headers: HeaderMap,
|
||||
) -> Result<Json<Envelope<StorageEndpointsResponse>>, AppError> {
|
||||
let ip = context::client_ip(&headers, addr.ip());
|
||||
let (_jar, _admin_id) = require_admin(&state, jar, &headers, ip).await?;
|
||||
let endpoints = storage::list_endpoints(&state).await?;
|
||||
let active_backend = if endpoints.iter().any(|endpoint| endpoint.is_active) {
|
||||
"s3"
|
||||
} else {
|
||||
"local"
|
||||
};
|
||||
let mut views = Vec::with_capacity(endpoints.len());
|
||||
for endpoint in endpoints {
|
||||
views.push(endpoint_view(&state, endpoint).await?);
|
||||
}
|
||||
|
||||
Ok(Json(Envelope {
|
||||
success: true,
|
||||
data: StorageEndpointsResponse {
|
||||
active_backend: active_backend.to_string(),
|
||||
endpoints: views,
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
async fn create_storage_endpoint(
|
||||
State(state): State<AppState>,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
ConnectInfo(addr): ConnectInfo<SocketAddr>,
|
||||
headers: HeaderMap,
|
||||
Json(req): Json<CreateStorageEndpointRequest>,
|
||||
) -> Result<Json<Envelope<StorageEndpointView>>, AppError> {
|
||||
let ip = context::client_ip(&headers, addr.ip());
|
||||
let (_jar, admin_id) = require_admin(&state, jar, &headers, ip).await?;
|
||||
|
||||
let name = validate_name(&req.name)?;
|
||||
ensure_name_available(&state, &name, None).await?;
|
||||
let internal_endpoint = validate_endpoint_url(&req.internal_endpoint, "内部 Endpoint")?;
|
||||
let public_endpoint = validate_endpoint_url(&req.public_endpoint, "公网 Endpoint")?;
|
||||
let bucket = validate_bucket(&req.bucket)?;
|
||||
let region = validate_region(req.region.as_deref().unwrap_or("garage"))?;
|
||||
let access_key = validate_credential(&req.access_key, "Access Key")?;
|
||||
let secret_key = validate_credential(&req.secret_key, "Secret Key")?;
|
||||
let presign_ttl_seconds = validate_presign_ttl(req.presign_ttl_seconds.unwrap_or(300))?;
|
||||
|
||||
let endpoint = sqlx::query_as::<_, storage::StorageEndpoint>(
|
||||
r#"
|
||||
INSERT INTO storage_endpoints (
|
||||
name, internal_endpoint, public_endpoint, bucket, region,
|
||||
access_key_encrypted, secret_key_encrypted, access_key_hint,
|
||||
force_path_style, presign_ttl_seconds, updated_by
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||
RETURNING id, name, internal_endpoint, public_endpoint, bucket, region,
|
||||
access_key_encrypted, secret_key_encrypted, access_key_hint,
|
||||
force_path_style, presign_ttl_seconds, is_active,
|
||||
last_test_at, last_test_ok, last_test_error,
|
||||
created_at, updated_at, deleted_at
|
||||
"#,
|
||||
)
|
||||
.bind(&name)
|
||||
.bind(&internal_endpoint)
|
||||
.bind(&public_endpoint)
|
||||
.bind(&bucket)
|
||||
.bind(®ion)
|
||||
.bind(settings::encrypt_secret(&state, &access_key)?)
|
||||
.bind(settings::encrypt_secret(&state, &secret_key)?)
|
||||
.bind(access_key_hint(&access_key))
|
||||
.bind(req.force_path_style.unwrap_or(true))
|
||||
.bind(presign_ttl_seconds)
|
||||
.bind(admin_id)
|
||||
.fetch_one(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "创建存储端点失败").with_source(err))?;
|
||||
|
||||
audit_storage_action(
|
||||
&state,
|
||||
admin_id,
|
||||
ip,
|
||||
"storage_endpoint.create",
|
||||
endpoint.id,
|
||||
serde_json::json!({ "name": endpoint.name }),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(Json(Envelope {
|
||||
success: true,
|
||||
data: endpoint_view(&state, endpoint).await?,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn update_storage_endpoint(
|
||||
State(state): State<AppState>,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
ConnectInfo(addr): ConnectInfo<SocketAddr>,
|
||||
headers: HeaderMap,
|
||||
Path(endpoint_id): Path<Uuid>,
|
||||
Json(req): Json<UpdateStorageEndpointRequest>,
|
||||
) -> Result<Json<Envelope<StorageEndpointView>>, AppError> {
|
||||
let ip = context::client_ip(&headers, addr.ip());
|
||||
let (_jar, admin_id) = require_admin(&state, jar, &headers, ip).await?;
|
||||
let existing = storage::get_endpoint(&state, endpoint_id).await?;
|
||||
ensure_configurable(&existing)?;
|
||||
if existing.is_active {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"活动端点不能直接编辑，请先新增并启用替代端点",
|
||||
));
|
||||
}
|
||||
let (object_count, _) = endpoint_usage(&state, endpoint_id).await?;
|
||||
if object_count > 0 {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"端点仍有关联对象，不能修改；请新增替代端点并等待旧对象过期",
|
||||
));
|
||||
}
|
||||
|
||||
let name = match req.name.as_deref() {
|
||||
Some(value) => validate_name(value)?,
|
||||
None => existing.name.clone(),
|
||||
};
|
||||
ensure_name_available(&state, &name, Some(endpoint_id)).await?;
|
||||
let internal_endpoint = match req.internal_endpoint.as_deref() {
|
||||
Some(value) => validate_endpoint_url(value, "内部 Endpoint")?,
|
||||
None => existing.internal_endpoint.clone(),
|
||||
};
|
||||
let public_endpoint = match req.public_endpoint.as_deref() {
|
||||
Some(value) => validate_endpoint_url(value, "公网 Endpoint")?,
|
||||
None => existing.public_endpoint.clone(),
|
||||
};
|
||||
let bucket = match req.bucket.as_deref() {
|
||||
Some(value) => validate_bucket(value)?,
|
||||
None => existing.bucket.clone(),
|
||||
};
|
||||
let region = match req.region.as_deref() {
|
||||
Some(value) => validate_region(value)?,
|
||||
None => existing.region.clone(),
|
||||
};
|
||||
let access_key_encrypted = match req.access_key.as_deref() {
|
||||
Some(value) if !value.trim().is_empty() => Some(settings::encrypt_secret(
|
||||
&state,
|
||||
&validate_credential(value, "Access Key")?,
|
||||
)?),
|
||||
_ => None,
|
||||
};
|
||||
let secret_key_encrypted = match req.secret_key.as_deref() {
|
||||
Some(value) if !value.trim().is_empty() => Some(settings::encrypt_secret(
|
||||
&state,
|
||||
&validate_credential(value, "Secret Key")?,
|
||||
)?),
|
||||
_ => None,
|
||||
};
|
||||
let access_key_hint_value = req
|
||||
.access_key
|
||||
.as_deref()
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.map(access_key_hint);
|
||||
let presign_ttl_seconds = validate_presign_ttl(
|
||||
req.presign_ttl_seconds
|
||||
.unwrap_or(existing.presign_ttl_seconds),
|
||||
)?;
|
||||
|
||||
let endpoint = sqlx::query_as::<_, storage::StorageEndpoint>(
|
||||
r#"
|
||||
UPDATE storage_endpoints
|
||||
SET name = $2,
|
||||
internal_endpoint = $3,
|
||||
public_endpoint = $4,
|
||||
bucket = $5,
|
||||
region = $6,
|
||||
access_key_encrypted = COALESCE($7, access_key_encrypted),
|
||||
secret_key_encrypted = COALESCE($8, secret_key_encrypted),
|
||||
access_key_hint = COALESCE($9, access_key_hint),
|
||||
force_path_style = $10,
|
||||
presign_ttl_seconds = $11,
|
||||
is_active = false,
|
||||
last_test_at = NULL,
|
||||
last_test_ok = NULL,
|
||||
last_test_error = NULL,
|
||||
updated_at = NOW(),
|
||||
updated_by = $12
|
||||
WHERE id = $1
|
||||
RETURNING id, name, internal_endpoint, public_endpoint, bucket, region,
|
||||
access_key_encrypted, secret_key_encrypted, access_key_hint,
|
||||
force_path_style, presign_ttl_seconds, is_active,
|
||||
last_test_at, last_test_ok, last_test_error,
|
||||
created_at, updated_at, deleted_at
|
||||
"#,
|
||||
)
|
||||
.bind(endpoint_id)
|
||||
.bind(&name)
|
||||
.bind(&internal_endpoint)
|
||||
.bind(&public_endpoint)
|
||||
.bind(&bucket)
|
||||
.bind(®ion)
|
||||
.bind(access_key_encrypted)
|
||||
.bind(secret_key_encrypted)
|
||||
.bind(access_key_hint_value)
|
||||
.bind(req.force_path_style.unwrap_or(existing.force_path_style))
|
||||
.bind(presign_ttl_seconds)
|
||||
.bind(admin_id)
|
||||
.fetch_one(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "更新存储端点失败").with_source(err))?;
|
||||
|
||||
audit_storage_action(
|
||||
&state,
|
||||
admin_id,
|
||||
ip,
|
||||
"storage_endpoint.update",
|
||||
endpoint.id,
|
||||
serde_json::json!({ "name": endpoint.name, "requires_reactivation": true }),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(Json(Envelope {
|
||||
success: true,
|
||||
data: endpoint_view(&state, endpoint).await?,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn test_storage_endpoint(
|
||||
State(state): State<AppState>,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
ConnectInfo(addr): ConnectInfo<SocketAddr>,
|
||||
headers: HeaderMap,
|
||||
Path(endpoint_id): Path<Uuid>,
|
||||
) -> Result<Json<Envelope<StorageActionResponse>>, AppError> {
|
||||
let ip = context::client_ip(&headers, addr.ip());
|
||||
let (_jar, admin_id) = require_admin(&state, jar, &headers, ip).await?;
|
||||
let endpoint = storage::get_endpoint(&state, endpoint_id).await?;
|
||||
ensure_configurable(&endpoint)?;
|
||||
if let Err(err) = storage::test_endpoint(&state, &endpoint).await {
|
||||
let detail = storage_test_message(&err);
|
||||
record_test_result(&state, endpoint_id, false, Some(&detail), admin_id).await?;
|
||||
audit_storage_action(
|
||||
&state,
|
||||
admin_id,
|
||||
ip,
|
||||
"storage_endpoint.test_failed",
|
||||
endpoint_id,
|
||||
serde_json::json!({ "message": detail }),
|
||||
)
|
||||
.await;
|
||||
return Err(AppError::new(ErrorCode::StorageUnavailable, detail));
|
||||
}
|
||||
|
||||
record_test_result(&state, endpoint_id, true, None, admin_id).await?;
|
||||
let endpoint = storage::get_endpoint(&state, endpoint_id).await?;
|
||||
ensure_configurable(&endpoint)?;
|
||||
audit_storage_action(
|
||||
&state,
|
||||
admin_id,
|
||||
ip,
|
||||
"storage_endpoint.test_succeeded",
|
||||
endpoint_id,
|
||||
serde_json::json!({}),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(Json(Envelope {
|
||||
success: true,
|
||||
data: StorageActionResponse {
|
||||
message: "S3 内部读写删与公网签名下载测试通过".to_string(),
|
||||
endpoint: endpoint_view(&state, endpoint).await?,
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
async fn activate_storage_endpoint(
|
||||
State(state): State<AppState>,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
ConnectInfo(addr): ConnectInfo<SocketAddr>,
|
||||
headers: HeaderMap,
|
||||
Path(endpoint_id): Path<Uuid>,
|
||||
) -> Result<Json<Envelope<StorageActionResponse>>, AppError> {
|
||||
let ip = context::client_ip(&headers, addr.ip());
|
||||
let (_jar, admin_id) = require_admin(&state, jar, &headers, ip).await?;
|
||||
let endpoint = storage::get_endpoint(&state, endpoint_id).await?;
|
||||
ensure_configurable(&endpoint)?;
|
||||
let tested_config_updated_at = endpoint.updated_at;
|
||||
|
||||
if let Err(err) = storage::test_endpoint(&state, &endpoint).await {
|
||||
let detail = storage_test_message(&err);
|
||||
record_test_result(&state, endpoint_id, false, Some(&detail), admin_id).await?;
|
||||
return Err(AppError::new(ErrorCode::StorageUnavailable, detail));
|
||||
}
|
||||
|
||||
let mut tx = state.db.begin().await.map_err(|err| {
|
||||
AppError::new(ErrorCode::Internal, "开启存储切换事务失败").with_source(err)
|
||||
})?;
|
||||
sqlx::query(
|
||||
"UPDATE storage_endpoints SET is_active = false, updated_at = NOW() WHERE is_active = true",
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "停用旧存储端点失败").with_source(err))?;
|
||||
let activated = sqlx::query(
|
||||
r#"
|
||||
UPDATE storage_endpoints
|
||||
SET is_active = true,
|
||||
last_test_at = NOW(),
|
||||
last_test_ok = true,
|
||||
last_test_error = NULL,
|
||||
updated_at = NOW(),
|
||||
updated_by = $2
|
||||
WHERE id = $1 AND updated_at = $3 AND deleted_at IS NULL
|
||||
"#,
|
||||
)
|
||||
.bind(endpoint_id)
|
||||
.bind(admin_id)
|
||||
.bind(tested_config_updated_at)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "启用存储端点失败").with_source(err))?;
|
||||
if activated.rows_affected() == 0 {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"端点配置在测试期间发生变化，请重新测试",
|
||||
));
|
||||
}
|
||||
tx.commit().await.map_err(|err| {
|
||||
AppError::new(ErrorCode::Internal, "提交存储切换事务失败").with_source(err)
|
||||
})?;
|
||||
|
||||
let endpoint = storage::get_endpoint(&state, endpoint_id).await?;
|
||||
audit_storage_action(
|
||||
&state,
|
||||
admin_id,
|
||||
ip,
|
||||
"storage_endpoint.activate",
|
||||
endpoint_id,
|
||||
serde_json::json!({ "name": endpoint.name }),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(Json(Envelope {
|
||||
success: true,
|
||||
data: StorageActionResponse {
|
||||
message: "S3 端点已启用，新生成文件将写入该端点".to_string(),
|
||||
endpoint: endpoint_view(&state, endpoint).await?,
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
async fn delete_storage_endpoint(
|
||||
State(state): State<AppState>,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
ConnectInfo(addr): ConnectInfo<SocketAddr>,
|
||||
headers: HeaderMap,
|
||||
Path(endpoint_id): Path<Uuid>,
|
||||
) -> Result<Json<Envelope<serde_json::Value>>, AppError> {
|
||||
let ip = context::client_ip(&headers, addr.ip());
|
||||
let (_jar, admin_id) = require_admin(&state, jar, &headers, ip).await?;
|
||||
let endpoint = storage::get_endpoint(&state, endpoint_id).await?;
|
||||
ensure_configurable(&endpoint)?;
|
||||
if endpoint.is_active {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"活动端点不能删除，请先启用其他端点",
|
||||
));
|
||||
}
|
||||
|
||||
let (object_count, _): (i64, i64) = endpoint_usage(&state, endpoint_id).await?;
|
||||
if object_count > 0 {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
format!("端点仍关联 {object_count} 个对象，需等待对象过期清理后再删除"),
|
||||
));
|
||||
}
|
||||
|
||||
sqlx::query(
|
||||
"UPDATE storage_endpoints SET is_active = false, deleted_at = NOW(), updated_at = NOW(), updated_by = $2 WHERE id = $1",
|
||||
)
|
||||
.bind(endpoint_id)
|
||||
.bind(admin_id)
|
||||
.execute(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "归档存储端点失败").with_source(err))?;
|
||||
audit_storage_action(
|
||||
&state,
|
||||
admin_id,
|
||||
ip,
|
||||
"storage_endpoint.delete",
|
||||
endpoint_id,
|
||||
serde_json::json!({ "name": endpoint.name }),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(Json(Envelope {
|
||||
success: true,
|
||||
data: serde_json::json!({ "message": "存储端点已删除" }),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn endpoint_view(
|
||||
state: &AppState,
|
||||
endpoint: storage::StorageEndpoint,
|
||||
) -> Result<StorageEndpointView, AppError> {
|
||||
let (object_count, stored_bytes) = endpoint_usage(state, endpoint.id).await?;
|
||||
Ok(StorageEndpointView {
|
||||
id: endpoint.id,
|
||||
name: endpoint.name,
|
||||
internal_endpoint: endpoint.internal_endpoint,
|
||||
public_endpoint: endpoint.public_endpoint,
|
||||
bucket: endpoint.bucket,
|
||||
region: endpoint.region,
|
||||
access_key_hint: endpoint.access_key_hint,
|
||||
credentials_configured: !endpoint.access_key_encrypted.is_empty()
|
||||
&& !endpoint.secret_key_encrypted.is_empty(),
|
||||
force_path_style: endpoint.force_path_style,
|
||||
presign_ttl_seconds: endpoint.presign_ttl_seconds,
|
||||
is_active: endpoint.is_active,
|
||||
last_test_at: endpoint.last_test_at,
|
||||
last_test_ok: endpoint.last_test_ok,
|
||||
last_test_error: endpoint.last_test_error,
|
||||
object_count,
|
||||
stored_bytes,
|
||||
created_at: endpoint.created_at,
|
||||
updated_at: endpoint.updated_at,
|
||||
})
|
||||
}
|
||||
|
||||
async fn endpoint_usage(state: &AppState, endpoint_id: Uuid) -> Result<(i64, i64), AppError> {
|
||||
sqlx::query_as::<_, (i64, i64)>(
|
||||
r#"
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM task_files WHERE storage_endpoint_id = $1)
|
||||
+ (SELECT COUNT(*) FROM tasks WHERE zip_storage_endpoint_id = $1) AS object_count,
|
||||
COALESCE((SELECT SUM(compressed_size)::BIGINT FROM task_files WHERE storage_endpoint_id = $1), 0::BIGINT)
|
||||
+ COALESCE((SELECT SUM(zip_size)::BIGINT FROM tasks WHERE zip_storage_endpoint_id = $1), 0::BIGINT) AS stored_bytes
|
||||
"#,
|
||||
)
|
||||
.bind(endpoint_id)
|
||||
.fetch_one(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "统计存储使用量失败").with_source(err))
|
||||
}
|
||||
|
||||
async fn record_test_result(
|
||||
state: &AppState,
|
||||
endpoint_id: Uuid,
|
||||
ok: bool,
|
||||
error: Option<&str>,
|
||||
admin_id: Uuid,
|
||||
) -> Result<(), AppError> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE storage_endpoints
|
||||
SET last_test_at = NOW(),
|
||||
last_test_ok = $2,
|
||||
last_test_error = $3,
|
||||
updated_at = NOW(),
|
||||
updated_by = $4
|
||||
WHERE id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(endpoint_id)
|
||||
.bind(ok)
|
||||
.bind(error)
|
||||
.bind(admin_id)
|
||||
.execute(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "记录存储测试结果失败").with_source(err))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn ensure_name_available(
|
||||
state: &AppState,
|
||||
name: &str,
|
||||
except_id: Option<Uuid>,
|
||||
) -> Result<(), AppError> {
|
||||
let exists: bool = sqlx::query_scalar(
|
||||
"SELECT EXISTS(SELECT 1 FROM storage_endpoints WHERE deleted_at IS NULL AND lower(name) = lower($1) AND ($2::uuid IS NULL OR id <> $2))",
|
||||
)
|
||||
.bind(name)
|
||||
.bind(except_id)
|
||||
.fetch_one(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "校验存储端点名称失败").with_source(err))?;
|
||||
if exists {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"存储端点名称已存在",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_name(value: &str) -> Result<String, AppError> {
|
||||
let value = value.trim();
|
||||
if value.is_empty() || value.chars().count() > 100 {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"端点名称长度必须为 1-100 个字符",
|
||||
));
|
||||
}
|
||||
Ok(value.to_string())
|
||||
}
|
||||
|
||||
fn ensure_configurable(endpoint: &storage::StorageEndpoint) -> Result<(), AppError> {
|
||||
if endpoint.deleted_at.is_some() {
|
||||
return Err(AppError::new(ErrorCode::NotFound, "存储端点不存在"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_endpoint_url(value: &str, label: &str) -> Result<String, AppError> {
|
||||
let value = value.trim().trim_end_matches('/');
|
||||
let parsed = Url::parse(value).map_err(|err| {
|
||||
AppError::new(ErrorCode::InvalidRequest, format!("{label} 格式错误")).with_source(err)
|
||||
})?;
|
||||
if !matches!(parsed.scheme(), "http" | "https")
|
||||
|| !parsed.username().is_empty()
|
||||
|| parsed.password().is_some()
|
||||
|| parsed.query().is_some()
|
||||
|| parsed.fragment().is_some()
|
||||
|| parsed.host_str().is_none()
|
||||
{
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
format!("{label} 必须是无凭据、无查询参数的 HTTP(S) 地址"),
|
||||
));
|
||||
}
|
||||
Ok(value.to_string())
|
||||
}
|
||||
|
||||
fn validate_bucket(value: &str) -> Result<String, AppError> {
|
||||
let value = value.trim();
|
||||
let valid = (3..=63).contains(&value.len())
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || matches!(ch, b'.' | b'-'))
|
||||
&& value
|
||||
.as_bytes()
|
||||
.first()
|
||||
.is_some_and(u8::is_ascii_alphanumeric)
|
||||
&& value
|
||||
.as_bytes()
|
||||
.last()
|
||||
.is_some_and(u8::is_ascii_alphanumeric)
|
||||
&& !value.contains("..")
|
||||
&& !value.contains(".-")
|
||||
&& !value.contains("-.");
|
||||
if !valid {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"Bucket 名称不符合 S3 命名规则",
|
||||
));
|
||||
}
|
||||
Ok(value.to_string())
|
||||
}
|
||||
|
||||
fn validate_region(value: &str) -> Result<String, AppError> {
|
||||
let value = value.trim();
|
||||
if value.is_empty()
|
||||
|| value.len() > 100
|
||||
|| !value
|
||||
.bytes()
|
||||
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, b'-' | b'_'))
|
||||
{
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"Region 格式不正确",
|
||||
));
|
||||
}
|
||||
Ok(value.to_string())
|
||||
}
|
||||
|
||||
fn validate_credential(value: &str, label: &str) -> Result<String, AppError> {
|
||||
let value = value.trim();
|
||||
if value.len() < 3 || value.len() > 512 || value.chars().any(char::is_control) {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
format!("{label} 长度或格式不正确"),
|
||||
));
|
||||
}
|
||||
Ok(value.to_string())
|
||||
}
|
||||
|
||||
fn validate_presign_ttl(value: i32) -> Result<i32, AppError> {
|
||||
if !(60..=3600).contains(&value) {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::InvalidRequest,
|
||||
"签名有效期必须为 60-3600 秒",
|
||||
));
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
fn access_key_hint(value: &str) -> String {
|
||||
let chars = value.chars().collect::<Vec<_>>();
|
||||
if chars.len() <= 8 {
|
||||
return "********".to_string();
|
||||
}
|
||||
let start = chars.iter().take(4).collect::<String>();
|
||||
let end = chars.iter().rev().take(4).rev().collect::<String>();
|
||||
format!("{start}...{end}")
|
||||
}
|
||||
|
||||
fn storage_test_message(error: &AppError) -> String {
|
||||
let Some(source) = error.source.as_deref() else {
|
||||
return error.message.clone();
|
||||
};
|
||||
let source = source.replace(['\r', '\n'], " ");
|
||||
let mut detail = format!("{}：{}", error.message, source);
|
||||
if detail.len() > 500 {
|
||||
let mut end = 500;
|
||||
while !detail.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
detail.truncate(end);
|
||||
}
|
||||
detail
|
||||
}
|
||||
|
||||
async fn audit_storage_action(
|
||||
state: &AppState,
|
||||
admin_id: Uuid,
|
||||
ip: IpAddr,
|
||||
action: &str,
|
||||
endpoint_id: Uuid,
|
||||
details: serde_json::Value,
|
||||
) {
|
||||
let _ = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO audit_logs (user_id, action, resource_type, resource_id, details, ip_address)
|
||||
VALUES ($1, $2, 'storage_endpoint', $3, $4, $5::inet)
|
||||
"#,
|
||||
)
|
||||
.bind(admin_id)
|
||||
.bind(action)
|
||||
.bind(endpoint_id)
|
||||
.bind(details)
|
||||
.bind(ip.to_string())
|
||||
.execute(&state.db)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn endpoint_url_rejects_embedded_credentials() {
|
||||
assert!(validate_endpoint_url("https://user:pass@example.com", "Endpoint").is_err());
|
||||
assert!(validate_endpoint_url("https://files.example.com/", "Endpoint").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_validation_matches_s3_basics() {
|
||||
assert!(validate_bucket("imageforge-results").is_ok());
|
||||
assert!(validate_bucket("Bad_Bucket").is_err());
|
||||
assert!(validate_bucket("ab").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credentials_are_masked() {
|
||||
assert_eq!(access_key_hint("ABCD12345678WXYZ"), "ABCD...WXYZ");
|
||||
assert_eq!(access_key_hint("short"), "********");
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ use crate::services::compress;
|
||||
use crate::services::compress::{CompressionLevel, ImageFmt};
|
||||
use crate::services::idempotency;
|
||||
use crate::services::quota;
|
||||
use crate::services::storage;
|
||||
use crate::state::AppState;
|
||||
|
||||
use axum::extract::{ConnectInfo, Multipart, State};
|
||||
@@ -19,7 +20,6 @@ use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use sqlx::FromRow;
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub fn router() -> Router<AppState> {
|
||||
@@ -300,35 +300,31 @@ async fn compress_json(
|
||||
&& req.max_height.is_none();
|
||||
let charge_units = !skip_charge && compressed_size < original_size;
|
||||
|
||||
if charge_units {
|
||||
if let QuotaContext::Anonymous { session_id, ip } = "a_ctx {
|
||||
quota::consume_anonymous_units(&state, session_id, *ip, 1).await?;
|
||||
}
|
||||
}
|
||||
|
||||
if !state.config.storage_type.eq_ignore_ascii_case("local") {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::StorageUnavailable,
|
||||
"当前仅支持本地存储（STORAGE_TYPE=local）",
|
||||
));
|
||||
}
|
||||
|
||||
tokio::fs::create_dir_all(&state.config.storage_path)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "创建存储目录失败").with_source(err)
|
||||
})?;
|
||||
|
||||
let task_id = Uuid::new_v4();
|
||||
let file_id = Uuid::new_v4();
|
||||
let file_path = format!(
|
||||
"{}/{}.{}",
|
||||
state.config.storage_path,
|
||||
file_id,
|
||||
format_out.extension()
|
||||
);
|
||||
let retention_hours = retention.num_hours();
|
||||
let object_key =
|
||||
storage::result_key(retention_hours, task_id, file_id, format_out.extension());
|
||||
let stored =
|
||||
storage::store_bytes(&state, &object_key, compressed, format_out.content_type())
|
||||
.await?;
|
||||
|
||||
write_file(&file_path, &compressed).await?;
|
||||
if charge_units {
|
||||
if let QuotaContext::Anonymous { session_id, ip } = "a_ctx {
|
||||
if let Err(err) = quota::consume_anonymous_units(&state, session_id, *ip, 1).await {
|
||||
let _ = storage::delete_object(
|
||||
&state,
|
||||
&storage::ObjectLocator {
|
||||
backend: stored.backend.clone(),
|
||||
endpoint_id: stored.endpoint_id,
|
||||
key: stored.key.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let expires_at = Utc::now() + retention;
|
||||
|
||||
@@ -338,7 +334,7 @@ async fn compress_json(
|
||||
ip,
|
||||
task_id,
|
||||
file_id,
|
||||
&file_path,
|
||||
&stored,
|
||||
&req.file_name,
|
||||
req.max_width,
|
||||
req.max_height,
|
||||
@@ -350,12 +346,21 @@ async fn compress_json(
|
||||
compressed_size,
|
||||
saved_percent,
|
||||
expires_at,
|
||||
retention_hours,
|
||||
"a_ctx,
|
||||
charge_units,
|
||||
)
|
||||
.await
|
||||
{
|
||||
let _ = tokio::fs::remove_file(&file_path).await;
|
||||
let _ = storage::delete_object(
|
||||
&state,
|
||||
&storage::ObjectLocator {
|
||||
backend: stored.backend.clone(),
|
||||
endpoint_id: stored.endpoint_id,
|
||||
key: stored.key.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
@@ -671,27 +676,18 @@ async fn compress_direct(
|
||||
&& req.max_height.is_none();
|
||||
let charge_units = !skip_charge && compressed_size < original_size;
|
||||
|
||||
if !state.config.storage_type.eq_ignore_ascii_case("local") {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::StorageUnavailable,
|
||||
"当前仅支持本地存储（STORAGE_TYPE=local）",
|
||||
));
|
||||
}
|
||||
tokio::fs::create_dir_all(&state.config.storage_path)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "创建存储目录失败").with_source(err)
|
||||
})?;
|
||||
|
||||
let task_id = Uuid::new_v4();
|
||||
let file_id = Uuid::new_v4();
|
||||
let file_path = format!(
|
||||
"{}/{}.{}",
|
||||
state.config.storage_path,
|
||||
file_id,
|
||||
format_out.extension()
|
||||
);
|
||||
write_file(&file_path, &compressed).await?;
|
||||
let retention_hours = retention.num_hours();
|
||||
let object_key =
|
||||
storage::result_key(retention_hours, task_id, file_id, format_out.extension());
|
||||
let stored = storage::store_bytes(
|
||||
&state,
|
||||
&object_key,
|
||||
compressed.clone(),
|
||||
format_out.content_type(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let expires_at = Utc::now() + retention;
|
||||
|
||||
@@ -701,7 +697,7 @@ async fn compress_direct(
|
||||
ip,
|
||||
task_id,
|
||||
file_id,
|
||||
&file_path,
|
||||
&stored,
|
||||
&req.file_name,
|
||||
req.max_width,
|
||||
req.max_height,
|
||||
@@ -713,12 +709,21 @@ async fn compress_direct(
|
||||
compressed_size,
|
||||
saved_percent,
|
||||
expires_at,
|
||||
retention_hours,
|
||||
"a_ctx,
|
||||
charge_units,
|
||||
)
|
||||
.await
|
||||
{
|
||||
let _ = tokio::fs::remove_file(&file_path).await;
|
||||
let _ = storage::delete_object(
|
||||
&state,
|
||||
&storage::ObjectLocator {
|
||||
backend: stored.backend.clone(),
|
||||
endpoint_id: stored.endpoint_id,
|
||||
key: stored.key.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
@@ -800,19 +805,11 @@ async fn compress_direct(
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_file(path: &str, bytes: &[u8]) -> Result<(), AppError> {
|
||||
let mut file = tokio::fs::File::create(path).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "写入文件失败").with_source(err)
|
||||
})?;
|
||||
file.write_all(bytes).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "写入文件失败").with_source(err)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, FromRow)]
|
||||
struct DirectReplayRow {
|
||||
storage_path: Option<String>,
|
||||
storage_backend: String,
|
||||
storage_endpoint_id: Option<Uuid>,
|
||||
storage_key: Option<String>,
|
||||
output_format: String,
|
||||
file_status: String,
|
||||
expires_at: DateTime<Utc>,
|
||||
@@ -828,7 +825,9 @@ async fn load_direct_replay_bytes(
|
||||
sqlx::query_as::<_, DirectReplayRow>(
|
||||
r#"
|
||||
SELECT
|
||||
f.storage_path,
|
||||
f.storage_backend,
|
||||
f.storage_endpoint_id,
|
||||
COALESCE(f.storage_key, f.storage_path) AS storage_key,
|
||||
f.output_format,
|
||||
f.status::text AS file_status,
|
||||
t.expires_at
|
||||
@@ -846,7 +845,9 @@ async fn load_direct_replay_bytes(
|
||||
sqlx::query_as::<_, DirectReplayRow>(
|
||||
r#"
|
||||
SELECT
|
||||
f.storage_path,
|
||||
f.storage_backend,
|
||||
f.storage_endpoint_id,
|
||||
COALESCE(f.storage_key, f.storage_path) AS storage_key,
|
||||
f.output_format,
|
||||
f.status::text AS file_status,
|
||||
t.expires_at
|
||||
@@ -873,13 +874,18 @@ async fn load_direct_replay_bytes(
|
||||
if row.file_status != "completed" {
|
||||
return Err(AppError::new(ErrorCode::NotFound, "文件不存在"));
|
||||
}
|
||||
let Some(path) = row.storage_path else {
|
||||
let Some(key) = row.storage_key else {
|
||||
return Err(AppError::new(ErrorCode::NotFound, "文件不存在"));
|
||||
};
|
||||
|
||||
let bytes = tokio::fs::read(&path).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "读取文件失败").with_source(err)
|
||||
})?;
|
||||
let bytes = storage::read_bytes(
|
||||
state,
|
||||
&storage::ObjectLocator {
|
||||
backend: row.storage_backend,
|
||||
endpoint_id: row.storage_endpoint_id,
|
||||
key,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let fmt = compress::parse_output_format(&row.output_format)?;
|
||||
Ok((bytes, fmt))
|
||||
@@ -1082,7 +1088,7 @@ async fn record_task_and_metering(
|
||||
client_ip: IpAddr,
|
||||
task_id: Uuid,
|
||||
file_id: Uuid,
|
||||
file_path: &str,
|
||||
stored: &storage::StoredObject,
|
||||
original_name: &str,
|
||||
max_width: Option<u32>,
|
||||
max_height: Option<u32>,
|
||||
@@ -1094,6 +1100,7 @@ async fn record_task_and_metering(
|
||||
compressed_size: u64,
|
||||
saved_percent: f64,
|
||||
expires_at: DateTime<Utc>,
|
||||
retention_hours: i64,
|
||||
quota_ctx: &QuotaContext,
|
||||
charge_units: bool,
|
||||
) -> Result<(), AppError> {
|
||||
@@ -1123,13 +1130,13 @@ async fn record_task_and_metering(
|
||||
compression_rate,
|
||||
total_files, completed_files, failed_files,
|
||||
total_original_size, total_compressed_size,
|
||||
started_at, completed_at, expires_at
|
||||
started_at, completed_at, expires_at, retention_hours
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5::inet, $6::task_source, 'completed',
|
||||
$7::compression_level, $8, $9, $10, $11, $12,
|
||||
1, 1, 0,
|
||||
$13, $14,
|
||||
NOW(), NOW(), $15
|
||||
NOW(), NOW(), $15, $16
|
||||
)
|
||||
"#,
|
||||
)
|
||||
@@ -1148,6 +1155,7 @@ async fn record_task_and_metering(
|
||||
.bind(original_size as i64)
|
||||
.bind(compressed_size as i64)
|
||||
.bind(expires_at)
|
||||
.bind(retention_hours as i32)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "创建任务失败").with_source(err))?;
|
||||
@@ -1158,12 +1166,13 @@ async fn record_task_and_metering(
|
||||
id, task_id,
|
||||
original_name, original_format, output_format,
|
||||
original_size, compressed_size, saved_percent,
|
||||
storage_path, status, completed_at
|
||||
storage_path, storage_backend, storage_endpoint_id, storage_key, storage_etag,
|
||||
status, completed_at
|
||||
) VALUES (
|
||||
$1, $2,
|
||||
$3, $4, $5,
|
||||
$6, $7, $8,
|
||||
$9, 'completed', NOW()
|
||||
$9, $10, $11, $12, $13, 'completed', NOW()
|
||||
)
|
||||
"#,
|
||||
)
|
||||
@@ -1175,7 +1184,15 @@ async fn record_task_and_metering(
|
||||
.bind(original_size as i64)
|
||||
.bind(compressed_size as i64)
|
||||
.bind(saved_percent)
|
||||
.bind(file_path)
|
||||
.bind(if stored.backend == "local" {
|
||||
Some(stored.key.as_str())
|
||||
} else {
|
||||
None
|
||||
})
|
||||
.bind(&stored.backend)
|
||||
.bind(stored.endpoint_id)
|
||||
.bind(&stored.key)
|
||||
.bind(&stored.etag)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "创建文件记录失败").with_source(err))?;
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
use crate::api::context;
|
||||
use crate::error::{AppError, ErrorCode};
|
||||
use crate::services::storage;
|
||||
use crate::state::AppState;
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::extract::{ConnectInfo, Path, State};
|
||||
use axum::http::{header, HeaderMap};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::response::{IntoResponse, Redirect, Response};
|
||||
use axum::routing::get;
|
||||
use axum::Router;
|
||||
use chrono::{DateTime, Utc};
|
||||
@@ -25,7 +26,9 @@ pub fn router() -> Router<AppState> {
|
||||
|
||||
#[derive(Debug, FromRow)]
|
||||
struct DownloadRow {
|
||||
storage_path: Option<String>,
|
||||
storage_backend: String,
|
||||
storage_endpoint_id: Option<Uuid>,
|
||||
storage_key: Option<String>,
|
||||
output_format: String,
|
||||
original_name: String,
|
||||
file_status: String,
|
||||
@@ -47,7 +50,9 @@ async fn download_file(
|
||||
let row = sqlx::query_as::<_, DownloadRow>(
|
||||
r#"
|
||||
SELECT
|
||||
f.storage_path,
|
||||
f.storage_backend,
|
||||
f.storage_endpoint_id,
|
||||
COALESCE(f.storage_key, f.storage_path) AS storage_key,
|
||||
f.output_format,
|
||||
f.original_name,
|
||||
f.status::text AS file_status,
|
||||
@@ -75,32 +80,51 @@ async fn download_file(
|
||||
|
||||
authorize_download(&principal, &row)?;
|
||||
|
||||
let Some(path) = &row.storage_path else {
|
||||
let Some(key) = row.storage_key else {
|
||||
return Err(AppError::new(ErrorCode::NotFound, "文件不存在"));
|
||||
};
|
||||
let object = storage::ObjectLocator {
|
||||
backend: row.storage_backend,
|
||||
endpoint_id: row.storage_endpoint_id,
|
||||
key,
|
||||
};
|
||||
respond_object(
|
||||
&state,
|
||||
jar,
|
||||
&object,
|
||||
&output_file_name(&row.original_name, &row.output_format),
|
||||
content_type(&row.output_format),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
let file = tokio::fs::File::open(path).await.map_err(|err| {
|
||||
async fn respond_object(
|
||||
state: &AppState,
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
object: &storage::ObjectLocator,
|
||||
file_name: &str,
|
||||
content_type_value: &str,
|
||||
) -> Result<(axum_extra::extract::cookie::CookieJar, Response), AppError> {
|
||||
if let Some(url) =
|
||||
storage::presign_download(state, object, file_name, content_type_value).await?
|
||||
{
|
||||
return Ok((jar, Redirect::temporary(&url).into_response()));
|
||||
}
|
||||
|
||||
let file = tokio::fs::File::open(&object.key).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "读取文件失败").with_source(err)
|
||||
})?;
|
||||
let content_length = file.metadata().await.ok().map(|metadata| metadata.len());
|
||||
let body = Body::from_stream(ReaderStream::new(file));
|
||||
|
||||
let mut resp_headers = HeaderMap::new();
|
||||
resp_headers.insert(
|
||||
header::CONTENT_TYPE,
|
||||
content_type(&row.output_format).parse().unwrap(),
|
||||
);
|
||||
resp_headers.insert(
|
||||
header::CONTENT_DISPOSITION,
|
||||
content_disposition(&row.original_name)?,
|
||||
);
|
||||
resp_headers.insert(header::CONTENT_TYPE, content_type_value.parse().unwrap());
|
||||
resp_headers.insert(header::CONTENT_DISPOSITION, content_disposition(file_name)?);
|
||||
if let Some(content_length) = content_length {
|
||||
resp_headers.insert(
|
||||
header::CONTENT_LENGTH,
|
||||
content_length.to_string().parse().unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
Ok((jar, (resp_headers, body).into_response()))
|
||||
}
|
||||
|
||||
@@ -126,10 +150,36 @@ fn content_type(format: &str) -> &'static str {
|
||||
"jpeg" | "jpg" => "image/jpeg",
|
||||
"webp" => "image/webp",
|
||||
"avif" => "image/avif",
|
||||
"gif" => "image/gif",
|
||||
"bmp" => "image/bmp",
|
||||
"tif" | "tiff" => "image/tiff",
|
||||
"ico" => "image/x-icon",
|
||||
_ => "application/octet-stream",
|
||||
}
|
||||
}
|
||||
|
||||
fn output_file_name(original_name: &str, output_format: &str) -> String {
|
||||
let sanitized = sanitize_filename(original_name);
|
||||
let base = sanitized
|
||||
.rsplit_once('.')
|
||||
.map(|(value, _)| value)
|
||||
.unwrap_or(&sanitized)
|
||||
.trim_end_matches('.');
|
||||
let base = if base.is_empty() { "download" } else { base };
|
||||
let extension = match output_format.trim().to_ascii_lowercase().as_str() {
|
||||
"jpeg" | "jpg" => "jpg",
|
||||
"png" => "png",
|
||||
"webp" => "webp",
|
||||
"avif" => "avif",
|
||||
"gif" => "gif",
|
||||
"bmp" => "bmp",
|
||||
"tif" | "tiff" => "tiff",
|
||||
"ico" => "ico",
|
||||
_ => "bin",
|
||||
};
|
||||
format!("{base}.{extension}")
|
||||
}
|
||||
|
||||
fn sanitize_filename(name: &str) -> String {
|
||||
let mut out = name.trim().to_string();
|
||||
if out.is_empty() {
|
||||
@@ -177,11 +227,17 @@ struct TaskZipRow {
|
||||
status: String,
|
||||
completed_at: Option<DateTime<Utc>>,
|
||||
expires_at: DateTime<Utc>,
|
||||
retention_hours: i32,
|
||||
zip_storage_backend: Option<String>,
|
||||
zip_storage_endpoint_id: Option<Uuid>,
|
||||
zip_storage_key: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, FromRow)]
|
||||
struct TaskZipFileRow {
|
||||
storage_path: Option<String>,
|
||||
storage_backend: String,
|
||||
storage_endpoint_id: Option<Uuid>,
|
||||
storage_key: Option<String>,
|
||||
original_name: String,
|
||||
output_format: String,
|
||||
}
|
||||
@@ -203,7 +259,11 @@ async fn download_task_zip(
|
||||
session_id,
|
||||
status::text AS status,
|
||||
completed_at,
|
||||
expires_at
|
||||
expires_at,
|
||||
retention_hours,
|
||||
zip_storage_backend,
|
||||
zip_storage_endpoint_id,
|
||||
zip_storage_key
|
||||
FROM tasks
|
||||
WHERE id = $1
|
||||
"#,
|
||||
@@ -235,26 +295,29 @@ async fn download_task_zip(
|
||||
}
|
||||
}
|
||||
|
||||
if !state.config.storage_type.eq_ignore_ascii_case("local") {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::StorageUnavailable,
|
||||
"当前仅支持本地存储（STORAGE_TYPE=local）",
|
||||
));
|
||||
}
|
||||
|
||||
let zip_dir = format!("{}/zips", state.config.storage_path);
|
||||
tokio::fs::create_dir_all(&zip_dir).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "创建存储目录失败").with_source(err)
|
||||
})?;
|
||||
let zip_path = PathBuf::from(format!("{zip_dir}/{task_id}.zip"));
|
||||
|
||||
if tokio::fs::try_exists(&zip_path).await.unwrap_or(false) {
|
||||
return stream_zip(jar, zip_path, task_id).await;
|
||||
if let (Some(backend), Some(key)) = (
|
||||
task.zip_storage_backend.clone(),
|
||||
task.zip_storage_key.clone(),
|
||||
) {
|
||||
return respond_object(
|
||||
&state,
|
||||
jar,
|
||||
&storage::ObjectLocator {
|
||||
backend,
|
||||
endpoint_id: task.zip_storage_endpoint_id,
|
||||
key,
|
||||
},
|
||||
&format!("task_{task_id}.zip"),
|
||||
"application/zip",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let rows = sqlx::query_as::<_, TaskZipFileRow>(
|
||||
r#"
|
||||
SELECT storage_path, original_name, output_format
|
||||
SELECT storage_backend, storage_endpoint_id,
|
||||
COALESCE(storage_key, storage_path) AS storage_key,
|
||||
original_name, output_format
|
||||
FROM task_files
|
||||
WHERE task_id = $1 AND status = 'completed'
|
||||
ORDER BY created_at ASC
|
||||
@@ -269,53 +332,92 @@ async fn download_task_zip(
|
||||
return Err(AppError::new(ErrorCode::NotFound, "没有可打包的文件"));
|
||||
}
|
||||
|
||||
let mut used_names: HashMap<String, usize> = HashMap::new();
|
||||
let mut entries: Vec<(String, String)> = Vec::new();
|
||||
for row in rows {
|
||||
let Some(path) = row.storage_path else {
|
||||
continue;
|
||||
};
|
||||
let name = build_zip_entry_name(&row.original_name, &row.output_format, &mut used_names);
|
||||
entries.push((name, path));
|
||||
}
|
||||
if entries.is_empty() {
|
||||
return Err(AppError::new(ErrorCode::NotFound, "没有可打包的文件"));
|
||||
}
|
||||
|
||||
let zip_path_cloned = zip_path.clone();
|
||||
let task_id_str = task_id.to_string();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
generate_zip_file(&zip_path_cloned, &task_id_str, &entries)
|
||||
})
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "生成 ZIP 失败").with_source(err))?
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "生成 ZIP 失败").with_source(err))?;
|
||||
|
||||
stream_zip(jar, zip_path, task_id).await
|
||||
}
|
||||
|
||||
async fn stream_zip(
|
||||
jar: axum_extra::extract::cookie::CookieJar,
|
||||
zip_path: PathBuf,
|
||||
task_id: Uuid,
|
||||
) -> Result<(axum_extra::extract::cookie::CookieJar, Response), AppError> {
|
||||
let file = tokio::fs::File::open(&zip_path).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "读取 ZIP 失败").with_source(err)
|
||||
let temp_dir = PathBuf::from(format!(
|
||||
"{}/tmp/zips/{task_id}-{}",
|
||||
state.config.storage_path,
|
||||
Uuid::new_v4()
|
||||
));
|
||||
tokio::fs::create_dir_all(&temp_dir).await.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "创建 ZIP 临时目录失败").with_source(err)
|
||||
})?;
|
||||
let zip_path = temp_dir.join(format!("task_{task_id}.zip"));
|
||||
|
||||
let stream = ReaderStream::new(file);
|
||||
let body = Body::from_stream(stream);
|
||||
let build_result: Result<storage::StoredObject, AppError> = async {
|
||||
let mut used_names: HashMap<String, usize> = HashMap::new();
|
||||
let mut entries: Vec<(String, String)> = Vec::new();
|
||||
for (index, row) in rows.into_iter().enumerate() {
|
||||
let Some(key) = row.storage_key else {
|
||||
continue;
|
||||
};
|
||||
let path = temp_dir.join(format!("entry-{index}"));
|
||||
storage::download_to_file(
|
||||
&state,
|
||||
&storage::ObjectLocator {
|
||||
backend: row.storage_backend,
|
||||
endpoint_id: row.storage_endpoint_id,
|
||||
key,
|
||||
},
|
||||
&path,
|
||||
)
|
||||
.await?;
|
||||
let name =
|
||||
build_zip_entry_name(&row.original_name, &row.output_format, &mut used_names);
|
||||
entries.push((name, path.to_string_lossy().to_string()));
|
||||
}
|
||||
if entries.is_empty() {
|
||||
return Err(AppError::new(ErrorCode::NotFound, "没有可打包的文件"));
|
||||
}
|
||||
|
||||
let mut resp_headers = HeaderMap::new();
|
||||
resp_headers.insert(header::CONTENT_TYPE, "application/zip".parse().unwrap());
|
||||
resp_headers.insert(
|
||||
header::CONTENT_DISPOSITION,
|
||||
format!("attachment; filename=\"task_{task_id}.zip\"")
|
||||
.parse()
|
||||
.unwrap(),
|
||||
);
|
||||
let zip_path_cloned = zip_path.clone();
|
||||
let task_id_str = task_id.to_string();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
generate_zip_file(&zip_path_cloned, &task_id_str, &entries)
|
||||
})
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "生成 ZIP 失败").with_source(err))?
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "生成 ZIP 失败").with_source(err))?;
|
||||
|
||||
Ok((jar, (resp_headers, body).into_response()))
|
||||
let object_key = storage::archive_key(task.retention_hours as i64, task_id);
|
||||
storage::store_file(&state, &object_key, &zip_path, "application/zip").await
|
||||
}
|
||||
.await;
|
||||
|
||||
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
|
||||
let stored = build_result?;
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE tasks
|
||||
SET zip_storage_backend = $2,
|
||||
zip_storage_endpoint_id = $3,
|
||||
zip_storage_key = $4,
|
||||
zip_storage_etag = $5,
|
||||
zip_size = $6
|
||||
WHERE id = $1 AND zip_storage_key IS NULL
|
||||
"#,
|
||||
)
|
||||
.bind(task_id)
|
||||
.bind(&stored.backend)
|
||||
.bind(stored.endpoint_id)
|
||||
.bind(&stored.key)
|
||||
.bind(&stored.etag)
|
||||
.bind(stored.size as i64)
|
||||
.execute(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "记录 ZIP 对象失败").with_source(err))?;
|
||||
|
||||
respond_object(
|
||||
&state,
|
||||
jar,
|
||||
&storage::ObjectLocator {
|
||||
backend: stored.backend,
|
||||
endpoint_id: stored.endpoint_id,
|
||||
key: stored.key,
|
||||
},
|
||||
&format!("task_{task_id}.zip"),
|
||||
"application/zip",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn build_zip_entry_name(
|
||||
@@ -333,6 +435,10 @@ fn build_zip_entry_name(
|
||||
"png" => "png",
|
||||
"webp" => "webp",
|
||||
"avif" => "avif",
|
||||
"gif" => "gif",
|
||||
"bmp" => "bmp",
|
||||
"tif" | "tiff" => "tiff",
|
||||
"ico" => "ico",
|
||||
_ => "bin",
|
||||
};
|
||||
|
||||
@@ -411,4 +517,10 @@ mod tests {
|
||||
assert_eq!(sanitized, "a".repeat(119));
|
||||
assert!(sanitized.is_char_boundary(sanitized.len()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_file_name_matches_converted_format() {
|
||||
assert_eq!(output_file_name("photo.png", "webp"), "photo.webp");
|
||||
assert_eq!(output_file_name("没有扩展名", "jpeg"), "没有扩展名.jpg");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
mod admin;
|
||||
mod admin_storage;
|
||||
mod auth;
|
||||
mod billing;
|
||||
mod compress;
|
||||
@@ -62,5 +63,6 @@ fn v1_router() -> Router<AppState> {
|
||||
.merge(webhooks::router())
|
||||
.merge(user::router())
|
||||
.merge(admin::router())
|
||||
.merge(admin_storage::router())
|
||||
.fallback(response::not_found)
|
||||
}
|
||||
|
||||
100
src/api/tasks.rs
100
src/api/tasks.rs
@@ -6,6 +6,7 @@ use crate::services::billing::{BillingContext, Plan};
|
||||
use crate::services::compress;
|
||||
use crate::services::compress::{CompressionLevel, ImageFmt};
|
||||
use crate::services::idempotency;
|
||||
use crate::services::storage;
|
||||
use crate::state::AppState;
|
||||
|
||||
use axum::extract::{ConnectInfo, Multipart, Path, State};
|
||||
@@ -72,13 +73,6 @@ async fn create_batch_task(
|
||||
let ip = context::client_ip(&headers, addr.ip());
|
||||
let (jar, principal) = context::authenticate(&state, jar, &headers, ip).await?;
|
||||
|
||||
if !state.config.storage_type.eq_ignore_ascii_case("local") {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::StorageUnavailable,
|
||||
"当前仅支持本地存储（STORAGE_TYPE=local）",
|
||||
));
|
||||
}
|
||||
|
||||
let idempotency_key = headers
|
||||
.get("idempotency-key")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
@@ -217,13 +211,8 @@ async fn create_batch_task(
|
||||
}
|
||||
}?;
|
||||
|
||||
tokio::fs::create_dir_all(&state.config.storage_path)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
AppError::new(ErrorCode::StorageUnavailable, "创建存储目录失败").with_source(err)
|
||||
})?;
|
||||
|
||||
let expires_at = Utc::now() + retention;
|
||||
let retention_hours = retention.num_hours();
|
||||
let (user_id, session_id, api_key_id) = match &task_owner {
|
||||
TaskOwner::Anonymous { session_id } => (None, Some(session_id.clone()), None),
|
||||
TaskOwner::User { user_id } => (Some(*user_id), None, None),
|
||||
@@ -248,13 +237,13 @@ async fn create_batch_task(
|
||||
compression_rate,
|
||||
total_files, completed_files, failed_files,
|
||||
total_original_size, total_compressed_size,
|
||||
expires_at
|
||||
expires_at, retention_hours
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5::inet, $6::task_source, 'pending',
|
||||
$7::compression_level, $8, $9, $10, $11, $12,
|
||||
$13, 0, 0,
|
||||
$14, 0,
|
||||
$15
|
||||
$15, $16
|
||||
)
|
||||
"#,
|
||||
)
|
||||
@@ -273,6 +262,7 @@ async fn create_batch_task(
|
||||
.bind(files.len() as i32)
|
||||
.bind(total_original_size)
|
||||
.bind(expires_at)
|
||||
.bind(retention_hours as i32)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "创建任务失败").with_source(err))?;
|
||||
@@ -284,7 +274,7 @@ async fn create_batch_task(
|
||||
id, task_id,
|
||||
original_name, original_format, output_format,
|
||||
original_size,
|
||||
storage_path, status
|
||||
input_path, status
|
||||
) VALUES (
|
||||
$1, $2,
|
||||
$3, $4, $5,
|
||||
@@ -990,24 +980,61 @@ async fn delete_task(
|
||||
));
|
||||
}
|
||||
|
||||
let paths: Vec<Option<String>> =
|
||||
sqlx::query_scalar("SELECT storage_path FROM task_files WHERE task_id = $1")
|
||||
.bind(task_id)
|
||||
.fetch_all(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "查询文件失败").with_source(err))?;
|
||||
let files = sqlx::query_as::<_, TaskStorageRow>(
|
||||
r#"
|
||||
SELECT storage_backend, storage_endpoint_id,
|
||||
COALESCE(storage_key, storage_path) AS storage_key,
|
||||
input_path
|
||||
FROM task_files
|
||||
WHERE task_id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(task_id)
|
||||
.fetch_all(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "查询文件失败").with_source(err))?;
|
||||
|
||||
for p in paths.into_iter().flatten() {
|
||||
let _ = tokio::fs::remove_file(p).await;
|
||||
for file in files {
|
||||
if let Some(key) = file.storage_key {
|
||||
storage::delete_object(
|
||||
&state,
|
||||
&storage::ObjectLocator {
|
||||
backend: file.storage_backend,
|
||||
endpoint_id: file.storage_endpoint_id,
|
||||
key,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
if let Some(input_path) = file.input_path {
|
||||
let _ = tokio::fs::remove_file(input_path).await;
|
||||
}
|
||||
}
|
||||
|
||||
if state.config.storage_type.eq_ignore_ascii_case("local") {
|
||||
let zip_path = format!("{}/zips/{task_id}.zip", state.config.storage_path);
|
||||
let _ = tokio::fs::remove_file(zip_path).await;
|
||||
let orig_dir = format!("{}/orig/{task_id}", state.config.storage_path);
|
||||
let _ = tokio::fs::remove_dir_all(orig_dir).await;
|
||||
let zip = sqlx::query_as::<_, TaskZipStorageRow>(
|
||||
"SELECT zip_storage_backend, zip_storage_endpoint_id, zip_storage_key FROM tasks WHERE id = $1",
|
||||
)
|
||||
.bind(task_id)
|
||||
.fetch_one(&state.db)
|
||||
.await
|
||||
.map_err(|err| AppError::new(ErrorCode::Internal, "查询 ZIP 文件失败").with_source(err))?;
|
||||
if let (Some(backend), Some(key)) = (zip.zip_storage_backend, zip.zip_storage_key) {
|
||||
storage::delete_object(
|
||||
&state,
|
||||
&storage::ObjectLocator {
|
||||
backend,
|
||||
endpoint_id: zip.zip_storage_endpoint_id,
|
||||
key,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
let legacy_zip_path = format!("{}/zips/{task_id}.zip", state.config.storage_path);
|
||||
let _ = tokio::fs::remove_file(legacy_zip_path).await;
|
||||
let orig_dir = format!("{}/orig/{task_id}", state.config.storage_path);
|
||||
let _ = tokio::fs::remove_dir_all(orig_dir).await;
|
||||
|
||||
let deleted = sqlx::query("DELETE FROM tasks WHERE id = $1")
|
||||
.bind(task_id)
|
||||
.execute(&state.db)
|
||||
@@ -1027,6 +1054,21 @@ async fn delete_task(
|
||||
))
|
||||
}
|
||||
|
||||
#[derive(Debug, FromRow)]
|
||||
struct TaskStorageRow {
|
||||
storage_backend: String,
|
||||
storage_endpoint_id: Option<Uuid>,
|
||||
storage_key: Option<String>,
|
||||
input_path: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, FromRow)]
|
||||
struct TaskZipStorageRow {
|
||||
zip_storage_backend: Option<String>,
|
||||
zip_storage_endpoint_id: Option<Uuid>,
|
||||
zip_storage_key: Option<String>,
|
||||
}
|
||||
|
||||
fn authorize_task(
|
||||
principal: &context::Principal,
|
||||
user_id: Option<Uuid>,
|
||||
|
||||
@@ -500,7 +500,7 @@ async fn list_history(
|
||||
status: String,
|
||||
output_format: String,
|
||||
error_message: Option<String>,
|
||||
storage_path: Option<String>,
|
||||
has_storage: bool,
|
||||
}
|
||||
|
||||
let now = Utc::now();
|
||||
@@ -517,7 +517,7 @@ async fn list_history(
|
||||
status::text AS status,
|
||||
output_format,
|
||||
error_message,
|
||||
storage_path
|
||||
COALESCE(storage_key, storage_path) IS NOT NULL AS has_storage
|
||||
FROM task_files
|
||||
WHERE task_id = $1
|
||||
ORDER BY created_at ASC
|
||||
@@ -540,7 +540,7 @@ async fn list_history(
|
||||
output_format: file.output_format,
|
||||
error_message: file.error_message,
|
||||
download_url: if file.status == "completed"
|
||||
&& file.storage_path.is_some()
|
||||
&& file.has_storage
|
||||
&& task.expires_at > now
|
||||
{
|
||||
Some(format!("/downloads/{}", file.id))
|
||||
|
||||
Reference in New Issue
Block a user