feat: add runtime policy and observability
Some checks failed
CI / verify (push) Has been cancelled

This commit is contained in:
237899745
2026-07-25 20:00:11 +08:00
parent f3c7a77a37
commit 64b1169e8c
32 changed files with 1236 additions and 120 deletions

View File

@@ -16,6 +16,7 @@ use img_parts::{Bytes as ImgBytes, DynImage, ImageEXIF, ImageICC};
use oxipng::StripChunks;
use rgb::FromSlice;
use std::io::Cursor;
use std::time::Instant;
const TARGET_MIN_LONG_EDGE: u32 = 640;
const TARGET_MIN_SCALE: f64 = 0.55;
@@ -287,7 +288,12 @@ pub async fn compress_image_bytes(
max_height: Option<u32>,
preserve_metadata: bool,
) -> Result<Vec<u8>, AppError> {
let max_image_pixels = state.config.max_image_pixels;
let started = Instant::now();
let bytes_in = input.len() as u64;
let max_image_pixels = crate::services::settings::runtime_policy(state)
.await?
.file_limits
.max_image_pixels;
let permit = state
.image_processing_semaphore
.clone()
@@ -297,7 +303,7 @@ pub async fn compress_image_bytes(
AppError::new(ErrorCode::Internal, "图片处理并发控制器已关闭").with_source(err)
})?;
tokio::task::spawn_blocking(move || {
let result = match tokio::task::spawn_blocking(move || {
let _permit = permit;
compress_image_bytes_sync(
input,
@@ -313,9 +319,19 @@ pub async fn compress_image_bytes(
)
})
.await
.map_err(|err| {
AppError::new(ErrorCode::CompressionFailed, "图片处理任务异常退出").with_source(err)
})?
{
Ok(result) => result,
Err(err) => Err(
AppError::new(ErrorCode::CompressionFailed, "图片处理任务异常退出").with_source(err),
),
};
crate::services::metrics::record_compression(
state,
started.elapsed(),
bytes_in,
result.as_ref().ok().map(|bytes| bytes.len() as u64),
);
result
}
#[allow(clippy::too_many_arguments)]

273
src/services/metrics.rs Normal file
View File

@@ -0,0 +1,273 @@
use crate::error::ErrorCode;
use crate::state::AppState;
use std::fmt::Write;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::OnceLock;
use std::time::Duration;
pub const QUEUE_STREAM_KEY: &str = "stream:compress_jobs";
pub const QUEUE_GROUP_NAME: &str = "compress_workers";
pub const DEAD_STREAM_KEY: &str = "stream:compress_jobs:dead";
pub const CLUSTER_METRICS_KEY: &str = "metrics:imageforge";
const METHODS: [&str; 5] = ["GET", "POST", "PUT", "DELETE", "OTHER"];
const STATUS_CLASSES: [&str; 5] = ["2xx", "3xx", "4xx", "5xx", "other"];
const ERROR_CODES: [&str; 17] = [
"INVALID_REQUEST",
"INVALID_IMAGE",
"UNSUPPORTED_FORMAT",
"TOO_MANY_PIXELS",
"FILE_TOO_LARGE",
"INVALID_TOKEN",
"UNAUTHORIZED",
"FORBIDDEN",
"NOT_FOUND",
"IDEMPOTENCY_CONFLICT",
"RATE_LIMITED",
"QUOTA_EXCEEDED",
"EMAIL_NOT_VERIFIED",
"COMPRESSION_FAILED",
"STORAGE_UNAVAILABLE",
"MAIL_SEND_FAILED",
"INTERNAL",
];
const DURATION_BUCKETS: [f64; 9] = [0.01, 0.05, 0.1, 0.3, 1.0, 3.0, 10.0, 30.0, f64::INFINITY];
struct Histogram {
buckets: [AtomicU64; DURATION_BUCKETS.len()],
count: AtomicU64,
sum_micros: AtomicU64,
}
impl Histogram {
fn new() -> Self {
Self {
buckets: std::array::from_fn(|_| AtomicU64::new(0)),
count: AtomicU64::new(0),
sum_micros: AtomicU64::new(0),
}
}
fn observe(&self, duration: Duration) {
let seconds = duration.as_secs_f64();
for (index, upper_bound) in DURATION_BUCKETS.iter().enumerate() {
if seconds <= *upper_bound {
self.buckets[index].fetch_add(1, Ordering::Relaxed);
}
}
self.count.fetch_add(1, Ordering::Relaxed);
self.sum_micros.fetch_add(
duration.as_micros().min(u64::MAX as u128) as u64,
Ordering::Relaxed,
);
}
fn render(&self, output: &mut String, name: &str) {
for (index, upper_bound) in DURATION_BUCKETS.iter().enumerate() {
let label = if upper_bound.is_infinite() {
"+Inf".to_string()
} else {
upper_bound.to_string()
};
let _ = writeln!(
output,
"{name}_bucket{{le=\"{label}\"}} {}",
self.buckets[index].load(Ordering::Relaxed)
);
}
let _ = writeln!(
output,
"{name}_sum {}",
self.sum_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0
);
let _ = writeln!(
output,
"{name}_count {}",
self.count.load(Ordering::Relaxed)
);
}
}
struct Metrics {
http_requests: [[AtomicU64; STATUS_CLASSES.len()]; METHODS.len()],
http_duration: Histogram,
errors: [AtomicU64; ERROR_CODES.len()],
}
impl Metrics {
fn new() -> Self {
Self {
http_requests: std::array::from_fn(|_| std::array::from_fn(|_| AtomicU64::new(0))),
http_duration: Histogram::new(),
errors: std::array::from_fn(|_| AtomicU64::new(0)),
}
}
}
fn registry() -> &'static Metrics {
static METRICS: OnceLock<Metrics> = OnceLock::new();
METRICS.get_or_init(Metrics::new)
}
pub fn record_http(method: &str, status: u16, duration: Duration) {
let method_index = match method {
"GET" => 0,
"POST" => 1,
"PUT" => 2,
"DELETE" => 3,
_ => 4,
};
let status_index = match status {
200..=299 => 0,
300..=399 => 1,
400..=499 => 2,
500..=599 => 3,
_ => 4,
};
let metrics = registry();
metrics.http_requests[method_index][status_index].fetch_add(1, Ordering::Relaxed);
metrics.http_duration.observe(duration);
}
pub fn record_error(code: ErrorCode) {
registry().errors[error_index(code)].fetch_add(1, Ordering::Relaxed);
}
pub fn record_compression(
state: &AppState,
duration: Duration,
bytes_in: u64,
bytes_out: Option<u64>,
) {
let mut increments = vec![
("compression_bytes_in", bytes_in),
(
"compression_duration_micros",
duration.as_micros().min(u64::MAX as u128) as u64,
),
];
if let Some(bytes_out) = bytes_out {
increments.push(("compression_success", 1));
increments.push(("compression_bytes_out", bytes_out));
} else {
increments.push(("compression_failed", 1));
}
persist_cluster_increments(state, increments);
}
pub fn record_storage_fallback(state: &AppState) {
persist_cluster_increments(state, vec![("storage_fallbacks", 1)]);
}
pub fn record_dead_letter(state: &AppState) {
persist_cluster_increments(state, vec![("dead_letters", 1)]);
}
pub fn render() -> String {
let metrics = registry();
let mut output = String::with_capacity(8 * 1024);
output.push_str(
"# HELP imageforge_http_requests_total HTTP requests handled by method and status class.\n",
);
output.push_str("# TYPE imageforge_http_requests_total counter\n");
for (method_index, method) in METHODS.iter().enumerate() {
for (status_index, status_class) in STATUS_CLASSES.iter().enumerate() {
let _ = writeln!(
output,
"imageforge_http_requests_total{{method=\"{method}\",status_class=\"{status_class}\"}} {}",
metrics.http_requests[method_index][status_index].load(Ordering::Relaxed)
);
}
}
output.push_str("# HELP imageforge_http_request_duration_seconds HTTP request duration.\n");
output.push_str("# TYPE imageforge_http_request_duration_seconds histogram\n");
metrics
.http_duration
.render(&mut output, "imageforge_http_request_duration_seconds");
output.push_str("# HELP imageforge_errors_total Application errors by code.\n");
output.push_str("# TYPE imageforge_errors_total counter\n");
for (index, code) in ERROR_CODES.iter().enumerate() {
let _ = writeln!(
output,
"imageforge_errors_total{{code=\"{code}\"}} {}",
metrics.errors[index].load(Ordering::Relaxed)
);
}
output
}
fn persist_cluster_increments(state: &AppState, increments: Vec<(&'static str, u64)>) {
let mut connection = state.redis.clone();
tokio::spawn(async move {
let mut pipeline = redis::pipe();
for (field, amount) in increments {
pipeline
.cmd("HINCRBY")
.arg(CLUSTER_METRICS_KEY)
.arg(field)
.arg(amount.min(i64::MAX as u64) as i64)
.ignore();
}
if let Err(err) = pipeline.query_async::<_, ()>(&mut connection).await {
tracing::debug!(error = %err, "failed to persist cluster metric");
}
});
}
fn error_index(code: ErrorCode) -> usize {
match code {
ErrorCode::InvalidRequest => 0,
ErrorCode::InvalidImage => 1,
ErrorCode::UnsupportedFormat => 2,
ErrorCode::TooManyPixels => 3,
ErrorCode::FileTooLarge => 4,
ErrorCode::InvalidToken => 5,
ErrorCode::Unauthorized => 6,
ErrorCode::Forbidden => 7,
ErrorCode::NotFound => 8,
ErrorCode::IdempotencyConflict => 9,
ErrorCode::RateLimited => 10,
ErrorCode::QuotaExceeded => 11,
ErrorCode::EmailNotVerified => 12,
ErrorCode::CompressionFailed => 13,
ErrorCode::StorageUnavailable => 14,
ErrorCode::MailSendFailed => 15,
ErrorCode::Internal => 16,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn error_codes_and_slots_stay_aligned() {
let codes = [
ErrorCode::InvalidRequest,
ErrorCode::InvalidImage,
ErrorCode::UnsupportedFormat,
ErrorCode::TooManyPixels,
ErrorCode::FileTooLarge,
ErrorCode::InvalidToken,
ErrorCode::Unauthorized,
ErrorCode::Forbidden,
ErrorCode::NotFound,
ErrorCode::IdempotencyConflict,
ErrorCode::RateLimited,
ErrorCode::QuotaExceeded,
ErrorCode::EmailNotVerified,
ErrorCode::CompressionFailed,
ErrorCode::StorageUnavailable,
ErrorCode::MailSendFailed,
ErrorCode::Internal,
];
for (index, code) in codes.into_iter().enumerate() {
assert_eq!(error_index(code), index);
assert_eq!(ERROR_CODES[index], code.as_str());
}
}
}

View File

@@ -5,6 +5,7 @@ pub mod credentials;
pub mod filename;
pub mod idempotency;
pub mod mail;
pub mod metrics;
pub mod quota;
pub mod rate_limit;
pub mod settings;

View File

@@ -259,7 +259,10 @@ pub async fn consume_anonymous_units(
let mut conn = state.redis.clone();
let limit = state.config.anon_daily_units as i64;
let limit = crate::services::settings::runtime_policy(state)
.await?
.rate_limits
.anonymous_units_per_day as i64;
let ttl_seconds = 48 * 60 * 60;
let inc = units as i64;
@@ -299,7 +302,7 @@ pub async fn consume_anonymous_units(
if new_value < 0 {
return Err(AppError::new(
ErrorCode::QuotaExceeded,
"匿名试用次数已用完(每日 10 次)",
format!("匿名试用次数已用完(每日 {limit} 次)"),
));
}

View File

@@ -9,6 +9,13 @@ use rand::RngCore;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use sqlx::FromRow;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::RwLock;
use tokio::time::Instant;
const RUNTIME_POLICY_CACHE_TTL: Duration = Duration::from_secs(5);
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MailCustomSmtp {
@@ -38,6 +45,151 @@ fn default_true() -> bool {
true
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FeaturesConfigStored {
#[serde(default = "default_true")]
pub registration_enabled: bool,
#[serde(default = "default_true")]
pub api_key_enabled: bool,
#[serde(default = "default_true")]
pub anonymous_upload_enabled: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RateLimitsConfigStored {
#[serde(default = "default_anonymous_per_minute")]
pub anonymous_per_minute: u32,
#[serde(default = "default_anonymous_units_per_day")]
pub anonymous_units_per_day: u32,
#[serde(default = "default_user_per_minute")]
pub user_per_minute: u32,
#[serde(default = "default_api_key_per_minute")]
pub api_key_per_minute: u32,
#[serde(default = "default_login_ip_per_5_minutes")]
pub login_ip_per_5_minutes: u32,
#[serde(default = "default_login_identity_per_5_minutes")]
pub login_identity_per_5_minutes: u32,
#[serde(default = "default_register_ip_per_hour")]
pub register_ip_per_hour: u32,
#[serde(default = "default_verification_email_per_minute")]
pub verification_email_per_minute: u32,
#[serde(default = "default_email_verify_ip_per_15_minutes")]
pub email_verify_ip_per_15_minutes: u32,
#[serde(default = "default_forgot_password_ip_per_15_minutes")]
pub forgot_password_ip_per_15_minutes: u32,
#[serde(default = "default_forgot_password_email_per_15_minutes")]
pub forgot_password_email_per_15_minutes: u32,
#[serde(default = "default_password_reset_ip_per_15_minutes")]
pub password_reset_ip_per_15_minutes: u32,
#[serde(default = "default_password_reset_token_per_15_minutes")]
pub password_reset_token_per_15_minutes: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FileLimitsConfigStored {
#[serde(default = "default_max_image_pixels")]
pub max_image_pixels: u64,
}
#[derive(Debug, Clone)]
pub struct RuntimePolicy {
pub auth: AuthConfigStored,
pub features: FeaturesConfigStored,
pub rate_limits: RateLimitsConfigStored,
pub file_limits: FileLimitsConfigStored,
}
#[derive(Debug, Clone)]
struct CachedRuntimePolicy {
loaded_at: Instant,
policy: RuntimePolicy,
}
#[derive(Debug, Clone, Default)]
pub struct RuntimePolicyCache {
inner: Arc<RwLock<Option<CachedRuntimePolicy>>>,
}
impl RuntimePolicyCache {
pub fn new() -> Self {
Self::default()
}
async fn get(&self) -> Option<RuntimePolicy> {
let cache = self.inner.read().await;
cache.as_ref().and_then(|cached| {
(cached.loaded_at.elapsed() < RUNTIME_POLICY_CACHE_TTL).then(|| cached.policy.clone())
})
}
async fn set(&self, policy: RuntimePolicy) {
*self.inner.write().await = Some(CachedRuntimePolicy {
loaded_at: Instant::now(),
policy,
});
}
pub async fn invalidate(&self) {
*self.inner.write().await = None;
}
}
fn default_anonymous_per_minute() -> u32 {
10
}
fn default_anonymous_units_per_day() -> u32 {
10
}
fn default_user_per_minute() -> u32 {
60
}
fn default_api_key_per_minute() -> u32 {
100
}
fn default_login_ip_per_5_minutes() -> u32 {
30
}
fn default_login_identity_per_5_minutes() -> u32 {
10
}
fn default_register_ip_per_hour() -> u32 {
10
}
fn default_verification_email_per_minute() -> u32 {
1
}
fn default_email_verify_ip_per_15_minutes() -> u32 {
20
}
fn default_forgot_password_ip_per_15_minutes() -> u32 {
5
}
fn default_forgot_password_email_per_15_minutes() -> u32 {
3
}
fn default_password_reset_ip_per_15_minutes() -> u32 {
10
}
fn default_password_reset_token_per_15_minutes() -> u32 {
5
}
fn default_max_image_pixels() -> u64 {
40_000_000
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StripeConfigStored {
pub secret_key_encrypted: Option<String>,
@@ -73,6 +225,144 @@ pub async fn load_system_config<T: DeserializeOwned>(
Ok(Some(parsed))
}
pub async fn runtime_policy(state: &AppState) -> Result<RuntimePolicy, AppError> {
if let Some(policy) = state.runtime_policy_cache.get().await {
return Ok(policy);
}
#[derive(Debug, FromRow)]
struct ConfigValueRow {
key: String,
value: serde_json::Value,
}
let rows = sqlx::query_as::<_, ConfigValueRow>(
r#"
SELECT key, value
FROM system_config
WHERE key = ANY($1)
"#,
)
.bind(["auth", "features", "rate_limits", "file_limits"])
.fetch_all(&state.db)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "查询运行策略失败").with_source(err))?;
let mut auth = AuthConfigStored {
email_verification_required: true,
};
let mut features = FeaturesConfigStored {
registration_enabled: true,
api_key_enabled: true,
anonymous_upload_enabled: true,
};
let mut rate_limits = RateLimitsConfigStored {
anonymous_units_per_day: state.config.anon_daily_units,
..RateLimitsConfigStored::default()
};
let mut file_limits = FileLimitsConfigStored {
max_image_pixels: state.config.max_image_pixels,
};
for row in rows {
match row.key.as_str() {
"auth" => auth = parse_config_value(row.value, "认证配置")?,
"features" => features = parse_config_value(row.value, "功能开关")?,
"rate_limits" => rate_limits = parse_config_value(row.value, "限速配置")?,
"file_limits" => file_limits = parse_config_value(row.value, "文件限制")?,
_ => {}
}
}
features.anonymous_upload_enabled &= state.config.allow_anonymous_upload;
validate_rate_limits(&rate_limits)?;
validate_file_limits(&file_limits)?;
let policy = RuntimePolicy {
auth,
features,
rate_limits,
file_limits,
};
state.runtime_policy_cache.set(policy.clone()).await;
Ok(policy)
}
fn parse_config_value<T: DeserializeOwned>(
value: serde_json::Value,
label: &str,
) -> Result<T, AppError> {
serde_json::from_value(value).map_err(|err| {
AppError::new(ErrorCode::Internal, format!("{label}格式错误")).with_source(err)
})
}
pub fn validate_runtime_config_value(key: &str, value: &serde_json::Value) -> Result<(), AppError> {
match key {
"auth" => {
serde_json::from_value::<AuthConfigStored>(value.clone()).map_err(|err| {
AppError::new(ErrorCode::InvalidRequest, "认证配置格式错误").with_source(err)
})?;
}
"features" => {
serde_json::from_value::<FeaturesConfigStored>(value.clone()).map_err(|err| {
AppError::new(ErrorCode::InvalidRequest, "功能开关格式错误").with_source(err)
})?;
}
"rate_limits" => {
let config =
serde_json::from_value::<RateLimitsConfigStored>(value.clone()).map_err(|err| {
AppError::new(ErrorCode::InvalidRequest, "限速配置格式错误").with_source(err)
})?;
validate_rate_limits(&config)?;
}
"file_limits" => {
let config =
serde_json::from_value::<FileLimitsConfigStored>(value.clone()).map_err(|err| {
AppError::new(ErrorCode::InvalidRequest, "文件限制格式错误").with_source(err)
})?;
validate_file_limits(&config)?;
}
_ => {}
}
Ok(())
}
fn validate_rate_limits(config: &RateLimitsConfigStored) -> Result<(), AppError> {
let values = [
config.anonymous_per_minute,
config.anonymous_units_per_day,
config.user_per_minute,
config.api_key_per_minute,
config.login_ip_per_5_minutes,
config.login_identity_per_5_minutes,
config.register_ip_per_hour,
config.verification_email_per_minute,
config.email_verify_ip_per_15_minutes,
config.forgot_password_ip_per_15_minutes,
config.forgot_password_email_per_15_minutes,
config.password_reset_ip_per_15_minutes,
config.password_reset_token_per_15_minutes,
];
if values.iter().any(|value| !(1..=100_000).contains(value)) {
return Err(AppError::new(
ErrorCode::InvalidRequest,
"限速值必须在 1 到 100000 之间",
));
}
Ok(())
}
fn validate_file_limits(config: &FileLimitsConfigStored) -> Result<(), AppError> {
if !(1_000_000..=200_000_000).contains(&config.max_image_pixels) {
return Err(AppError::new(
ErrorCode::InvalidRequest,
"max_image_pixels 必须在 1000000 到 200000000 之间",
));
}
Ok(())
}
pub async fn upsert_system_config(
state: &AppState,
key: &str,
@@ -99,6 +389,10 @@ pub async fn upsert_system_config(
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "更新系统配置失败").with_source(err))?;
if matches!(key, "auth" | "features" | "rate_limits" | "file_limits") {
state.runtime_policy_cache.invalidate().await;
}
Ok(())
}
@@ -128,10 +422,10 @@ pub async fn load_mail_settings(state: &AppState) -> Result<Option<MailSettings>
}
pub async fn email_verification_required(state: &AppState) -> Result<bool, AppError> {
Ok(load_system_config::<AuthConfigStored>(state, "auth")
Ok(runtime_policy(state)
.await?
.map(|config| config.email_verification_required)
.unwrap_or(true))
.auth
.email_verification_required)
}
pub async fn load_stripe_secrets(state: &AppState) -> Result<Option<StripeSecrets>, AppError> {
@@ -243,6 +537,26 @@ pub async fn get_stripe_webhook_secret(state: &AppState) -> Result<String, AppEr
.ok_or_else(|| AppError::new(ErrorCode::InvalidRequest, "未配置 Stripe Webhook Secret"))
}
impl Default for RateLimitsConfigStored {
fn default() -> Self {
Self {
anonymous_per_minute: default_anonymous_per_minute(),
anonymous_units_per_day: default_anonymous_units_per_day(),
user_per_minute: default_user_per_minute(),
api_key_per_minute: default_api_key_per_minute(),
login_ip_per_5_minutes: default_login_ip_per_5_minutes(),
login_identity_per_5_minutes: default_login_identity_per_5_minutes(),
register_ip_per_hour: default_register_ip_per_hour(),
verification_email_per_minute: default_verification_email_per_minute(),
email_verify_ip_per_15_minutes: default_email_verify_ip_per_15_minutes(),
forgot_password_ip_per_15_minutes: default_forgot_password_ip_per_15_minutes(),
forgot_password_email_per_15_minutes: default_forgot_password_email_per_15_minutes(),
password_reset_ip_per_15_minutes: default_password_reset_ip_per_15_minutes(),
password_reset_token_per_15_minutes: default_password_reset_token_per_15_minutes(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -252,4 +566,32 @@ mod tests {
let config: AuthConfigStored = serde_json::from_value(serde_json::json!({})).unwrap();
assert!(config.email_verification_required);
}
#[test]
fn rate_limit_defaults_support_legacy_config_rows() {
let config: RateLimitsConfigStored = serde_json::from_value(serde_json::json!({
"anonymous_per_minute": 7,
"user_per_minute": 55
}))
.unwrap();
assert_eq!(config.anonymous_per_minute, 7);
assert_eq!(config.user_per_minute, 55);
assert_eq!(config.login_identity_per_5_minutes, 10);
assert_eq!(config.password_reset_token_per_15_minutes, 5);
}
#[test]
fn invalid_runtime_limits_are_rejected_before_persisting() {
let result = validate_runtime_config_value(
"rate_limits",
&serde_json::json!({ "anonymous_per_minute": 0 }),
);
assert!(result.is_err());
let result = validate_runtime_config_value(
"file_limits",
&serde_json::json!({ "max_image_pixels": 999_999 }),
);
assert!(result.is_err());
}
}

View File

@@ -161,7 +161,7 @@ where
if let Some(endpoint) = active_endpoint(state).await? {
match store_bytes_s3(state, &endpoint, key, bytes.clone(), content_type).await {
Ok(stored) => return Ok(stored),
Err(err) => log_local_fallback(&endpoint, key, &err),
Err(err) => log_local_fallback(state, &endpoint, key, &err),
}
}
@@ -233,7 +233,7 @@ pub async fn store_file(
if let Some(endpoint) = active_endpoint(state).await? {
match store_file_s3(state, &endpoint, key, path, content_type, metadata.len()).await {
Ok(stored) => return Ok(stored),
Err(err) => log_local_fallback(&endpoint, key, &err),
Err(err) => log_local_fallback(state, &endpoint, key, &err),
}
}
@@ -302,7 +302,8 @@ async fn store_file_local(
})
}
fn log_local_fallback(endpoint: &StorageEndpoint, key: &str, err: &AppError) {
fn log_local_fallback(state: &AppState, endpoint: &StorageEndpoint, key: &str, err: &AppError) {
crate::services::metrics::record_storage_fallback(state);
tracing::warn!(
storage_endpoint_id = %endpoint.id,
storage_endpoint = %endpoint.name,