237899745/ystp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: settle quota and bound compression work
Some checks failed
CI / verify (push) Has been cancelled
Details

Browse Source
This commit is contained in:
237899745
2026-07-25 21:40:57 +08:00
parent 64b1169e8c
commit b67275b3e8
12 changed files with 1150 additions and 657 deletions
Download Patch File Download Diff File Expand all files Collapse all files

368
src/services/compress.rs
View File

@@ -16,18 +16,79 @@ use img_parts::{Bytes as ImgBytes, DynImage, ImageEXIF, ImageICC};
use oxipng::StripChunks;
use rgb::FromSlice;
use std::io::Cursor;
use std::time::Instant;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
const TARGET_MIN_LONG_EDGE: u32 = 640;
const TARGET_MIN_SCALE: f64 = 0.55;
const TARGET_SCALE_SEARCH_ATTEMPTS: usize = 6;
const TARGET_QUALITY_INTERPOLATION_ATTEMPTS: usize = 2;
const COMPRESSION_TIME_BUDGET: Duration = Duration::from_secs(30);
const COMPRESSION_TIMEOUT_GRACE: Duration = Duration::from_secs(1);
const JPEG_TARGET_MIN_QUALITY: u8 = 25;
const WEBP_TARGET_MIN_QUALITY: u8 = 30;
const AVIF_TARGET_MIN_QUALITY: u8 = 38;
const METADATA_TARGET_OVERHEAD: u64 = 1024;
#[derive(Clone)]
struct CompressionDeadline {
deadline: Option<Instant>,
cancelled: Arc<AtomicBool>,
}
impl CompressionDeadline {
fn new(budget: Duration) -> Self {
Self {
deadline: Instant::now().checked_add(budget),
cancelled: Arc::new(AtomicBool::new(false)),
}
}
#[cfg(test)]
fn unlimited() -> Self {
Self {
deadline: None,
cancelled: Arc::new(AtomicBool::new(false)),
}
}
fn cancel(&self) {
self.cancelled.store(true, Ordering::Relaxed);
}
fn expired(&self) -> bool {
self.cancelled.load(Ordering::Relaxed)
|| self
.deadline
.is_some_and(|deadline| Instant::now() >= deadline)
}
fn remaining(&self) -> Duration {
if self.cancelled.load(Ordering::Relaxed) {
return Duration::ZERO;
}
self.deadline
.map(|deadline| deadline.saturating_duration_since(Instant::now()))
.unwrap_or(Duration::MAX)
}
fn check(&self) -> Result<(), AppError> {
if self.expired() {
return Err(compression_timeout_error());
}
Ok(())
}
}
fn compression_timeout_error() -> AppError {
AppError::new(
ErrorCode::CompressionFailed,
"图片处理超过 30 秒，请降低分辨率或改用 JPEG/WebP 格式",
)
}
#[derive(Debug, Clone, Copy)]
pub enum CompressionLevel {
High,
@@ -294,16 +355,27 @@ pub async fn compress_image_bytes(
.await?
.file_limits
.max_image_pixels;
let permit = state
.image_processing_semaphore
.clone()
.acquire_owned()
.await
.map_err(|err| {
AppError::new(ErrorCode::Internal, "图片处理并发控制器已关闭").with_source(err)
})?;
let deadline = CompressionDeadline::new(COMPRESSION_TIME_BUDGET);
let permit = match tokio::time::timeout(
deadline.remaining(),
state.image_processing_semaphore.clone().acquire_owned(),
)
.await
{
Ok(Ok(permit)) => permit,
Ok(Err(err)) => {
return Err(
AppError::new(ErrorCode::Internal, "图片处理并发控制器已关闭").with_source(err),
)
}
Err(_) => {
crate::services::metrics::record_compression(state, started.elapsed(), bytes_in, None);
return Err(compression_timeout_error());
}
};
let result = match tokio::task::spawn_blocking(move || {
let blocking_deadline = deadline.clone();
let handle = tokio::task::spawn_blocking(move || {
let _permit = permit;
compress_image_bytes_sync(
input,
@@ -316,14 +388,25 @@ pub async fn compress_image_bytes(
max_height,
preserve_metadata,
max_image_pixels,
&blocking_deadline,
)
})
});
let result = match tokio::time::timeout(
deadline
.remaining()
.saturating_add(COMPRESSION_TIMEOUT_GRACE),
handle,
)
.await
{
Ok(result) => result,
Err(err) => Err(
Ok(Ok(result)) => result,
Ok(Err(err)) => Err(
AppError::new(ErrorCode::CompressionFailed, "图片处理任务异常退出").with_source(err),
),
Err(_) => {
deadline.cancel();
Err(compression_timeout_error())
}
};
crate::services::metrics::record_compression(
state,
@@ -346,7 +429,9 @@ fn compress_image_bytes_sync(
max_height: Option<u32>,
preserve_metadata: bool,
max_image_pixels: u64,
deadline: &CompressionDeadline,
) -> Result<Vec<u8>, AppError> {
deadline.check()?;
let original_size = input.len() as u64;
#[cfg(not(target_os = "linux"))]
@@ -358,6 +443,7 @@ fn compress_image_bytes_sync(
}
let orientation = inspect_image(&input, max_image_pixels)?;
deadline.check()?;
if is_animated_image(&input, format_in)? {
return Err(AppError::new(
@@ -365,6 +451,7 @@ fn compress_image_bytes_sync(
format!("暂不支持动画 {}", format_in.as_str().to_ascii_uppercase()),
));
}
deadline.check()?;
let retention_rate = effective_rate(compression_rate, level);
// 优先使用直接指定的目标大小，其次根据百分比计算
@@ -446,24 +533,26 @@ fn compress_image_bytes_sync(
})?
} else {
let mut image = decode_image(&input, format_in)?;
deadline.check()?;
image.apply_orientation(orientation);
let (image, did_resize) = resize_if_needed(image, max_width, max_height);
transformed |= did_resize;
deadline.check()?;
match format_out {
ImageFmt::Png => encode_png(image, strength_rate, preserve_metadata)?,
ImageFmt::Jpeg => match encoding_target_size {
Some(target) => encode_jpeg_target(image, target)?,
Some(target) => encode_jpeg_target(image, target, deadline)?,
None => encode_jpeg(image, strength_rate)?,
},
ImageFmt::Webp => match encoding_target_size {
Some(target) => encode_webp_target(image, target)?,
Some(target) => encode_webp_target(image, target, deadline)?,
None => encode_webp(image, strength_rate)?,
},
ImageFmt::Avif => match encoding_target_size {
Some(target) => encode_avif_target(image, target)?,
Some(target) => encode_avif_target(image, target, deadline)?,
None => encode_avif(image, strength_rate)?,
},
ImageFmt::Gif => encode_gif(image, strength_rate)?,
@@ -472,7 +561,6 @@ fn compress_image_bytes_sync(
ImageFmt::Ico => encode_ico(image)?,
}
};
if preserve_metadata {
output = apply_metadata(output, icc_profile, exif)?;
}
@@ -775,28 +863,61 @@ fn encode_avif_raw(raw: &[u8], w: u32, h: u32, quality: u8) -> Result<Vec<u8>, A
Ok(encoded.avif_file)
}
fn encode_jpeg_target(image: DynamicImage, target_size: u64) -> Result<Vec<u8>, AppError> {
encode_with_auto_resize(image, target_size, JPEG_TARGET_MIN_QUALITY, 95, |img, q| {
let rgb = jpeg_rgb(img);
let (w, h) = rgb.dimensions();
encode_jpeg_raw(rgb.as_raw(), w, h, q)
})
fn encode_jpeg_target(
image: DynamicImage,
target_size: u64,
deadline: &CompressionDeadline,
) -> Result<Vec<u8>, AppError> {
encode_with_auto_resize(
image,
target_size,
JPEG_TARGET_MIN_QUALITY,
95,
deadline,
|img, q| {
let rgb = jpeg_rgb(img);
let (w, h) = rgb.dimensions();
encode_jpeg_raw(rgb.as_raw(), w, h, q)
},
)
}
fn encode_webp_target(image: DynamicImage, target_size: u64) -> Result<Vec<u8>, AppError> {
encode_with_auto_resize(image, target_size, WEBP_TARGET_MIN_QUALITY, 95, |img, q| {
let rgba = img.to_rgba8();
let (w, h) = rgba.dimensions();
encode_webp_raw(rgba.as_raw(), w, h, q)
})
fn encode_webp_target(
image: DynamicImage,
target_size: u64,
deadline: &CompressionDeadline,
) -> Result<Vec<u8>, AppError> {
encode_with_auto_resize(
image,
target_size,
WEBP_TARGET_MIN_QUALITY,
95,
deadline,
|img, q| {
let rgba = img.to_rgba8();
let (w, h) = rgba.dimensions();
encode_webp_raw(rgba.as_raw(), w, h, q)
},
)
}
fn encode_avif_target(image: DynamicImage, target_size: u64) -> Result<Vec<u8>, AppError> {
encode_with_auto_resize(image, target_size, AVIF_TARGET_MIN_QUALITY, 95, |img, q| {
let rgba = img.to_rgba8();
let (w, h) = rgba.dimensions();
encode_avif_raw(rgba.as_raw(), w, h, q)
})
fn encode_avif_target(
image: DynamicImage,
target_size: u64,
deadline: &CompressionDeadline,
) -> Result<Vec<u8>, AppError> {
encode_with_auto_resize(
image,
target_size,
AVIF_TARGET_MIN_QUALITY,
95,
deadline,
|img, q| {
let rgba = img.to_rgba8();
let (w, h) = rgba.dimensions();
encode_avif_raw(rgba.as_raw(), w, h, q)
},
)
}
/// 目标体积压缩（质量优先 + 有边界的降尺寸）
@@ -811,18 +932,27 @@ fn encode_with_auto_resize<F>(
target_size: u64,
min_q: u8,
max_q: u8,
deadline: &CompressionDeadline,
mut encode_fn: F,
) -> Result<Vec<u8>, AppError>
where
F: FnMut(&DynamicImage, u8) -> Result<Vec<u8>, AppError>,
{
deadline.check()?;
let (orig_w, orig_h) = image.dimensions();
let long_edge = orig_w.max(orig_h);
let long_edge_floor = TARGET_MIN_LONG_EDGE.min(long_edge) as f64 / long_edge as f64;
let min_scale = TARGET_MIN_SCALE.max(long_edge_floor).min(1.0);
let (full_result, full_dimensions) =
encode_target_at_scale(&image, 1.0, min_q, max_q, target_size, &mut encode_fn)?;
let (full_result, full_dimensions) = encode_target_at_scale(
&image,
1.0,
min_q,
max_q,
target_size,
deadline,
&mut encode_fn,
)?;
if full_result.len() as u64 <= target_size {
return Ok(full_result);
}
@@ -834,8 +964,16 @@ where
));
}
let (min_result, min_dimensions) =
encode_target_at_scale(&image, min_scale, min_q, max_q, target_size, &mut encode_fn)?;
deadline.check()?;
let (min_result, min_dimensions) = encode_target_at_scale(
&image,
min_scale,
min_q,
max_q,
target_size,
deadline,
&mut encode_fn,
)?;
if min_result.len() as u64 > target_size {
return Err(target_unreachable_error(
target_size,
@@ -849,7 +987,10 @@ where
let mut over_scale = 1.0;
let mut over_dimensions = full_dimensions;
for _ in 0..TARGET_SCALE_SEARCH_ATTEMPTS {
for _ in 0..scale_search_attempts(orig_w, orig_h) {
if deadline.expired() {
return Ok(best_under);
}
let candidate_scale = (under_scale + over_scale) / 2.0;
let candidate_dimensions = dimensions_at_scale(orig_w, orig_h, candidate_scale);
@@ -868,6 +1009,7 @@ where
min_q,
max_q,
target_size,
deadline,
&mut encode_fn,
)?;
if candidate_result.len() as u64 <= target_size {
@@ -883,6 +1025,14 @@ where
Ok(best_under)
}
fn scale_search_attempts(width: u32, height: u32) -> usize {
match u64::from(width).saturating_mul(u64::from(height)) {
12_000_000.. => 3,
4_000_000.. => 4,
_ => TARGET_SCALE_SEARCH_ATTEMPTS,
}
}
fn dimensions_at_scale(orig_w: u32, orig_h: u32, scale: f64) -> (u32, u32) {
(
((orig_w as f64 * scale).round() as u32).clamp(1, orig_w),
@@ -896,11 +1046,13 @@ fn encode_target_at_scale<F>(
min_q: u8,
max_q: u8,
target_size: u64,
deadline: &CompressionDeadline,
encode_fn: &mut F,
) -> Result<(Vec<u8>, (u32, u32)), AppError>
where
F: FnMut(&DynamicImage, u8) -> Result<Vec<u8>, AppError>,
{
deadline.check()?;
let (orig_w, orig_h) = image.dimensions();
let dimensions = dimensions_at_scale(orig_w, orig_h, scale);
let resized = (dimensions != (orig_w, orig_h)).then(|| {
@@ -910,8 +1062,16 @@ where
image::imageops::FilterType::Lanczos3,
)
});
deadline.check()?;
let candidate = resized.as_ref().unwrap_or(image);
let result = encode_target_quality_with_image(candidate, min_q, max_q, target_size, encode_fn)?;
let result = encode_target_quality_with_image(
candidate,
min_q,
max_q,
target_size,
deadline,
encode_fn,
)?;
Ok((result, dimensions))
}
@@ -921,11 +1081,13 @@ fn encode_target_quality_with_image<F>(
min_q: u8,
max_q: u8,
target_size: u64,
deadline: &CompressionDeadline,
encode_fn: &mut F,
) -> Result<Vec<u8>, AppError>
where
F: FnMut(&DynamicImage, u8) -> Result<Vec<u8>, AppError>,
{
deadline.check()?;
// Start with the highest quality. If it already fits, no lower-quality
// encodes can improve the result.
let max_quality = encode_fn(image, max_q)?;
@@ -934,6 +1096,7 @@ where
return Ok(max_quality);
}
deadline.check()?;
let min_quality = encode_fn(image, min_q)?;
let min_size = min_quality.len() as u64;
if min_size > target_size {
@@ -950,6 +1113,9 @@ where
// jumps near the target first; the binary phase still proves the exact
// highest fitting integer quality, so output quality is not approximated.
for _ in 0..TARGET_QUALITY_INTERPOLATION_ATTEMPTS {
if deadline.expired() {
return Ok(best_under);
}
if under_q.saturating_add(1) >= over_q {
break;
}
@@ -979,6 +1145,9 @@ where
}
while under_q.saturating_add(1) < over_q {
if deadline.expired() {
return Ok(best_under);
}
let mid = under_q + (over_q - under_q) / 2;
let bytes = encode_fn(image, mid)?;
let size = bytes.len() as u64;
@@ -1372,6 +1541,7 @@ mod tests {
None,
false,
1_000_000,
&CompressionDeadline::unlimited(),
)
.unwrap();
assert_eq!(detect_format(&output).unwrap(), ImageFmt::Webp);
@@ -1406,10 +1576,17 @@ mod tests {
let calls = Cell::new(0);
let image = DynamicImage::new_rgb8(800, 600);
let result = encode_with_auto_resize(image, 100, 40, 95, |_image, quality| {
calls.set(calls.get() + 1);
Ok(vec![0; quality as usize])
})
let result = encode_with_auto_resize(
image,
100,
40,
95,
&CompressionDeadline::unlimited(),
|_image, quality| {
calls.set(calls.get() + 1);
Ok(vec![0; quality as usize])
},
)
.unwrap();
assert_eq!(result.len(), 95);
@@ -1422,12 +1599,18 @@ mod tests {
let calls = RefCell::new(Vec::new());
let image = DynamicImage::new_rgb8(32, 32);
let result =
encode_target_quality_with_image(&image, 25, 95, 50_000, &mut |_image, quality| {
let result = encode_target_quality_with_image(
&image,
25,
95,
50_000,
&CompressionDeadline::unlimited(),
&mut |_image, quality| {
calls.borrow_mut().push(quality);
Ok(vec![0; usize::from(quality) * 1_000])
})
.unwrap();
},
)
.unwrap();
assert_eq!(result.len(), 50_000);
assert_eq!(*calls.borrow(), vec![95, 25, 50, 51]);
@@ -1436,10 +1619,17 @@ mod tests {
#[test]
fn target_encoder_can_reduce_a_landscape_at_the_long_edge_floor() {
let image = DynamicImage::new_rgb8(960, 640);
let result = encode_with_auto_resize(image, 40_000, 40, 40, |image, _quality| {
let (width, height) = image.dimensions();
Ok(vec![0; (width as usize * height as usize) / 10])
})
let result = encode_with_auto_resize(
image,
40_000,
40,
40,
&CompressionDeadline::unlimited(),
|image, _quality| {
let (width, height) = image.dimensions();
Ok(vec![0; (width as usize * height as usize) / 10])
},
)
.unwrap();
assert!(result.len() <= 40_000);
@@ -1455,6 +1645,7 @@ mod tests {
1_000,
40,
40,
&CompressionDeadline::unlimited(),
|_image, _quality| {
calls.set(calls.get() + 1);
Ok(vec![0; 50_000])
@@ -1467,6 +1658,74 @@ mod tests {
assert_eq!(calls.get(), 2);
}
#[test]
fn target_encoder_rejects_work_after_the_deadline() {
use std::cell::Cell;
let calls = Cell::new(0);
let deadline = CompressionDeadline::unlimited();
deadline.cancel();
let error = encode_with_auto_resize(
DynamicImage::new_rgb8(960, 640),
40_000,
40,
40,
&deadline,
|_image, _quality| {
calls.set(calls.get() + 1);
Ok(vec![0; 1])
},
)
.unwrap_err();
assert_eq!(error.code, ErrorCode::CompressionFailed);
assert_eq!(calls.get(), 0);
}
#[test]
fn compression_deadline_reports_no_remaining_time_after_cancellation() {
let deadline = CompressionDeadline::unlimited();
assert_eq!(deadline.remaining(), Duration::MAX);
deadline.cancel();
assert_eq!(deadline.remaining(), Duration::ZERO);
}
#[test]
fn target_encoder_returns_the_best_candidate_when_search_is_cancelled() {
use std::cell::Cell;
let calls = Cell::new(0);
let deadline = CompressionDeadline::unlimited();
let controller = deadline.clone();
let result = encode_with_auto_resize(
DynamicImage::new_rgb8(960, 640),
40_000,
40,
40,
&deadline,
|image, _quality| {
calls.set(calls.get() + 1);
if calls.get() == 2 {
controller.cancel();
}
let (width, height) = image.dimensions();
Ok(vec![0; (width as usize * height as usize) / 10])
},
)
.unwrap();
assert!(result.len() <= 40_000);
assert_eq!(calls.get(), 2);
}
#[test]
fn large_images_use_a_smaller_scale_search_budget() {
assert_eq!(scale_search_attempts(4_000, 3_000), 3);
assert_eq!(scale_search_attempts(2_000, 2_000), 4);
assert_eq!(scale_search_attempts(1_999, 2_000), 6);
}
#[test]
fn png_fast_path_enforces_the_pixel_limit() {
let input = encode_png(DynamicImage::new_rgb8(20, 20), 100, false).unwrap();
@@ -1481,6 +1740,7 @@ mod tests {
None,
true,
399,
&CompressionDeadline::unlimited(),
)
.unwrap_err();
@@ -1506,6 +1766,7 @@ mod tests {
None,
false,
1_000_000,
&CompressionDeadline::unlimited(),
)
.unwrap();
let decoded = image::load_from_memory(&output).unwrap();
@@ -1564,6 +1825,7 @@ mod tests {
None,
false,
1_000_000,
&CompressionDeadline::unlimited(),
)
.unwrap();