From 0ff9eae56d7b86657b2d3405e60ec3068110ee86 Mon Sep 17 00:00:00 2001 From: 237899745 <237899745@users.noreply.git.workyai.cn> Date: Sat, 25 Jul 2026 11:20:45 +0800 Subject: [PATCH] feat: optimize compression and production deployment --- .dockerignore | 2 + .gitignore | 1 + Cargo.lock | 17 +- Cargo.toml | 1 + README.md | 12 +- docker/.env.production.example | 45 ++ docker/Dockerfile | 28 +- docker/docker-compose.prod.yml | 128 +++++ docs/api.md | 6 +- docs/deployment.md | 775 ++++------------------------- frontend/src/pages/DocsPage.vue | 2 +- frontend/src/pages/HomePage.vue | 4 +- scripts/benchmark_compression.py | 404 +++++++++++++++ scripts/requirements-benchmark.txt | 2 + src/services/compress.rs | 226 ++++++--- 15 files changed, 882 insertions(+), 771 deletions(-) create mode 100644 docker/.env.production.example create mode 100644 docker/docker-compose.prod.yml create mode 100644 scripts/benchmark_compression.py create mode 100644 scripts/requirements-benchmark.txt diff --git a/.dockerignore b/.dockerignore index 5d813ac..8fd6d29 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,7 @@ .git .env +.env.* +.bench target uploads frontend/node_modules diff --git a/.gitignore b/.gitignore index f8834f1..bc21a98 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ /uploads /static /logs +/.bench /frontend/node_modules /frontend/dist diff --git a/Cargo.lock b/Cargo.lock index 7b41938..86b862a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -982,7 +982,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1642,6 +1642,7 @@ dependencies = [ "hmac", "image", "img-parts", + "jpeg-encoder", "jsonwebtoken", "lettre", "oxipng", @@ -1762,6 +1763,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-encoder" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b0b36cbb4e6704f12f5b5d7b01dac593982c6550859ebd5a66fb15c9ea27fd5" + [[package]] name = "js-sys" version = "0.3.83" @@ -2130,7 +2137,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2553,7 +2560,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -2870,7 +2877,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3550,7 +3557,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e56392c..e1f0d06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ aes-gcm = "0.10" # Keep image-rs limited to formats exposed by the API. AVIF decoding uses # libdav1d on Linux; encoding is handled by the direct ravif dependency. image = { version = "0.25", default-features = false, features = ["bmp", "gif", "ico", "jpeg", "png", "tiff", "webp"] } +jpeg-encoder = "0.7" oxipng = "9" ravif = { version = "0.11", default-features = false, features = ["threading"] } webp = { version = "0.3", default-features = false } diff --git a/README.md b/README.md index ae65841..0605e69 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ - **图片压缩**：支持 PNG/JPG/JPEG/WebP/AVIF/GIF/BMP/TIFF/ICO（GIF 仅静态，支持格式转换） - **批量处理**：支持多图片同时上传和处理 -- **压缩率**：1-100（数值越小压缩越强，100 为不压缩） +- **压缩率**：1-100（JPEG/WebP/AVIF 以该比例为体积上限；无损格式按安全方式尽力优化） - **用户系统**：注册、登录、API Key 管理 - **计费与用量**：套餐/订阅/配额/发票 - **管理员后台**：用户管理、系统监控、配置管理 @@ -84,7 +84,7 @@ imageforge/ - Rust（建议使用最新 stable；当前依赖链要求较新的 Rust，建议 `>= 1.85`） - PostgreSQL 16+ - Redis 7+ -- Node.js 20+（前端构建） +- Node.js 22+（前端构建） ### 本地开发 @@ -107,10 +107,14 @@ cd frontend && npm run dev ### Docker 部署 ```bash -# 该 compose 仅包含 postgres/redis，服务本体请按 docs/deployment.md 构建运行 -docker compose -f docker/docker-compose.dev.yml up -d +cp docker/.env.production.example .env.production +# 修改全部密钥与 PUBLIC_BASE_URL 后启动完整生产栈 +docker compose --env-file .env.production -f docker/docker-compose.prod.yml up -d --build +curl --fail http://127.0.0.1:8080/health ``` +详细配置、更新、备份与真实图片质量回归见 [部署指南](./docs/deployment.md)。 + ## 文档索引 - [开工前确认清单](./docs/confirm.md) diff --git a/docker/.env.production.example b/docker/.env.production.example new file mode 100644 index 0000000..595b992 --- /dev/null +++ b/docker/.env.production.example @@ -0,0 +1,45 @@ +# Image tag built by docker/docker-compose.prod.yml +IMAGEFORGE_TAG=local + +# Public listener and URL +IMAGEFORGE_BIND_ADDRESS=0.0.0.0 +IMAGEFORGE_PORT=8080 +PUBLIC_BASE_URL=http://192.0.2.10:8080 + +# Replace every secret before starting the stack. +POSTGRES_PASSWORD=replace-with-a-long-random-password +JWT_SECRET=replace-with-at-least-32-random-bytes +API_KEY_PEPPER=replace-with-an-independent-random-secret + +# Initial administrator created on first startup. +ADMIN_EMAIL=admin@example.com +ADMIN_USERNAME=admin +ADMIN_PASSWORD=replace-with-a-strong-admin-password + +# A four-core host should start with two image jobs per process. +DATABASE_MAX_CONNECTIONS=10 +WORKER_CONCURRENCY=2 +IMAGE_PROCESSING_CONCURRENCY=2 + +ALLOW_ANONYMOUS_UPLOAD=true +ANON_MAX_FILE_SIZE_MB=5 +ANON_MAX_FILES_PER_BATCH=5 +ANON_DAILY_UNITS=10 +ANON_RETENTION_HOURS=24 +MAX_IMAGE_PIXELS=40000000 +IDEMPOTENCY_TTL_HOURS=24 + +# Enable only when the API port is reachable exclusively through a trusted proxy. +TRUST_PROXY_HEADERS=false +MAIL_ENABLED=false +MAIL_LOG_LINKS_WHEN_DISABLED=false +# STRIPE_SECRET_KEY=sk_live_replace_me +# STRIPE_WEBHOOK_SECRET=whsec_replace_me +# MAIL_PROVIDER=custom +# MAIL_FROM=noreply@example.com +# MAIL_PASSWORD=replace-with-smtp-authorization-code +# MAIL_FROM_NAME=ImageForge +# MAIL_SMTP_HOST=smtp.example.com +# MAIL_SMTP_PORT=465 +# MAIL_SMTP_ENCRYPTION=ssl +RUST_LOG=info,tower_http=info,imageforge=info diff --git a/docker/Dockerfile b/docker/Dockerfile index 103c4cf..a30ee79 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,6 @@ -FROM rust:1.92-bookworm AS builder +# syntax=docker/dockerfile:1.7 + +FROM rust:1.92-trixie AS builder WORKDIR /app @@ -14,35 +16,45 @@ COPY src ./src COPY migrations ./migrations COPY templates ./templates -RUN cargo build --release +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/app/target,sharing=locked \ + cargo build --release --locked \ + && cp /app/target/release/imageforge /app/imageforge -FROM node:20-alpine AS frontend-builder +FROM node:22-alpine AS frontend-builder WORKDIR /app/frontend COPY frontend/package*.json ./ -RUN npm ci +RUN --mount=type=cache,target=/root/.npm,sharing=locked npm ci COPY frontend ./ RUN npm run build -FROM debian:bookworm-slim +FROM debian:trixie-slim RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ - libdav1d6 \ + curl \ + libdav1d7 \ && rm -rf /var/lib/apt/lists/* +RUN groupadd --system --gid 10001 imageforge \ + && useradd --system --uid 10001 --gid imageforge --home-dir /app --shell /usr/sbin/nologin imageforge + WORKDIR /app -COPY --from=builder /app/target/release/imageforge ./imageforge +COPY --from=builder /app/imageforge ./imageforge COPY --from=frontend-builder /app/frontend/dist ./static COPY migrations ./migrations -RUN mkdir -p uploads +RUN mkdir -p uploads \ + && chown imageforge:imageforge uploads ENV HOST=0.0.0.0 ENV PORT=8080 EXPOSE 8080 +USER 10001:10001 + CMD ["./imageforge"] diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml new file mode 100644 index 0000000..f55be87 --- /dev/null +++ b/docker/docker-compose.prod.yml @@ -0,0 +1,128 @@ +name: imageforge + +x-imageforge-environment: &imageforge-environment + DATABASE_URL: postgres://imageforge:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}@postgres:5432/imageforge + DATABASE_MAX_CONNECTIONS: ${DATABASE_MAX_CONNECTIONS:-10} + REDIS_URL: redis://redis:6379 + JWT_SECRET: ${JWT_SECRET:?JWT_SECRET is required} + JWT_EXPIRY_HOURS: ${JWT_EXPIRY_HOURS:-168} + API_KEY_PEPPER: ${API_KEY_PEPPER:?API_KEY_PEPPER is required} + BILLING_PROVIDER: ${BILLING_PROVIDER:-stripe} + STORAGE_TYPE: local + STORAGE_PATH: /app/uploads + PUBLIC_BASE_URL: ${PUBLIC_BASE_URL:-http://localhost:8080} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + IMAGE_PROCESSING_CONCURRENCY: ${IMAGE_PROCESSING_CONCURRENCY:-2} + ALLOW_ANONYMOUS_UPLOAD: ${ALLOW_ANONYMOUS_UPLOAD:-true} + ANON_MAX_FILE_SIZE_MB: ${ANON_MAX_FILE_SIZE_MB:-5} + ANON_MAX_FILES_PER_BATCH: ${ANON_MAX_FILES_PER_BATCH:-5} + ANON_DAILY_UNITS: ${ANON_DAILY_UNITS:-10} + ANON_RETENTION_HOURS: ${ANON_RETENTION_HOURS:-24} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + IDEMPOTENCY_TTL_HOURS: ${IDEMPOTENCY_TTL_HOURS:-24} + TRUST_PROXY_HEADERS: ${TRUST_PROXY_HEADERS:-false} + RUST_LOG: ${RUST_LOG:-info,tower_http=info,imageforge=info} + +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_DB: imageforge + POSTGRES_USER: imageforge + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U imageforge -d imageforge"] + interval: 5s + timeout: 3s + retries: 20 + restart: unless-stopped + + redis: + image: redis:7-alpine + command: ["redis-server", "--appendonly", "yes", "--maxmemory-policy", "noeviction"] + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 20 + restart: unless-stopped + + api: + image: imageforge:${IMAGEFORGE_TAG:-local} + build: + context: .. + dockerfile: docker/Dockerfile + init: true + environment: + <<: *imageforge-environment + IMAGEFORGE_ROLE: api + ADMIN_EMAIL: ${ADMIN_EMAIL:-} + ADMIN_USERNAME: ${ADMIN_USERNAME:-} + ADMIN_PASSWORD: ${ADMIN_PASSWORD:-} + STRIPE_SECRET_KEY: "${STRIPE_SECRET_KEY:-}" + STRIPE_WEBHOOK_SECRET: "${STRIPE_WEBHOOK_SECRET:-}" + MAIL_ENABLED: ${MAIL_ENABLED:-false} + MAIL_LOG_LINKS_WHEN_DISABLED: ${MAIL_LOG_LINKS_WHEN_DISABLED:-false} + MAIL_PROVIDER: ${MAIL_PROVIDER:-qq} + MAIL_FROM: ${MAIL_FROM:-noreply@example.com} + MAIL_PASSWORD: "${MAIL_PASSWORD:-}" + MAIL_FROM_NAME: ${MAIL_FROM_NAME:-ImageForge} + MAIL_SMTP_HOST: "${MAIL_SMTP_HOST:-}" + MAIL_SMTP_PORT: "${MAIL_SMTP_PORT:-}" + MAIL_SMTP_ENCRYPTION: "${MAIL_SMTP_ENCRYPTION:-}" + ports: + - "${IMAGEFORGE_BIND_ADDRESS:-0.0.0.0}:${IMAGEFORGE_PORT:-8080}:8080" + volumes: + - uploads:/app/uploads + tmpfs: + - /tmp:size=256m,mode=1777 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:8080/health"] + interval: 10s + timeout: 3s + retries: 12 + start_period: 20s + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + read_only: true + restart: unless-stopped + + worker: + image: imageforge:${IMAGEFORGE_TAG:-local} + init: true + environment: + <<: *imageforge-environment + IMAGEFORGE_ROLE: worker + volumes: + - uploads:/app/uploads + tmpfs: + - /tmp:size=256m,mode=1777 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + api: + condition: service_healthy + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + read_only: true + restart: unless-stopped + +volumes: + postgres_data: + redis_data: + uploads: diff --git a/docs/api.md b/docs/api.md index 8239eab..d68dd18 100644 --- a/docs/api.md +++ b/docs/api.md @@ -272,9 +272,9 @@ Idempotency-Key: # 建议 | 字段 | 类型 | 必填 | 说明 | |---|---|---:|---| | `file` | File | 是 | 图片文件 | -| `compression_rate` | Integer | 否 | 压缩率 1-100（压缩后体积占原图比例，数值越小压缩越强，100 表示不压缩），优先级高于 `level` | +| `compression_rate` | Integer | 否 | 压缩率 1-100；JPEG/WebP/AVIF 以该比例为体积上限，无损格式为尽力优化，100 表示不压缩；优先级高于 `level` | | `level` | String | 否 | `high` / `medium` / `low`（兼容参数，默认 `medium`） | -| `output_format` | String | 否 | 输出格式：`png/jpeg/webp/avif/gif/bmp/tiff/ico`（默认保持原格式） | +| `output_format` | String | 否 | 输出格式：`png/jpeg/webp/avif/gif/bmp/tiff/ico`（默认保持原格式；ICO 自动等比缩至 256x256 边界） | | `max_width` | Integer | 否 | 最大宽度（等比缩放） | | `max_height` | Integer | 否 | 最大高度（等比缩放） | | `target_size_bytes` | Integer | 否 | 目标体积（字节），仅 `jpeg/webp/avif` 输出支持；会优先保清晰度并在必要时小幅缩放 | @@ -340,7 +340,7 @@ Idempotency-Key: # 建议 | 字段 | 类型 | 必填 | 说明 | |---|---|---:|---| | `files[]` | File[] | 是 | 图片文件数组（上限由套餐决定） | -| `compression_rate` | Integer | 否 | 压缩率 1-100（压缩后体积占原图比例，数值越小压缩越强，100 表示不压缩），优先级高于 `level` | +| `compression_rate` | Integer | 否 | 压缩率 1-100；JPEG/WebP/AVIF 以该比例为体积上限，无损格式为尽力优化，100 表示不压缩；优先级高于 `level` | | `level` | String | 否 | `high` / `medium` / `low`（兼容参数） | | `output_format` | String | 否 | 输出格式：`png/jpeg/webp/avif/gif/bmp/tiff/ico`（默认保持原格式） | | `preserve_metadata` | Boolean | 否 | 是否保留元数据（默认 `false`） | diff --git a/docs/deployment.md b/docs/deployment.md index 87b08c5..87340ae 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,714 +1,155 @@ # 部署指南 -## 环境准备 - -### 系统要求 - -- Linux (Ubuntu 22.04+ / Debian 12+ 推荐) -- 2+ CPU 核心（启用独立 Worker 建议 4+） -- 4GB+ 内存 -- 50GB+ 磁盘空间 - -### 依赖安装 - -```bash -# Ubuntu/Debian -sudo apt update -sudo apt install -y \ - build-essential \ - pkg-config \ - libssl-dev \ - libpq-dev \ - cmake \ - libdav1d-dev \ - nasm \ - libjpeg-dev \ - libpng-dev \ - libwebp-dev - -# 安装 Rust -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -source ~/.cargo/env - -# 初始化数据库会用到 psql（建议安装 PostgreSQL client） -sudo apt install -y postgresql-client - -# 安装 Node.js (前端构建) -curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - -sudo apt install -y nodejs -``` - ---- - -## 本地开发 - -### 1. 启动数据库服务 - -```bash -# 使用 Docker Compose 启动 PostgreSQL 和 Redis -docker-compose -f docker/docker-compose.dev.yml up -d -``` - -`docker/docker-compose.dev.yml`: -```yaml -version: '3.8' - -services: - postgres: - image: postgres:16-alpine - environment: - POSTGRES_USER: imageforge - POSTGRES_PASSWORD: devpassword - POSTGRES_DB: imageforge - ports: - - "5432:5432" - volumes: - - postgres_data:/var/lib/postgresql/data - - redis: - image: redis:7-alpine - ports: - - "6379:6379" - volumes: - - redis_data:/data - - # 规划：接入 S3 存储后可增加 MinIO；当前版本仅支持本地存储 - # minio: - # image: minio/minio:RELEASE.2024-01-28T20-20-01Z - # command: server /data --console-address ":9001" - # environment: - # MINIO_ROOT_USER: minioadmin - # MINIO_ROOT_PASSWORD: minioadmin - # ports: - # - "9000:9000" - # - "9001:9001" - # volumes: - # - minio_data:/data - -volumes: - postgres_data: - redis_data: - minio_data: -``` - -### 2. 配置环境变量 - -```bash -cp .env.example .env -``` - -`.env.example`: -```bash -# 运行模式：建议将 API 与 Worker 分开运行 -IMAGEFORGE_ROLE=api # api | worker - -# 服务配置 -HOST=0.0.0.0 -PORT=8080 -PUBLIC_BASE_URL=http://localhost:8080 -RUST_LOG=info,imageforge=debug - -# 数据库 -DATABASE_URL=postgres://imageforge:devpassword@localhost:5432/imageforge - -# Redis -REDIS_URL=redis://localhost:6379 - -# Worker 并发（每个批量任务内同时处理的文件数） -WORKER_CONCURRENCY=4 - -# 图片处理全局并发（每个 API/Worker 进程） -IMAGE_PROCESSING_CONCURRENCY=4 - -# 仅在后端端口不对公网开放、请求必经可信代理时启用 -TRUST_PROXY_HEADERS=false - -# JWT（网站/管理后台） -JWT_SECRET=your-super-secret-key-change-in-production -JWT_EXPIRY_HOURS=168 - -# API Key -API_KEY_PEPPER=please-change-this-in-production - -# 存储（当前实现仅支持 local） -STORAGE_TYPE=local -STORAGE_PATH=./uploads - -# 计费（已确认：Stripe） -BILLING_PROVIDER=stripe -STRIPE_SECRET_KEY=sk_test_xxx -STRIPE_WEBHOOK_SECRET=whsec_xxx - -# 限制（默认值；最终以套餐/用户覆盖为准） -ALLOW_ANONYMOUS_UPLOAD=true -ANON_MAX_FILE_SIZE_MB=5 -ANON_MAX_FILES_PER_BATCH=5 -ANON_DAILY_UNITS=10 -MAX_IMAGE_PIXELS=40000000 -IDEMPOTENCY_TTL_HOURS=24 - -# 结果保留（匿名默认；登录用户按套餐 retention_days） -ANON_RETENTION_HOURS=24 - -# 管理员初始账户 -ADMIN_EMAIL=admin@example.com -ADMIN_PASSWORD=changeme123 -``` - -### 3. 初始化数据库 - -API 或 Worker 启动时会通过 SQLx 自动、顺序执行 `migrations/` 下尚未应用的迁移。迁移失败时进程会退出，不会在不完整的数据库结构上继续提供服务。 - -### 4. 启动开发服务器 - -```bash -# 后端 API (热重载) -cargo install cargo-watch -IMAGEFORGE_ROLE=api cargo watch -x run - -# 后端 Worker（另一个终端，处理异步/批量任务） -IMAGEFORGE_ROLE=worker cargo watch -x run - -# 前端 (另一个终端) -cd frontend -npm install -npm run dev -``` - -### 5. Stripe Webhook 本地调试（可选） - -本地调试 Stripe 订阅/支付状态，通常需要将 Stripe Webhook 转发到本机： - -```bash -# 1) 安装并登录 Stripe CLI（按官方文档） -# 2) 监听并转发到你的后端回调地址 -stripe listen --forward-to http://localhost:8080/api/v1/webhooks/stripe - -# CLI 会输出一个 whsec_...，写入 .env 的 STRIPE_WEBHOOK_SECRET -``` - ---- - ## 生产部署 -> 注意：以下内容为生产部署模板示例；仓库当前首期仅提供开发用 `docker/docker-compose.dev.yml`，生产 compose/Dockerfile/nginx/k8s 等可在开工阶段按需落地并调整。 +仓库提供完整的 `docker/Dockerfile` 与 `docker/docker-compose.prod.yml`。生产编排包含 API、Worker、PostgreSQL 和 Redis；数据库与 Redis 不发布宿主机端口，API 和 Worker 使用同一上传卷，并以非 root、只读根文件系统运行。 -### 方案一：Docker Compose（推荐小规模） +### 环境要求 -`docker/docker-compose.prod.yml`: -```yaml -version: '3.8' +- Linux x86_64 +- Docker Engine 26+ 与 Docker Compose 2.20+ +- 最低 2 核 CPU、4GB 内存；启用 AVIF 和独立 Worker 时建议 4 核、8GB 内存 +- 首次构建可访问 Docker Hub 与 crates.io -services: - api: - build: - context: .. - dockerfile: docker/Dockerfile - environment: - - IMAGEFORGE_ROLE=api - - TRUST_PROXY_HEADERS=true - - BILLING_PROVIDER=stripe - - PUBLIC_BASE_URL=https://your-domain.com - - STRIPE_SECRET_KEY=${STRIPE_SECRET_KEY} - - STRIPE_WEBHOOK_SECRET=${STRIPE_WEBHOOK_SECRET} - - DATABASE_URL=postgres://imageforge:${DB_PASSWORD}@postgres:5432/imageforge - - REDIS_URL=redis://redis:6379 - - JWT_SECRET=${JWT_SECRET} - - API_KEY_PEPPER=${API_KEY_PEPPER} - - STORAGE_TYPE=local - - STORAGE_PATH=/app/uploads - expose: - - "8080" - volumes: - - uploads:/app/uploads - depends_on: - - postgres - - redis - restart: unless-stopped +Debian 13、4 核 CPU、8GB 内存的实测起始值为 `WORKER_CONCURRENCY=2` 和 `IMAGE_PROCESSING_CONCURRENCY=2`。AVIF 是 CPU 密集型编码，不要直接把并发设置为 CPU 核数的数倍。 - worker: - build: - context: .. - dockerfile: docker/Dockerfile - environment: - - IMAGEFORGE_ROLE=worker - - DATABASE_URL=postgres://imageforge:${DB_PASSWORD}@postgres:5432/imageforge - - REDIS_URL=redis://redis:6379 - - JWT_SECRET=${JWT_SECRET} - - API_KEY_PEPPER=${API_KEY_PEPPER} - - STORAGE_TYPE=local - - STORAGE_PATH=/app/uploads - volumes: - - uploads:/app/uploads - depends_on: - - postgres - - redis - restart: unless-stopped +### 首次启动 - postgres: - image: postgres:16-alpine - environment: - POSTGRES_USER: imageforge - POSTGRES_PASSWORD: ${DB_PASSWORD} - POSTGRES_DB: imageforge - volumes: - - postgres_data:/var/lib/postgresql/data - restart: unless-stopped +```bash +cp docker/.env.production.example .env.production +chmod 600 .env.production - redis: - image: redis:7-alpine - volumes: - - redis_data:/data - restart: unless-stopped +# 分别生成 POSTGRES_PASSWORD、JWT_SECRET、API_KEY_PEPPER 和管理员密码。 +# hex 不包含数据库 URL 与 .env 需要转义的保留字符。 +openssl rand -hex 32 - nginx: - image: nginx:alpine - ports: - - "80:80" - - "443:443" - volumes: - - ./nginx.conf:/etc/nginx/nginx.conf:ro - - /etc/letsencrypt:/etc/letsencrypt:ro - depends_on: - - api - restart: unless-stopped +# 编辑公开地址和全部 replace-with-* 值后检查配置。 +docker compose \ + --env-file .env.production \ + -f docker/docker-compose.prod.yml \ + config --quiet -volumes: - uploads: - postgres_data: - redis_data: +docker compose \ + --env-file .env.production \ + -f docker/docker-compose.prod.yml \ + up -d --build ``` -`docker/Dockerfile`: -```dockerfile -FROM rust:1.92-bookworm AS builder +API 健康后 Worker 才会启动，避免两个进程在首次部署时同时执行迁移。 -WORKDIR /app - -RUN apt-get update && apt-get install -y --no-install-recommends \ - cmake \ - libdav1d-dev \ - nasm \ - pkg-config \ - && rm -rf /var/lib/apt/lists/* - -COPY Cargo.toml Cargo.lock ./ -COPY src ./src -COPY migrations ./migrations -COPY templates ./templates - -RUN cargo build --release - -# 前端构建阶段 -FROM node:20-alpine AS frontend-builder - -WORKDIR /app/frontend -COPY frontend/package*.json ./ -RUN npm ci -COPY frontend ./ -RUN npm run build - -# 运行阶段 -FROM debian:bookworm-slim - -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - libdav1d6 \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -COPY --from=builder /app/target/release/imageforge ./imageforge -COPY --from=frontend-builder /app/frontend/dist ./static -COPY migrations ./migrations - -RUN mkdir -p uploads - -ENV HOST=0.0.0.0 -ENV PORT=8080 - -EXPOSE 8080 - -CMD ["./imageforge"] +```bash +docker compose --env-file .env.production -f docker/docker-compose.prod.yml ps +curl --fail http://127.0.0.1:8080/health ``` -`docker/nginx.conf`: +预期健康响应： + +```json +{"status":"healthy","database":"connected","redis":"connected"} +``` + +### 更新与回滚 + +更新代码后保留 `.env.production` 和命名卷，重新构建并滚动重建： + +```bash +git pull --ff-only +docker compose --env-file .env.production -f docker/docker-compose.prod.yml build api +docker compose --env-file .env.production -f docker/docker-compose.prod.yml up -d +``` + +生产镜像应使用不可变的 `IMAGEFORGE_TAG`。回滚时把该值改回上一镜像标签，然后再次运行 `up -d`。 + +### 反向代理 + +直接通过服务器地址访问时保持 `TRUST_PROXY_HEADERS=false`。只有当 8080 端口不对客户端开放、所有请求都经过可信反向代理时，才设置为 `true`，并由代理覆盖 `X-Forwarded-For` 与 `X-Forwarded-Proto`。 + +代理至少需要： + ```nginx -events { - worker_connections 1024; -} - -http { - include mime.types; - default_type application/octet-stream; - - # 日志 - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - - # 文件上传大小限制 - client_max_body_size 100M; - - # Gzip - gzip on; - gzip_types text/plain text/css application/json application/javascript; - - upstream backend { - server api:8080; - } - - server { - listen 80; - server_name your-domain.com; - return 301 https://$server_name$request_uri; - } - - server { - listen 443 ssl http2; - server_name your-domain.com; - - ssl_certificate /etc/letsencrypt/live/your-domain.com/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/your-domain.com/privkey.pem; - - # SSL 配置 - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256; - ssl_prefer_server_ciphers off; - - # 静态文件 - location /static/ { - proxy_pass http://backend; - expires 30d; - add_header Cache-Control "public, immutable"; - } - - # WebSocket - location /ws/ { - proxy_pass http://backend; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; - proxy_read_timeout 86400; - } - - # API 和其他请求 - location / { - proxy_pass http://backend; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header X-Forwarded-Proto $scheme; - } - } +location / { + proxy_pass http://127.0.0.1:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + client_max_body_size 100m; + proxy_read_timeout 300s; } ``` -### 部署步骤 +### 日志与备份 ```bash -# 1. 创建 .env 文件 -cat > .env << EOF -DB_PASSWORD=your-secure-db-password -JWT_SECRET=your-very-long-random-jwt-secret-at-least-32-chars -STRIPE_SECRET_KEY=sk_live_xxx -STRIPE_WEBHOOK_SECRET=whsec_xxx -EOF +docker compose --env-file .env.production -f docker/docker-compose.prod.yml logs -f api worker -# 2. 获取 SSL 证书 -sudo certbot certonly --standalone -d your-domain.com - -# 3. 构建并启动 -docker-compose -f docker/docker-compose.prod.yml up -d --build - -# 4. 查看日志 -docker-compose -f docker/docker-compose.prod.yml logs -f api +docker compose --env-file .env.production -f docker/docker-compose.prod.yml \ + exec -T postgres pg_dump -U imageforge imageforge | gzip > imageforge.sql.gz ``` ---- +除数据库外，还要备份 Compose 的 `uploads` 命名卷。恢复前停止 API 与 Worker，避免数据库记录与文件卷产生时间差。 -### 方案二：Kubernetes（大规模） - -`k8s/deployment.yaml`: -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: imageforge -spec: - replicas: 3 - selector: - matchLabels: - app: imageforge - template: - metadata: - labels: - app: imageforge - spec: - containers: - - name: imageforge - image: your-registry/imageforge:latest - ports: - - containerPort: 8080 - env: - - name: IMAGEFORGE_ROLE - value: api - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: imageforge-secrets - key: database-url - - name: REDIS_URL - valueFrom: - secretKeyRef: - name: imageforge-secrets - key: redis-url - - name: JWT_SECRET - valueFrom: - secretKeyRef: - name: imageforge-secrets - key: jwt-secret - resources: - requests: - memory: "512Mi" - cpu: "500m" - limits: - memory: "2Gi" - cpu: "2000m" - livenessProbe: - httpGet: - path: /health - port: 8080 - initialDelaySeconds: 10 - periodSeconds: 30 - readinessProbe: - httpGet: - path: /health - port: 8080 - initialDelaySeconds: 5 - periodSeconds: 10 ---- -apiVersion: v1 -kind: Service -metadata: - name: imageforge -spec: - selector: - app: imageforge - ports: - - port: 80 - targetPort: 8080 - type: ClusterIP ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: imageforge - annotations: - kubernetes.io/ingress.class: nginx - cert-manager.io/cluster-issuer: letsencrypt-prod -spec: - tls: - - hosts: - - your-domain.com - secretName: imageforge-tls - rules: - - host: your-domain.com - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: imageforge - port: - number: 80 - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: imageforge-worker -spec: - replicas: 2 - selector: - matchLabels: - app: imageforge-worker - template: - metadata: - labels: - app: imageforge-worker - spec: - containers: - - name: imageforge-worker - image: your-registry/imageforge:latest - env: - - name: IMAGEFORGE_ROLE - value: worker - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: imageforge-secrets - key: database-url - - name: REDIS_URL - valueFrom: - secretKeyRef: - name: imageforge-secrets - key: redis-url - - name: JWT_SECRET - valueFrom: - secretKeyRef: - name: imageforge-secrets - key: jwt-secret - resources: - requests: - memory: "512Mi" - cpu: "500m" - limits: - memory: "4Gi" - cpu: "4000m" -``` - ---- - -## 监控与日志 - -### Prometheus 指标 - -应用暴露 `/metrics` 端点： - -```rust -// 在代码中添加指标 -use prometheus::{Counter, Histogram}; - -lazy_static! { - static ref COMPRESSION_REQUESTS: Counter = Counter::new( - "imageforge_compression_requests_total", - "Total number of compression requests" - ).unwrap(); - - static ref COMPRESSION_DURATION: Histogram = Histogram::with_opts( - HistogramOpts::new( - "imageforge_compression_duration_seconds", - "Time spent compressing images" - ) - ).unwrap(); -} -``` - -### Grafana 仪表板 - -监控项目： -- 请求量 / QPS -- 响应时间 P50/P95/P99 -- 错误率 -- 压缩任务队列长度 -- CPU / 内存使用率 -- 磁盘使用率 - -### 日志聚合 - -使用 ELK Stack 或 Loki： - -```yaml -# docker-compose 添加 Loki -loki: - image: grafana/loki:2.9.0 - ports: - - "3100:3100" - command: -config.file=/etc/loki/local-config.yaml - -promtail: - image: grafana/promtail:2.9.0 - volumes: - - /var/log:/var/log - - ./promtail-config.yml:/etc/promtail/config.yml - command: -config.file=/etc/promtail/config.yml -``` - ---- - -## 备份策略 - -### 数据库备份 +## 本地开发 ```bash -#!/bin/bash -# backup.sh +cp .env.example .env +docker compose -f docker/docker-compose.dev.yml up -d -DATE=$(date +%Y%m%d_%H%M%S) -BACKUP_DIR=/backups +# 终端 1 +IMAGEFORGE_ROLE=api cargo run -# PostgreSQL 备份 -docker exec postgres pg_dump -U imageforge imageforge | gzip > $BACKUP_DIR/db_$DATE.sql.gz +# 终端 2 +IMAGEFORGE_ROLE=worker cargo run -# 保留最近 7 天的备份 -find $BACKUP_DIR -name "db_*.sql.gz" -mtime +7 -delete - -# 可选：上传到 S3 -# aws s3 cp $BACKUP_DIR/db_$DATE.sql.gz s3://your-bucket/backups/ +# 终端 3 +cd frontend +npm ci +npm run dev ``` -添加到 crontab： +API 或 Worker 启动时会通过 SQLx 顺序执行尚未应用的迁移。迁移失败时进程退出，不会在不完整的数据库结构上继续提供服务。 + +## 压缩质量回归 + +基准工具会下载固定 Picsum 照片，分别生成 JPEG、PNG、WebP、AVIF 输入，再通过真实 HTTP API 测量目标达标率、格式签名、SSIM、PSNR、分辨率和耗时。 + ```bash -0 3 * * * /path/to/backup.sh +python3 -m venv .venv-benchmark +. .venv-benchmark/bin/activate +pip install -r scripts/requirements-benchmark.txt + +python scripts/benchmark_compression.py generate --output-dir .bench/corpus + +IMAGEFORGE_BENCH_EMAIL=admin@example.com \ +IMAGEFORGE_BENCH_PASSWORD='replace-me' \ +python scripts/benchmark_compression.py run \ + --base-url http://127.0.0.1:8080 \ + --input-dir .bench/corpus \ + --output-dir .bench/results \ + --formats jpeg,webp,avif \ + --rates 30,50,70 \ + --strict ``` -### 上传文件备份 - -如果使用本地存储，定期同步到 S3： -```bash -aws s3 sync /app/uploads s3://your-bucket/uploads --delete -``` - ---- +`--strict` 会在请求失败、输出格式不匹配或目标体积未达标时返回非零退出码。测试素材与结果位于 `.bench/`，不会提交到 Git。 ## 故障排查 -### 常见问题 - -**1. 数据库连接失败** ```bash -# 检查 PostgreSQL 状态 -docker-compose logs postgres +# 容器与健康状态 +docker compose --env-file .env.production -f docker/docker-compose.prod.yml ps -# 测试连接 -docker exec -it postgres psql -U imageforge -d imageforge -c "SELECT 1" -``` +# 最近日志 +docker compose --env-file .env.production -f docker/docker-compose.prod.yml logs --tail=200 api worker -**2. 压缩失败** -```bash -# 检查应用日志 -docker-compose logs api | grep ERROR -docker-compose logs worker | grep ERROR +# 数据库迁移状态 +docker compose --env-file .env.production -f docker/docker-compose.prod.yml \ + exec -T postgres psql -U imageforge -d imageforge \ + -c 'SELECT version, success FROM _sqlx_migrations ORDER BY version;' -# 检查磁盘空间 +# 主机资源 +docker stats df -h ``` -**3. 内存不足** -```bash -# 查看内存使用 -docker stats - -# 调整容器内存限制 -``` - -**4. 上传超时** -```bash -# 检查 Nginx 配置 -# client_max_body_size 和 proxy_read_timeout -``` - -### 健康检查端点 - -``` -GET /health -{ - "status": "healthy", - "database": "connected", - "redis": "connected", - "storage": "available", - "uptime": 3600 -} -``` +若图片压缩长时间排队，先检查 CPU，再下调 `IMAGE_PROCESSING_CONCURRENCY`；若 API 健康但批量任务不推进，检查 Worker 日志和 Redis 状态。 diff --git a/frontend/src/pages/DocsPage.vue b/frontend/src/pages/DocsPage.vue index 7013620..f9b9005 100644 --- a/frontend/src/pages/DocsPage.vue +++ b/frontend/src/pages/DocsPage.vue @@ -20,7 +20,7 @@
  • Base URL：https://ys.workyai.cn/api/v1
  • 认证方式：X-API-Key（推荐）或 Authorization: Bearer <token>
  • 支持格式：PNG / JPG / JPEG / WebP / AVIF / GIF（静态）/ BMP / TIFF / ICO（支持 output_format 转码）
    • -
  • 压缩率：compression_rate 1-100，表示压缩后体积占原图比例，100 为不压缩
    • +
  • 压缩率：compression_rate 1-100；JPEG/WebP/AVIF 以该比例为体积上限，无损格式为尽力优化
  • 计量：成功压缩 1 个文件计 1 次；若体积未变小或压缩率为 100，则不扣额度
    • diff --git a/frontend/src/pages/HomePage.vue b/frontend/src/pages/HomePage.vue index db40342..5d0e14a 100644 --- a/frontend/src/pages/HomePage.vue +++ b/frontend/src/pages/HomePage.vue @@ -532,7 +532,9 @@ async function resendVerification() { step="1" class="w-full" /> -
    数值越小压缩越强，目标为压缩后体积占原图比例（100% 为不压缩）。
    +
    + 数值越小压缩越强；JPEG/WebP/AVIF 会以此为体积上限，无损格式按安全方式优化并显示实际结果。 +
    diff --git a/scripts/benchmark_compression.py b/scripts/benchmark_compression.py new file mode 100644 index 0000000..72a0b5f --- /dev/null +++ b/scripts/benchmark_compression.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python3 +"""Generate a real-photo corpus and benchmark ImageForge through its HTTP API.""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import os +import statistics +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from datetime import datetime, timezone +from http.cookiejar import CookieJar +from pathlib import Path +from typing import Any + +import numpy as np +from PIL import Image, ImageOps + + +CORPUS = ( + ("photo_1015_jpeg", 1015, "JPEG", "jpg"), + ("photo_1016_png", 1016, "PNG", "png"), + ("photo_1025_webp", 1025, "WEBP", "webp"), + ("photo_1039_avif", 1039, "AVIF", "avif"), +) +FORMAT_EXTENSIONS = {"jpeg": "jpg", "webp": "webp", "avif": "avif"} +PIL_FORMATS = {"jpeg": "JPEG", "webp": "WEBP", "avif": "AVIF"} + + +def parse_csv_arg(value: str, cast: type = str) -> list[Any]: + return [cast(item.strip()) for item in value.split(",") if item.strip()] + + +def open_image(path: Path) -> Image.Image: + with Image.open(path) as image: + return ImageOps.exif_transpose(image).convert("RGB") + + +def download(url: str) -> bytes: + request = urllib.request.Request(url, headers={"User-Agent": "ImageForge benchmark/1.0"}) + with urllib.request.urlopen(request, timeout=90) as response: + return response.read() + + +def generate_corpus(output_dir: Path, width: int, height: int) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + manifest: list[dict[str, Any]] = [] + + for name, image_id, image_format, extension in CORPUS: + source_url = f"https://picsum.photos/id/{image_id}/{width}/{height}.jpg" + source_bytes = download(source_url) + source_path = output_dir / f".{name}.source.jpg" + source_path.write_bytes(source_bytes) + image = open_image(source_path) + source_path.unlink() + + # Normalize dimensions so format and content complexity, not resolution, drive comparisons. + image = ImageOps.fit(image, (width, height), method=Image.Resampling.LANCZOS) + output_path = output_dir / f"{name}.{extension}" + if image_format == "JPEG": + image.save(output_path, format=image_format, quality=95, subsampling=0, optimize=True) + elif image_format == "PNG": + image.save(output_path, format=image_format, optimize=True, compress_level=9) + elif image_format == "WEBP": + image.save(output_path, format=image_format, lossless=True, method=6) + else: + image.save(output_path, format=image_format, quality=95, speed=6) + + manifest.append( + { + "file": output_path.name, + "source_url": source_url, + "picsum_id": image_id, + "format": image_format.lower(), + "width": image.width, + "height": image.height, + "size_bytes": output_path.stat().st_size, + } + ) + + (output_dir / "manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + print(f"generated {len(manifest)} images in {output_dir}") + + +def multipart_body(file_path: Path, fields: dict[str, str]) -> tuple[bytes, str]: + boundary = f"----imageforge-{uuid.uuid4().hex}" + chunks: list[bytes] = [] + for name, value in fields.items(): + chunks.extend( + ( + f"--{boundary}\r\n".encode(), + f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode(), + value.encode(), + b"\r\n", + ) + ) + + chunks.extend( + ( + f"--{boundary}\r\n".encode(), + ( + f'Content-Disposition: form-data; name="file"; ' + f'filename="{file_path.name}"\r\n' + ).encode(), + b"Content-Type: application/octet-stream\r\n\r\n", + file_path.read_bytes(), + b"\r\n", + f"--{boundary}--\r\n".encode(), + ) + ) + return b"".join(chunks), f"multipart/form-data; boundary={boundary}" + + +def request_json( + opener: urllib.request.OpenerDirector, + url: str, + data: bytes, + headers: dict[str, str], + timeout: int, +) -> dict[str, Any]: + request = urllib.request.Request(url, data=data, headers=headers, method="POST") + try: + with opener.open(request, timeout=timeout) as response: + payload = json.load(response) + except urllib.error.HTTPError as error: + detail = error.read().decode("utf-8", errors="replace") + raise RuntimeError(f"HTTP {error.code}: {detail}") from error + + if not payload.get("success") or "data" not in payload: + raise RuntimeError(f"unexpected API response: {payload}") + return payload["data"] + + +def login( + opener: urllib.request.OpenerDirector, + base_url: str, + email: str, + password: str, + timeout: int, +) -> str: + body = json.dumps({"email": email, "password": password}).encode() + data = request_json( + opener, + f"{base_url}/api/v1/auth/login", + body, + {"Content-Type": "application/json"}, + timeout, + ) + return str(data["token"]) + + +def block_ssim(reference: np.ndarray, candidate: np.ndarray, block: int = 8) -> float: + ref = 0.2126 * reference[..., 0] + 0.7152 * reference[..., 1] + 0.0722 * reference[..., 2] + out = 0.2126 * candidate[..., 0] + 0.7152 * candidate[..., 1] + 0.0722 * candidate[..., 2] + height = (ref.shape[0] // block) * block + width = (ref.shape[1] // block) * block + if height == 0 or width == 0: + height, width, block = ref.shape[0], ref.shape[1], 1 + + def blocks(array: np.ndarray) -> np.ndarray: + return ( + array[:height, :width] + .reshape(height // block, block, width // block, block) + .transpose(0, 2, 1, 3) + ) + + ref_blocks = blocks(ref) + out_blocks = blocks(out) + axes = (-1, -2) + ref_mean = ref_blocks.mean(axis=axes) + out_mean = out_blocks.mean(axis=axes) + ref_var = ref_blocks.var(axis=axes) + out_var = out_blocks.var(axis=axes) + covariance = ((ref_blocks - ref_mean[..., None, None]) * (out_blocks - out_mean[..., None, None])).mean(axis=axes) + c1 = (0.01 * 255.0) ** 2 + c2 = (0.03 * 255.0) ** 2 + numerator = (2 * ref_mean * out_mean + c1) * (2 * covariance + c2) + denominator = (ref_mean**2 + out_mean**2 + c1) * (ref_var + out_var + c2) + return float(np.mean(numerator / np.maximum(denominator, 1e-12))) + + +def image_metrics(reference_path: Path, output_path: Path) -> dict[str, Any]: + reference = open_image(reference_path) + with Image.open(output_path) as opened: + detected_format = (opened.format or "unknown").upper() + output = ImageOps.exif_transpose(opened).convert("RGB") + + output_width, output_height = output.size + if output.size != reference.size: + output = output.resize(reference.size, Image.Resampling.LANCZOS) + + ref_array = np.asarray(reference, dtype=np.float64) + out_array = np.asarray(output, dtype=np.float64) + mse = float(np.mean((ref_array - out_array) ** 2)) + psnr = 99.0 if mse == 0 else 20.0 * math.log10(255.0 / math.sqrt(mse)) + return { + "detected_format": detected_format, + "width": output_width, + "height": output_height, + "pixel_ratio_pct": output_width * output_height * 100.0 / (reference.width * reference.height), + "ssim": block_ssim(ref_array, out_array), + "psnr_db": psnr, + } + + +def summarize(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + groups: dict[tuple[str, int], list[dict[str, Any]]] = {} + for row in rows: + if row.get("error"): + continue + groups.setdefault((str(row["output_format"]), int(row["requested_rate"])), []).append(row) + + summary: list[dict[str, Any]] = [] + for (output_format, requested_rate), group in sorted(groups.items()): + summary.append( + { + "output_format": output_format, + "requested_rate": requested_rate, + "cases": len(group), + "target_met": sum(bool(row["target_met"]) for row in group), + "format_ok": sum(bool(row["format_ok"]) for row in group), + "mean_actual_rate_pct": statistics.mean(float(row["actual_rate_pct"]) for row in group), + "mean_saved_pct": statistics.mean(float(row["saved_pct"]) for row in group), + "mean_ssim": statistics.mean(float(row["ssim"]) for row in group), + "mean_psnr_db": statistics.mean(float(row["psnr_db"]) for row in group), + "mean_pixel_ratio_pct": statistics.mean(float(row["pixel_ratio_pct"]) for row in group), + "median_elapsed_ms": statistics.median(float(row["elapsed_ms"]) for row in group), + } + ) + return summary + + +def benchmark(args: argparse.Namespace) -> int: + input_dir = Path(args.input_dir) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + inputs = sorted(path for path in input_dir.iterdir() if path.suffix.lower() in {".jpg", ".jpeg", ".png", ".webp", ".avif"}) + if not inputs: + raise RuntimeError(f"no benchmark images found in {input_dir}") + + rates = parse_csv_arg(args.rates, int) + formats = [str(value).lower() for value in parse_csv_arg(args.formats)] + unsupported = sorted(set(formats) - set(FORMAT_EXTENSIONS)) + if unsupported: + raise RuntimeError(f"unsupported output formats: {', '.join(unsupported)}") + + base_url = args.base_url.rstrip("/") + opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(CookieJar())) + token = args.token or os.getenv("IMAGEFORGE_BENCH_TOKEN", "") + if not token: + email = args.email or os.getenv("IMAGEFORGE_BENCH_EMAIL", "") + password = args.password or os.getenv("IMAGEFORGE_BENCH_PASSWORD", "") + if email and password: + token = login(opener, base_url, email, password, args.timeout) + + auth_headers = {"Authorization": f"Bearer {token}"} if token else {} + rows: list[dict[str, Any]] = [] + total = len(inputs) * len(formats) * len(rates) + case_number = 0 + + for input_path in inputs: + original_size = input_path.stat().st_size + for output_format in formats: + for rate in rates: + case_number += 1 + output_path = output_dir / f"{input_path.stem}__{output_format}__r{rate}.{FORMAT_EXTENSIONS[output_format]}" + row: dict[str, Any] = { + "input": input_path.name, + "input_format": input_path.suffix.lower().lstrip("."), + "output_format": output_format, + "requested_rate": rate, + "original_size": original_size, + } + try: + body, content_type = multipart_body( + input_path, + {"compression_rate": str(rate), "output_format": output_format}, + ) + started = time.perf_counter() + data = request_json( + opener, + f"{base_url}/api/v1/compress", + body, + {**auth_headers, "Content-Type": content_type}, + args.timeout, + ) + elapsed_ms = (time.perf_counter() - started) * 1000.0 + + download_request = urllib.request.Request( + urllib.parse.urljoin(f"{base_url}/", str(data["download_url"]).lstrip("/")), + headers=auth_headers, + ) + with opener.open(download_request, timeout=args.timeout) as response: + output_path.write_bytes(response.read()) + + compressed_size = output_path.stat().st_size + metrics = image_metrics(input_path, output_path) + tolerance_bytes = max(1024, int(original_size * 0.01)) + target_bytes = original_size * rate / 100.0 + row.update( + { + "compressed_size": compressed_size, + "actual_rate_pct": compressed_size * 100.0 / original_size, + "saved_pct": max(0.0, (original_size - compressed_size) * 100.0 / original_size), + "target_error_pct_points": compressed_size * 100.0 / original_size - rate, + "target_met": compressed_size <= target_bytes + tolerance_bytes, + "format_ok": metrics["detected_format"] == PIL_FORMATS[output_format], + "api_size_matches": int(data["compressed_size"]) == compressed_size, + "elapsed_ms": elapsed_ms, + **metrics, + "error": "", + } + ) + except Exception as error: # Continue to expose the full failure matrix. + row["error"] = str(error) + rows.append(row) + status = "ERROR" if row.get("error") else f"{row['actual_rate_pct']:.1f}% SSIM={row['ssim']:.4f}" + print(f"[{case_number:02d}/{total:02d}] {input_path.name} -> {output_format} r{rate}: {status}", flush=True) + if args.delay: + time.sleep(args.delay) + + summary = summarize(rows) + report = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "base_url": base_url, + "inputs": len(inputs), + "cases": len(rows), + "errors": sum(bool(row.get("error")) for row in rows), + "format_failures": sum(not bool(row.get("format_ok")) for row in rows if not row.get("error")), + "target_failures": sum(not bool(row.get("target_met")) for row in rows if not row.get("error")), + "summary": summary, + "results": rows, + } + (output_dir / "report.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + fieldnames = sorted({key for row in rows for key in row}) + with (output_dir / "results.csv").open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + print("\nformat rate target format-ok actual% saved% SSIM PSNR pixel% median-ms") + for item in summary: + print( + f"{item['output_format']:>6} {item['requested_rate']:>4} " + f"{item['target_met']}/{item['cases']} {item['format_ok']}/{item['cases']} " + f"{item['mean_actual_rate_pct']:>7.2f} {item['mean_saved_pct']:>6.2f} " + f"{item['mean_ssim']:.4f} {item['mean_psnr_db']:>5.2f} " + f"{item['mean_pixel_ratio_pct']:>6.2f} {item['median_elapsed_ms']:>9.1f}" + ) + print(f"\nreport: {output_dir / 'report.json'}") + return 1 if args.strict and (report["errors"] or report["format_failures"] or report["target_failures"]) else 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + generate = subparsers.add_parser("generate", help="download and build the fixed real-photo corpus") + generate.add_argument("--output-dir", default=".bench/corpus") + generate.add_argument("--width", type=int, default=960) + generate.add_argument("--height", type=int, default=640) + + run = subparsers.add_parser("run", help="benchmark a running ImageForge deployment") + run.add_argument("--base-url", default="http://127.0.0.1:8080") + run.add_argument("--input-dir", default=".bench/corpus") + run.add_argument("--output-dir", default=".bench/results") + run.add_argument("--formats", default="jpeg,webp,avif") + run.add_argument("--rates", default="30,50,70") + run.add_argument("--email", default="") + run.add_argument("--password", default="") + run.add_argument("--token", default="") + run.add_argument("--timeout", type=int, default=300) + run.add_argument("--delay", type=float, default=0.05) + run.add_argument("--strict", action="store_true") + return parser + + +def main() -> int: + args = build_parser().parse_args() + if args.command == "generate": + generate_corpus(Path(args.output_dir), args.width, args.height) + return 0 + return benchmark(args) + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except KeyboardInterrupt: + raise SystemExit(130) from None + except Exception as error: + print(f"benchmark failed: {error}", file=sys.stderr) + raise SystemExit(1) from error diff --git a/scripts/requirements-benchmark.txt b/scripts/requirements-benchmark.txt new file mode 100644 index 0000000..d118c0e --- /dev/null +++ b/scripts/requirements-benchmark.txt @@ -0,0 +1,2 @@ +numpy>=2,<3 +Pillow>=11,<13 diff --git a/src/services/compress.rs b/src/services/compress.rs index 7408d9e..8261aed 100644 --- a/src/services/compress.rs +++ b/src/services/compress.rs @@ -4,7 +4,6 @@ use crate::state::AppState; use image::codecs::bmp::BmpEncoder; use image::codecs::gif::{GifDecoder, GifEncoder}; use image::codecs::ico::IcoEncoder; -use image::codecs::jpeg::JpegEncoder; use image::codecs::png::PngEncoder; use image::codecs::tiff::TiffEncoder; use image::{AnimationDecoder, GenericImageView}; @@ -14,12 +13,13 @@ use oxipng::StripChunks; use rgb::FromSlice; use std::io::Cursor; -const TARGET_MIN_DIMENSION: u32 = 640; +const TARGET_MIN_LONG_EDGE: u32 = 640; const TARGET_MIN_SCALE: f64 = 0.55; const TARGET_RESIZE_ATTEMPTS: usize = 5; +const TARGET_SCALE_REFINEMENT_ATTEMPTS: usize = 3; -const JPEG_TARGET_MIN_QUALITY: u8 = 40; -const WEBP_TARGET_MIN_QUALITY: u8 = 42; +const JPEG_TARGET_MIN_QUALITY: u8 = 25; +const WEBP_TARGET_MIN_QUALITY: u8 = 30; const AVIF_TARGET_MIN_QUALITY: u8 = 38; #[derive(Debug, Clone, Copy)] @@ -378,7 +378,7 @@ fn compress_image_bytes_sync( output = apply_metadata(output, icc_profile, exif)?; } - if !resized && output.len() >= input.len() { + if format_in == format_out && !resized && output.len() >= input.len() { if preserve_metadata { return Ok(input); } @@ -475,10 +475,16 @@ fn encode_jpeg_with_quality(image: DynamicImage, quality: u8) -> Result, } fn encode_jpeg_raw(raw: &[u8], w: u32, h: u32, quality: u8) -> Result, AppError> { + let width = u16::try_from(w) + .map_err(|_| AppError::new(ErrorCode::InvalidImage, "JPEG 宽度不能超过 65535 像素"))?; + let height = u16::try_from(h) + .map_err(|_| AppError::new(ErrorCode::InvalidImage, "JPEG 高度不能超过 65535 像素"))?; let mut out = Vec::new(); - let mut encoder = JpegEncoder::new_with_quality(&mut out, quality); + let mut encoder = jpeg_encoder::Encoder::new(&mut out, quality); + encoder.set_optimized_huffman_tables(true); + encoder.set_progressive(true); encoder - .encode(raw, w, h, ExtendedColorType::Rgb8) + .encode(raw, width, height, jpeg_encoder::ColorType::Rgb) .map_err(|err| { AppError::new(ErrorCode::CompressionFailed, "JPEG 编码失败").with_source(err) })?; @@ -573,36 +579,33 @@ where F: FnMut(&DynamicImage, u8) -> Result, AppError>, { let (orig_w, orig_h) = image.dimensions(); - let min_w = ((orig_w as f64 * TARGET_MIN_SCALE).round() as u32) - .max(TARGET_MIN_DIMENSION.min(orig_w)) - .max(1); - let min_h = ((orig_h as f64 * TARGET_MIN_SCALE).round() as u32) - .max(TARGET_MIN_DIMENSION.min(orig_h)) - .max(1); + let long_edge = orig_w.max(orig_h); + let long_edge_floor = TARGET_MIN_LONG_EDGE.min(long_edge) as f64 / long_edge as f64; + let min_scale = TARGET_MIN_SCALE.max(long_edge_floor).min(1.0); let mut scales = Vec::with_capacity(TARGET_RESIZE_ATTEMPTS + 1); scales.push(1.0); for step in 1..=TARGET_RESIZE_ATTEMPTS { let ratio = step as f64 / TARGET_RESIZE_ATTEMPTS as f64; - let scale = 1.0 - (1.0 - TARGET_MIN_SCALE) * ratio; - scales.push(scale.max(TARGET_MIN_SCALE)); + scales.push(1.0 - (1.0 - min_scale) * ratio); } - let mut best_under: Option<(Vec, u32, u32, u64)> = None; - let mut best_over: Option<(Vec, u32, u32, u64)> = None; + let mut best_over: Option<(Vec, u64, u64)> = None; + let mut previous_over_scale = 1.0; + let mut last_dimensions: Option<(u32, u32)> = None; for scale in scales { let new_w = ((orig_w as f64 * scale).round() as u32).clamp(1, orig_w); let new_h = ((orig_h as f64 * scale).round() as u32).clamp(1, orig_h); - - if new_w < min_w || new_h < min_h { + if last_dimensions == Some((new_w, new_h)) { continue; } + last_dimensions = Some((new_w, new_h)); let resized = if new_w == orig_w && new_h == orig_h { image.clone() } else { - image.resize(new_w, new_h, image::imageops::FilterType::Lanczos3) + image.resize_exact(new_w, new_h, image::imageops::FilterType::Lanczos3) }; let result = @@ -610,52 +613,69 @@ where let result_size = result.len() as u64; if result_size <= target_size { - let should_update = match &best_under { - None => true, - Some((_bytes, best_w, best_h, best_size)) => { - let new_pixels = (new_w as u64).saturating_mul(new_h as u64); - let best_pixels = (*best_w as u64).saturating_mul(*best_h as u64); - new_pixels > best_pixels - || (new_pixels == best_pixels && result_size > *best_size) + if new_w == orig_w && new_h == orig_h { + return Ok(result); + } + + // The first passing coarse scale has the highest resolution. Refine the + // boundary between it and the preceding failing scale before returning. + let mut best_under = result; + let mut under_scale = scale; + let mut over_scale = previous_over_scale; + let mut under_dimensions = (new_w, new_h); + + for _ in 0..TARGET_SCALE_REFINEMENT_ATTEMPTS { + let candidate_scale = (under_scale + over_scale) / 2.0; + let candidate_w = + ((orig_w as f64 * candidate_scale).round() as u32).clamp(1, orig_w); + let candidate_h = + ((orig_h as f64 * candidate_scale).round() as u32).clamp(1, orig_h); + if (candidate_w, candidate_h) == under_dimensions { + break; } - }; - if should_update { - best_under = Some((result, new_w, new_h, result_size)); + let candidate = image.resize_exact( + candidate_w, + candidate_h, + image::imageops::FilterType::Lanczos3, + ); + let candidate_result = encode_target_quality_with_image( + &candidate, + min_q, + max_q, + target_size, + &mut encode_fn, + )?; + if candidate_result.len() as u64 <= target_size { + best_under = candidate_result; + under_scale = candidate_scale; + under_dimensions = (candidate_w, candidate_h); + } else { + over_scale = candidate_scale; + } } - if new_w == orig_w && new_h == orig_h && target_size.saturating_sub(result_size) <= 1024 - { - break; - } + return Ok(best_under); } else { let should_update = match &best_over { None => true, - Some((_bytes, best_w, best_h, best_size)) => { + Some((_bytes, best_size, best_pixels)) => { let over = result_size.saturating_sub(target_size); let best_over_by = best_size.saturating_sub(target_size); - if over < best_over_by { - true - } else if over == best_over_by { - let new_pixels = (new_w as u64).saturating_mul(new_h as u64); - let best_pixels = (*best_w as u64).saturating_mul(*best_h as u64); - new_pixels > best_pixels - } else { - false - } + let new_pixels = (new_w as u64).saturating_mul(new_h as u64); + over < best_over_by || (over == best_over_by && new_pixels > *best_pixels) } }; if should_update { - best_over = Some((result, new_w, new_h, result_size)); + let pixels = (new_w as u64).saturating_mul(new_h as u64); + best_over = Some((result, result_size, pixels)); } + previous_over_scale = scale; } } - if let Some((bytes, _, _, _)) = best_under { - return Ok(bytes); - } - if let Some((bytes, _, _, _)) = best_over { + if let Some((bytes, _, _)) = best_over { return Ok(bytes); } @@ -673,51 +693,34 @@ fn encode_target_quality_with_image( where F: FnMut(&DynamicImage, u8) -> Result, AppError>, { - let mut best: Option> = None; - let mut best_diff = u64::MAX; - let mut best_is_under = false; - - let mut consider = |bytes: Vec| { - let size = bytes.len() as u64; - let is_under = size <= target_size; - let diff = size.abs_diff(target_size); - - let should_update = match (best_is_under, is_under) { - (false, true) => true, - (true, false) => false, - _ => diff < best_diff, - }; - - if should_update { - best_diff = diff; - best_is_under = is_under; - best = Some(bytes); - } - }; - - consider(encode_fn(image, min_q)?); - if min_q != max_q { - consider(encode_fn(image, max_q)?); + // Start with the highest quality. If it already fits, no lower-quality + // encodes can improve the result. + let max_quality = encode_fn(image, max_q)?; + if max_quality.len() as u64 <= target_size || min_q == max_q { + return Ok(max_quality); } - let mut low = min_q; - let mut high = max_q; - for _ in 0..12 { - if low > high { - break; - } + let min_quality = encode_fn(image, min_q)?; + if min_quality.len() as u64 > target_size { + return Ok(min_quality); + } + + let mut best_under = min_quality; + let mut low = min_q.saturating_add(1); + let mut high = max_q.saturating_sub(1); + while low <= high { let mid = (low + high) / 2; let bytes = encode_fn(image, mid)?; let size = bytes.len() as u64; - consider(bytes); if size > target_size { high = mid.saturating_sub(1); } else { + best_under = bytes; low = mid.saturating_add(1); } } - best.ok_or_else(|| AppError::new(ErrorCode::CompressionFailed, "压缩失败")) + Ok(best_under) } fn encode_gif(image: DynamicImage, rate: u8) -> Result, AppError> { @@ -765,6 +768,8 @@ fn encode_tiff(image: DynamicImage) -> Result, AppError> { } fn encode_ico(image: DynamicImage) -> Result, AppError> { + // A single ICO directory entry can represent at most 256x256 pixels. + let (image, _) = resize_if_needed(image, Some(256), Some(256)); let rgba = image.to_rgba8(); let (w, h) = rgba.dimensions(); let mut out = Vec::new(); @@ -941,4 +946,61 @@ mod tests { assert_eq!(target_size_from_rate(10_000, 55), 5_500); assert_eq!(target_size_from_rate(10_000, 100), 10_000); } + + #[test] + fn target_encoder_stops_when_full_resolution_meets_target() { + use std::cell::Cell; + + let calls = Cell::new(0); + let image = DynamicImage::new_rgb8(800, 600); + let result = encode_with_auto_resize(image, 100, 40, 95, |_image, quality| { + calls.set(calls.get() + 1); + Ok(vec![0; quality as usize]) + }) + .unwrap(); + + assert_eq!(result.len(), 95); + assert_eq!(calls.get(), 1); + } + + #[test] + fn target_encoder_can_reduce_a_landscape_at_the_long_edge_floor() { + let image = DynamicImage::new_rgb8(960, 640); + let result = encode_with_auto_resize(image, 40_000, 40, 40, |image, _quality| { + let (width, height) = image.dimensions(); + Ok(vec![0; (width as usize * height as usize) / 10]) + }) + .unwrap(); + + assert!(result.len() <= 40_000); + } + + #[test] + fn format_conversion_never_returns_the_original_encoding() { + let input = encode_png(DynamicImage::new_rgba8(10, 10), 100, false).unwrap(); + let output = compress_image_bytes_sync( + input, + ImageFmt::Png, + ImageFmt::Bmp, + CompressionLevel::Medium, + None, + None, + None, + None, + false, + 1_000_000, + ) + .unwrap(); + + assert!(output.starts_with(b"BM")); + } + + #[test] + fn ico_encoder_fits_large_images_within_the_format_limit() { + let output = encode_ico(DynamicImage::new_rgba8(960, 640)).unwrap(); + let decoded = image::load_from_memory(&output).unwrap(); + + assert!(output.starts_with(b"\x00\x00\x01\x00")); + assert_eq!(decoded.dimensions(), (256, 171)); + } }