Compare commits

..

29 Commits

Author SHA1 Message Date
237899745
03b3a08185 fix: bound large WebP lossless probe cost
Some checks failed
CI / verify (push) Has been cancelled
2026-07-26 12:22:53 +08:00
237899745
66454b6325 fix: honor target image size across compression paths
Some checks failed
CI / verify (push) Has been cancelled
2026-07-26 12:14:05 +08:00
237899745
72f36c631e fix(worker): qualify file attempt claim
Some checks failed
CI / verify (push) Has been cancelled
2026-07-26 11:06:53 +08:00
237899745
cdec19977c ci: require all external state tests
Some checks failed
CI / verify (push) Has been cancelled
2026-07-26 10:54:22 +08:00
237899745
408e09cda8 fix(storage): make queue and object ownership durable 2026-07-26 10:54:13 +08:00
237899745
f2d490edce fix(billing): enforce one effective subscription per user 2026-07-26 10:54:01 +08:00
237899745
e90f6ec604 ci: harden external test tool bootstrap
All checks were successful
CI / verify (push) Successful in 11m0s
2026-07-26 08:20:00 +08:00
237899745
9c1d749a2a ci: validate RustSec snapshot layout
Some checks failed
CI / verify (push) Failing after 10m43s
2026-07-26 08:06:17 +08:00
237899745
25e889190d ci: audit from retryable RustSec snapshot
Some checks failed
CI / verify (push) Failing after 14m3s
2026-07-26 07:47:34 +08:00
237899745
b4ace3bc68 ci: verify production dav1d baseline
Some checks failed
CI / verify (push) Failing after 14m27s
2026-07-26 07:30:37 +08:00
237899745
8e2ed3a306 ci: install Linux image build dependencies
Some checks failed
CI / verify (push) Failing after 5m5s
2026-07-26 07:15:41 +08:00
237899745
4138c737ce ci: remove external action dependencies
Some checks failed
CI / verify (push) Failing after 4m41s
2026-07-26 07:09:15 +08:00
237899745
1e0c1ab539 ci: avoid unavailable Gitea cache backend
Some checks failed
CI / verify (push) Failing after 1m33s
2026-07-26 07:05:04 +08:00
237899745
fbc82dfa07 fix: keep ZIP builds alive after request cancellation
Some checks are pending
CI / verify (push) Waiting to run
2026-07-26 06:46:37 +08:00
237899745
4d0e8aa70a fix: finalize failed batch enqueue atomically 2026-07-26 06:46:29 +08:00
237899745
380e89b058 ci: run external-state invariants
Some checks failed
CI / verify (push) Has been cancelled
2026-07-26 06:36:06 +08:00
237899745
f8f5da04db docs: document migration and quota safeguards
Some checks failed
CI / verify (push) Has been cancelled
2026-07-26 05:48:14 +08:00
237899745
910e60ab59 fix: settle anonymous single-file reservations 2026-07-26 05:47:54 +08:00
237899745
65694cee15 fix: reconcile pre-watermark Stripe invoices 2026-07-26 05:47:38 +08:00
237899745
923ba495c4 fix: fence concurrent ZIP archive builds 2026-07-26 05:47:19 +08:00
237899745
037e83e92f fix: make Stripe invoice webhooks monotonic
Some checks failed
CI / verify (push) Has been cancelled
2026-07-26 04:53:21 +08:00
237899745
3aefacec6b fix: charge anonymous batch reservations correctly 2026-07-26 04:53:09 +08:00
237899745
326a678249 fix: reconcile ambiguous Stripe event ordering
Some checks failed
CI / verify (push) Has been cancelled
2026-07-26 03:08:23 +08:00
237899745
08000cc16e fix: serialize Stripe checkout provisioning 2026-07-26 03:07:30 +08:00
237899745
03d0e43d4d fix: revoke stale account recovery credentials 2026-07-26 03:07:04 +08:00
237899745
1f55bd45ca test: require the selected worker storage backend
Some checks failed
CI / verify (push) Has been cancelled
2026-07-26 02:15:00 +08:00
237899745
fd9a828225 fix: make Stripe subscription events monotonic
Some checks failed
CI / verify (push) Has been cancelled
2026-07-26 02:04:17 +08:00
237899745
57892f6509 fix: fence worker retries and object publication 2026-07-26 02:03:57 +08:00
237899745
016ce9ebab fix: secure email change recovery flow 2026-07-26 02:03:43 +08:00
60 changed files with 11618 additions and 1496 deletions

View File

@@ -23,6 +23,11 @@ WORKER_CONCURRENCY=4
# 单进程图片处理并发上限API 与 Worker 均生效,默认等于 CPU 线程数)
IMAGE_PROCESSING_CONCURRENCY=4
# ZIP 使用任务租约做 single-flight总大小按解压前源文件字节计算。
ZIP_BUILD_CONCURRENCY=2
ZIP_MAX_ENTRIES=200
ZIP_MAX_UNCOMPRESSED_BYTES=2147483648
# 仅当后端只能由可信反向代理访问时启用,否则客户端可伪造来源 IP
TRUST_PROXY_HEADERS=false
@@ -40,6 +45,8 @@ STORAGE_PATH=./uploads
BILLING_PROVIDER=stripe
STRIPE_SECRET_KEY=sk_test_xxx
STRIPE_WEBHOOK_SECRET=whsec_xxx
# Keep the official endpoint in production; override only for isolated mocks.
STRIPE_API_BASE_URL=https://api.stripe.com
# 邮件服务(注册验证 + 密码重置)
MAIL_ENABLED=false

1
.gitattributes vendored
View File

@@ -1 +1,2 @@
migrations/*.sql text eol=crlf
*.sh text eol=lf

View File

@@ -8,9 +8,45 @@ on:
jobs:
verify:
runs-on: ubuntu-latest
services:
postgres:
image: postgres:16-alpine
env:
POSTGRES_PASSWORD: codex_test
POSTGRES_DB: imageforge_test
options: >-
--health-cmd "pg_isready -U postgres -d imageforge_test"
--health-interval 5s
--health-timeout 3s
--health-retries 20
redis:
image: redis:7-alpine
options: >-
--health-cmd "redis-cli ping"
--health-interval 5s
--health-timeout 3s
--health-retries 20
env:
DATABASE_URL: postgres://postgres:codex_test@postgres:5432/imageforge_test
REDIS_URL: redis://redis:6379/
IMAGEFORGE_TEST_DATABASE_URL: postgres://postgres:codex_test@postgres:5432/imageforge_test
IMAGEFORGE_TEST_REDIS_URL: redis://redis:6379/
JWT_SECRET: imageforge-ci-jwt-secret
API_KEY_PEPPER: imageforge-ci-api-key-pepper
EXPECTED_EXTERNAL_TESTS: '13'
steps:
- name: Checkout
uses: actions/checkout@v4
env:
GITEA_JOB_TOKEN: ${{ gitea.token }}
run: |
set -Eeuo pipefail
auth="$(printf 'x-access-token:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
git init .
git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
git -c http.extraHeader="Authorization: Basic ${auth}" \
fetch --no-tags --depth=1 origin "$GITHUB_REF"
git checkout --detach "$GITHUB_SHA"
test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
- name: Verify migration line endings
run: |
@@ -24,39 +60,61 @@ jobs:
invalid.append(str(path))
if invalid:
raise SystemExit("migrations must use CRLF: " + ", ".join(invalid))
raise SystemExit("migrations must use CRLF: " + ", ".join(invalid))
PY
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: '1.92'
components: rustfmt, clippy
- name: Install native build dependencies
run: |
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
cmake libdav1d-dev nasm pkg-config
pkg-config --atleast-version=1.3.0 dav1d
rm -rf /var/lib/apt/lists/*
- name: Cache Rust build
uses: Swatinem/rust-cache@v2
- name: Install Rust toolchain
run: |
set -Eeuo pipefail
curl --proto '=https' --tlsv1.2 --fail --silent --show-error \
--location --retry 10 --retry-connrefused https://sh.rustup.rs \
| sh -s -- -y --default-toolchain none --profile minimal
/root/.cargo/bin/rustup toolchain install 1.92 \
--profile minimal --component rustfmt --component clippy --no-self-update
/root/.cargo/bin/rustup default 1.92
echo /root/.cargo/bin >> "$GITHUB_PATH"
/root/.cargo/bin/rustc --version --verbose
- name: Check Rust formatting
run: cargo fmt --all -- --check
- name: Run Clippy
run: cargo clippy --all-targets -- -D warnings
run: cargo clippy --all-targets --all-features --locked -- -D warnings
- name: Run Rust tests
run: cargo test --all-targets
run: cargo test --all-targets --locked
- name: Run external-state tests
run: bash scripts/run_external_state_tests.sh
- name: Install cargo-audit
run: cargo install cargo-audit --locked --version 0.22.2
- name: Audit Rust dependencies
run: cargo audit
run: |
set -Eeuo pipefail
audit_db="$RUNNER_TEMP/rustsec-advisory-db"
mkdir -p "$audit_db"
curl --proto '=https' --tlsv1.2 --fail --silent --show-error \
--location --retry 10 --retry-all-errors \
https://codeload.github.com/RustSec/advisory-db/tar.gz/refs/heads/main \
| tar -xz --strip-components=1 -C "$audit_db"
test -f "$audit_db/support.toml"
test -d "$audit_db/crates"
cargo audit --db "$audit_db" --no-fetch
- name: Install Node.js
uses: actions/setup-node@v4
with:
node-version: '22'
cache: npm
cache-dependency-path: frontend/package-lock.json
run: |
node --version | grep --extended-regexp '^v22\.'
npm --version
- name: Build frontend
working-directory: frontend

1
Cargo.lock generated
View File

@@ -2215,6 +2215,7 @@ dependencies = [
"time",
"tokio",
"tokio-util",
"tower",
"tower-http",
"tracing",
"tracing-subscriber",

View File

@@ -67,3 +67,6 @@ zip = "2"
[target.'cfg(target_os = "linux")'.dependencies]
image = { version = "0.25", default-features = false, features = ["avif-native"] }
ravif = { version = "0.11", default-features = false, features = ["asm", "threading"] }
[dev-dependencies]
tower = { version = "0.5", features = ["util"] }

View File

@@ -27,6 +27,11 @@ WORKER_TASK_CONCURRENCY=4
WORKER_CONCURRENCY=2
IMAGE_PROCESSING_CONCURRENCY=4
# Two concurrent 2 GiB ZIP builds require roughly 8 GiB of temporary disk.
ZIP_BUILD_CONCURRENCY=2
ZIP_MAX_ENTRIES=200
ZIP_MAX_UNCOMPRESSED_BYTES=2147483648
# Resource ceilings tuned for an 8-core / 16 GB application host.
POSTGRES_MEMORY_LIMIT=2g
REDIS_MEMORY_LIMIT=1g
@@ -48,6 +53,7 @@ MAIL_ENABLED=false
MAIL_LOG_LINKS_WHEN_DISABLED=false
# STRIPE_SECRET_KEY=sk_live_replace_me
# STRIPE_WEBHOOK_SECRET=whsec_replace_me
# STRIPE_API_BASE_URL=https://api.stripe.com
# MAIL_PROVIDER=custom
# MAIL_FROM=noreply@example.com
# MAIL_PASSWORD=replace-with-smtp-authorization-code

View File

@@ -13,6 +13,9 @@ x-imageforge-environment: &imageforge-environment
WORKER_TASK_CONCURRENCY: ${WORKER_TASK_CONCURRENCY:-4}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
IMAGE_PROCESSING_CONCURRENCY: ${IMAGE_PROCESSING_CONCURRENCY:-2}
ZIP_BUILD_CONCURRENCY: ${ZIP_BUILD_CONCURRENCY:-2}
ZIP_MAX_ENTRIES: ${ZIP_MAX_ENTRIES:-200}
ZIP_MAX_UNCOMPRESSED_BYTES: ${ZIP_MAX_UNCOMPRESSED_BYTES:-2147483648}
ALLOW_ANONYMOUS_UPLOAD: ${ALLOW_ANONYMOUS_UPLOAD:-true}
ANON_MAX_FILE_SIZE_MB: ${ANON_MAX_FILE_SIZE_MB:-5}
ANON_MAX_FILES_PER_BATCH: ${ANON_MAX_FILES_PER_BATCH:-5}
@@ -74,6 +77,7 @@ services:
ADMIN_PASSWORD: ${ADMIN_PASSWORD:-}
STRIPE_SECRET_KEY: "${STRIPE_SECRET_KEY:-}"
STRIPE_WEBHOOK_SECRET: "${STRIPE_WEBHOOK_SECRET:-}"
STRIPE_API_BASE_URL: ${STRIPE_API_BASE_URL:-https://api.stripe.com}
MAIL_ENABLED: ${MAIL_ENABLED:-false}
MAIL_LOG_LINKS_WHEN_DISABLED: ${MAIL_LOG_LINKS_WHEN_DISABLED:-false}
MAIL_PROVIDER: ${MAIL_PROVIDER:-qq}

View File

@@ -40,6 +40,8 @@ http {
location /downloads/ {
proxy_pass http://imageforge_api;
proxy_read_timeout 300s;
proxy_send_timeout 300s;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;

View File

@@ -214,9 +214,11 @@ Content-Type: application/json
响应:
```json
{ "success": true, "data": { "message": "邮箱验证成功" } }
{ "success": true, "data": { "message": "邮箱验证成功", "session_invalidated": false } }
```
同一路径也用于确认邮箱变更。邮箱变更成功时 `session_invalidated=true`,服务端会提升 `token_version`,客户端必须清除旧 JWT 并重新登录。
### 4.7 请求密码重置
```http
POST /auth/forgot-password
@@ -274,7 +276,7 @@ Idempotency-Key: <key> # 建议
| `output_format` | String | 否 | 输出格式:`png/jpeg/webp/avif/gif/bmp/tiff/ico`默认保持原格式ICO 自动等比缩至 256x256 边界) |
| `max_width` | Integer | 否 | 大于 0 的最大宽度(等比缩放) |
| `max_height` | Integer | 否 | 大于 0 的最大高度(等比缩放) |
| `target_size_bytes` | Integer | 否 | 不小于 1024 的目标体积(字节),仅 `jpeg/webp/avif` 输出支持;不能与 `compression_rate` 同时指定 |
| `target_size_bytes` | Integer | 否 | 不小于 1024 的最大输出体积(字节),仅 `jpeg/webp/avif` 输出支持;系统选择上限内的最高保真结果,不用填充凑到固定大小;不能与 `compression_rate` 同时指定 |
| `preserve_metadata` | Boolean | 否 | 是否保留 EXIF/ICC默认 `false`);元数据输出仅支持 `jpeg/png/webp` |
处理约束:
@@ -447,7 +449,9 @@ Authorization: Bearer <token>
"id": "550e8400-e29b-41d4-a716-446655440000",
"email": "user@example.com",
"username": "myusername",
"role": "user"
"role": "user",
"email_verified": true,
"pending_email": null
}
}
```
@@ -588,7 +592,6 @@ Authorization: Bearer <token>
POST /billing/checkout
Authorization: Bearer <token>
Content-Type: application/json
Idempotency-Key: <key>
```
请求体:
@@ -601,6 +604,8 @@ Idempotency-Key: <key>
{ "success": true, "data": { "checkout_url": "https://pay.example.com/..." } }
```
Checkout 的 Customer 与 Session 幂等键由服务端按用户和待支付记录生成,客户端无需也不能决定该幂等边界。同一用户同一时间只允许一个未过期的 Checkout已有未取消 Stripe 订阅时返回 `409 IDEMPOTENCY_CONFLICT`,套餐调整必须使用 Portal。
### 9.5 打开客户 Portal管理支付方式/取消订阅)
```http
POST /billing/portal
@@ -696,6 +701,17 @@ Authorization: Bearer <admin_token>
Content-Type: application/json
```
修改用户名只需提交 `username`。修改邮箱必须同时提交当前密码:
```json
{
"email": "new@example.com",
"current_password": "current-password"
}
```
邮箱验证开启时,新邮箱只写入 `pending_email`,确认前主邮箱、登录邮箱和密码恢复地址均保持不变;确认后服务端原子切换主邮箱并撤销全部旧 JWT 与未使用的密码重置链接。邮箱验证关闭时,当前密码校验通过后立即切换邮箱,响应会返回替换当前会话使用的新 `token`
### 11.6 S3 存储端点
```http

View File

@@ -278,6 +278,7 @@ dotenvy = "0.15"
- 图片压缩使用 `spawn_blocking` 避免阻塞异步线程
- `WORKER_TASK_CONCURRENCY` 控制任务级并发,避免大批量任务独占 Worker
- `WORKER_CONCURRENCY` 控制单任务内文件并发,`IMAGE_PROCESSING_CONCURRENCY` 作为进程级 CPU 闸门
- `ZIP_BUILD_CONCURRENCY` 是 API 进程级 ZIP 闸门;数据库租约保证同一任务跨实例只构建一次,`ZIP_MAX_ENTRIES``ZIP_MAX_UNCOMPRESSED_BYTES` 在下载源对象前拒绝超预算任务
```rust
// 在独立线程池中执行 CPU 密集型压缩
@@ -290,6 +291,7 @@ let result = tokio::task::spawn_blocking(move || {
- 流式处理大文件
- 限制并发压缩任务数
- 压缩完成后立即清理临时文件
- ZIP attempt 使用独立临时目录和对象键;发布 CAS 失败时立即删除,两项默认并发且每项 2 GiB 上限时应至少预留约 8 GiB 临时磁盘余量
### 3. 缓存策略
- Redis 缓存用户会话

View File

@@ -100,6 +100,8 @@ RETURNING used_units;
说明:
- 批量任务的计量仍以“成功文件数”为准;失败文件(含 `QUOTA_EXCEEDED`)不计费。
- 前端建议在上传前调用 `GET /billing/usage`登录或读取配额头API做本地提示/拦截。
- 匿名批量任务先按文件数预留当日额度,终态结算只退还失败或未完成文件。未提供 `compression_rate` 属于正常压缩并计量;只有显式 `compression_rate=100`、同格式且无缩放的原样请求免计量。
- 匿名单文件同样先预留,但响应中的 `units_charged` 只由实际输出决定:原样请求或输出未缩小均为 0。预留日期、session/IP 和任务 ID 会持久化;失败、跨日及进程中断由 Redis marker 幂等退款,不能退到请求结束时的新日期。
---
@@ -130,14 +132,20 @@ RETURNING used_units;
- `payments.provider_payment_id` ↔ Stripe `payment_intent.id`(或 charge id按实现选
### 4.2 Checkout / Portal
- Checkout后端创建 Stripe Checkout Session前端跳转 `checkout_url`
- Checkout后端按用户行锁串行创建 Stripe Checkout Session前端跳转 `checkout_url`服务端使用稳定 Customer 幂等键和待支付记录 ID 对应的 Session 幂等键,不依赖客户端 `Idempotency-Key`
- 每个用户只能映射一个非空 Stripe Customer同时只能存在一个未过期待支付记录和一个未取消 Stripe 订阅。并发请求复用同一 SessionCustomer 映射未持久化时禁止返回或创建 Session。
- 已有未取消 Stripe 订阅的用户不能再次进入订阅 Checkout升级、降级、续费和取消统一走 Portal避免多重周期扣费。
- Portal后端创建 Stripe Billing Portal Session前端跳转管理支付方式/取消订阅。
### 4.3 Stripe Webhook商用必须
要求:
- **验签**:使用 `STRIPE_WEBHOOK_SECRET` 校验 `Stripe-Signature`
- **事件幂等**:按 `provider_event_id` 去重(落库 `webhook_events`)。
- **乱序容忍**以 Stripe 事件时间 + 当前 DB 状态做“只前进不回退”更新
- **乱序容忍**订阅对象按 `(event.created, 事件优先级)` 保存独立水位;`deleted` 即使先到也会保留 tombstone`created/updated` 不得恢复已取消订阅
- **同秒歧义**:两个不同事件具有相同 `(event.created, 事件优先级)` 时,不能用不透明的 Event ID 排序,必须从 Stripe 拉取当前订阅快照并以快照响应时间推进水位。
- **迁移对账**:历史版本用本地 `subscriptions.updated_at` 播种的非终态水位会标记为待对账API 后台任务持租约获取 Stripe 快照,成功后才清除标记。未映射 Customer 或 Price 的受管订阅事件返回失败并等待重试,不能标记为已处理。
- **发票一致性**`invoices(provider, provider_invoice_id)` 唯一,发票事件也使用对象水位;新 `invoice.paid` 不会被迟到的旧 `invoice.payment_failed` 回退。同秒同等级事件从 Stripe 获取权威发票快照,未映射 Customer 时返回失败重试。迁移前已有 Stripe 发票会播种为待对账哨兵,首个后续事件必须先取权威快照;非 `paid` 状态不允许保留 `paid_at`
- **并发一致性**`subscriptions(provider, provider_subscription_id)` 唯一,订阅业务写入与 `webhook_events=processed` 在同一事务提交。
- **可重放**:保存原始 payload脱敏用于排查。
建议首期处理的事件(示例):

View File

@@ -310,6 +310,15 @@ CREATE UNIQUE INDEX idx_webhook_events_unique ON webhook_events(provider, provid
CREATE INDEX idx_webhook_events_status ON webhook_events(status);
```
Stripe 运行时还通过迁移维护三组一致性结构:
- `billing_checkout_sessions` 持久化每用户唯一的待支付 Session 及处理租约,防止并发创建多个 Customer/Session。
- `provider_object_event_watermarks` 以 Stripe `event.created` 和事件等级保存对象水位;同秒同等级的不同事件标记为歧义并触发权威快照,不能按 Event ID 字典序决定先后。
- `stripe_subscription_reconciliations` 保存历史非因果水位的租约化对账任务,允许多 API 实例用 `FOR UPDATE SKIP LOCKED` 安全消费。
迁移 `020` 会为尚无水位的历史 Stripe 发票写入 `requires_reconciliation=true` 哨兵。首个后续发票事件必须从 Stripe 获取当前对象后才能覆盖本地记录;`invoices_paid_at_status_check` 同时保证只有 `paid` 状态可以携带 `paid_at`
数据库唯一索引同时保证非空 `users.billing_customer_id` 全局唯一、`subscriptions(provider, provider_subscription_id)` 唯一、非空 `invoices(provider, provider_invoice_id)` 唯一,以及每用户最多一条未取消 Stripe 订阅。部署这些索引前必须先清理存量冲突,具体检查见 `docs/deployment.md`
### 4.8 tasks - 压缩任务
```sql
CREATE TABLE tasks (
@@ -348,7 +357,10 @@ CREATE TABLE tasks (
zip_storage_endpoint_id UUID REFERENCES storage_endpoints(id) ON DELETE RESTRICT,
zip_storage_key TEXT,
zip_storage_etag TEXT,
zip_size BIGINT
zip_size BIGINT,
zip_build_token UUID,
zip_build_lease_until TIMESTAMPTZ,
zip_build_attempt BIGINT NOT NULL DEFAULT 0
);
CREATE INDEX idx_tasks_user_id ON tasks(user_id);
@@ -358,6 +370,10 @@ CREATE INDEX idx_tasks_created_at ON tasks(created_at);
CREATE INDEX idx_tasks_expires_at ON tasks(expires_at);
```
`zip_build_token/zip_build_lease_until` 是跨 API 实例的任务级 single-flight 租约。每个构建 attempt 写入独立对象键,只有 token 匹配的 CAS 更新可以发布到 `zip_storage_*`;失败或失租 attempt 必须删除对象。
匿名单文件预留单独存入 `anonymous_single_reservations`,不依赖尚未创建的 `tasks` 外键。`pending` 超时或 `refund_pending` 记录由 Worker 维护循环使用任务级 Redis marker 补偿;`charged/refunded` 记录保留 7 天后清理。
### 4.9 task_files - 任务文件
```sql
CREATE TABLE task_files (

View File

@@ -13,6 +13,8 @@
Debian 13、4 核 CPU、8GB 内存的起始值建议为 `WORKER_TASK_CONCURRENCY=2``WORKER_CONCURRENCY=2``IMAGE_PROCESSING_CONCURRENCY=2`8 核应用服务器可从 `4/2/4` 开始。三者分别表示同时处理的任务数、单任务内文件数和单进程 CPU 图片处理上限。最后一项是全局 CPU 闸门,因此不要把它设置为 CPU 核数的数倍。任务并发提高后,数据库连接池建议至少为 `WORKER_TASK_CONCURRENCY * WORKER_CONCURRENCY + 4`,生产示例使用 16。
8 核 16GB 应用服务器的 ZIP 起始值为 `ZIP_BUILD_CONCURRENCY=2``ZIP_MAX_ENTRIES=200``ZIP_MAX_UNCOMPRESSED_BYTES=2147483648`。ZIP 使用 stored 模式,构建时同时存在下载源和归档文件,按两个 2 GiB 构建估算应至少保留约 8 GiB 临时磁盘余量;磁盘较小时应先降低总字节或并发,而不是提高 HTTP 超时。
生产 Compose 默认按 8 核 16GB 主机设置可覆盖的资源上限API 3GB、Worker 8GB、PostgreSQL 2GB、Redis 1GB对应变量为 `API_MEMORY_LIMIT``WORKER_MEMORY_LIMIT``POSTGRES_MEMORY_LIMIT``REDIS_MEMORY_LIMIT`。Redis 的 `REDIS_MAXMEMORY` 默认 768MB达到上限后返回写入错误而不是继续挤占宿主机内存。
### 首次启动
@@ -55,15 +57,55 @@ curl --fail http://127.0.0.1:8080/metrics
### 更新与回滚
更新代码后保留 `.env.production` 和命名卷,重新构建并滚动重建:
更新代码后保留 `.env.production` 和命名卷。包含迁移 `017``022` 的版本不能让旧、新 API 或 Worker 并行滚动:旧 API 不理解 ZIP 构建租约,旧 Worker 不理解任务 attempt fencing。先备份数据库并停止旧 API/Worker再构建新镜像。
迁移 `017` 会在发现重复 Customer 或同用户多条未取消 Stripe 订阅时主动失败,迁移 `019` 会在发现同一 Stripe 发票对应多行时主动失败。部署前先检查并人工对账,三个查询都必须返回 0 行:
```sql
SELECT billing_customer_id, COUNT(*)
FROM users
WHERE billing_customer_id IS NOT NULL AND billing_customer_id <> ''
GROUP BY billing_customer_id
HAVING COUNT(*) > 1;
SELECT user_id, COUNT(*)
FROM subscriptions
WHERE provider = 'stripe' AND status <> 'canceled'
GROUP BY user_id
HAVING COUNT(*) > 1;
SELECT provider, provider_invoice_id, COUNT(*)
FROM invoices
WHERE provider_invoice_id IS NOT NULL
GROUP BY provider, provider_invoice_id
HAVING COUNT(*) > 1;
```
推荐顺序:
```bash
git pull --ff-only
docker compose --env-file .env.production -f docker/docker-compose.prod.yml stop api worker
docker compose --env-file .env.production -f docker/docker-compose.prod.yml build api
docker compose --env-file .env.production -f docker/docker-compose.prod.yml up -d
docker compose --env-file .env.production -f docker/docker-compose.prod.yml up -d postgres redis api
docker compose --env-file .env.production -f docker/docker-compose.prod.yml up -d worker
```
生产镜像应使用不可变的 `IMAGEFORGE_TAG`。回滚时把该值改回上一镜像标签,然后再次运行 `up -d`
新 API 启动后会消费迁移 `018` 创建的订阅对账队列。迁移 `020` 为历史发票写入待对账哨兵,发票不主动批量拉取,而是在首个后续事件到达时取 Stripe 快照。启动 Worker 前应确认 API 健康、`STRIPE_SECRET_KEY` 可用且服务器能访问 `STRIPE_API_BASE_URL`;订阅对账可以后台继续,但必须监控失败项:
```sql
SELECT status, COUNT(*)
FROM stripe_subscription_reconciliations
GROUP BY status;
SELECT object_type, requires_reconciliation, COUNT(*)
FROM provider_object_event_watermarks
WHERE provider = 'stripe'
GROUP BY object_type, requires_reconciliation
ORDER BY object_type, requires_reconciliation;
```
`failed` 会指数退避重试;持续失败通常表示 Stripe 凭据、网络、Customer/Price 映射不完整。上线验收要求订阅队列的 `pending/processing/failed` 最终归零,且 subscription 水位不再待对账invoice 水位在对应发票首个后续事件到达前保持 `requires_reconciliation=true` 属于预期状态。生产镜像应使用不可变的 `IMAGEFORGE_TAG`。数据库迁移已应用后,不能只回滚旧二进制;应保留新 schema并使用兼容该 schema 的修复镜像。
### 反向代理

View File

@@ -357,9 +357,11 @@ Content-Type: application/json
**响应**
```json
{ "success": true, "data": { "message": "邮箱验证成功" } }
{ "success": true, "data": { "message": "邮箱验证成功", "session_invalidated": false } }
```
邮箱变更复用该确认入口,但申请变更必须先通过当前密码校验。新邮箱确认前不会替换 `users.email`,因此不能作为密码恢复地址;确认时会原子切换邮箱、提升 `token_version`、撤销未使用的密码重置链接,并向旧邮箱发送安全通知。反向地,成功修改或重置密码也会在同一用户行锁事务内撤销所有未使用重置链接和待确认邮箱变更。邮箱变更响应的 `session_invalidated``true`,客户端应要求重新登录。
### 6.3 请求密码重置
```http

View File

@@ -66,7 +66,7 @@ flowchart LR
| 低级会员 Pro | 7 天 | `results/7d/``archives/7d/` | 9 天 |
| 高级会员 Business | 15 天 | `results/15d/``archives/15d/` | 17 天 |
Worker 每 5 分钟按 `expires_at` 精确删除对象删除成功后才删除数据库任务。S3 生命周期多保留 2 天,只负责处理数据库故障、进程崩溃或上传后未能落库的孤儿对象,不能作为精确会员权限判断。未完成的分片上传 1 天后由生命周期中止。
Worker 每 5 分钟按 `expires_at` 精确删除对象,删除成功后才删除数据库任务。ZIP 构建 attempt 位于对应 `archives/<retention>/.../attempts/` 前缀,发布失败会立即删除,进程崩溃遗留项仍由同一前缀生命周期兜底。S3 生命周期多保留 2 天,只负责处理数据库故障、进程崩溃或上传后未能落库的孤儿对象,不能作为精确会员权限判断。未完成的分片上传 1 天后由生命周期中止。
## 5. 119 首期容量

View File

@@ -31,7 +31,7 @@
### 2.2 JWT 使用建议
- 对外 API支持 Bearer Token适合 CLI/SDK
- 网站Vue3优先使用 HttpOnly Cookie 承载会话(降低 XSS 泄露风险),如使用 localStorage 必须配合严格 CSP。
- JWT 包含用户 `token_version`;修改或重置密码会递增版本,使此前签发的 JWT 立即失效。
- JWT 包含用户 `token_version`;修改或重置密码会递增版本,使此前签发的 JWT 立即失效。成功修改或重置密码时,服务端持有用户行锁并在同一事务中消费该用户全部未使用重置链接、撤销全部待确认邮箱变更,避免旧恢复凭据再次接管账号。
---

View File

@@ -24,6 +24,7 @@ interface UploadItem {
status: ItemStatus
result?: CompressResponse
error?: string
targetSizeBytes?: number
}
const auth = useAuthStore()
@@ -135,6 +136,28 @@ function getTargetSizeBytes(): number | undefined {
return Math.round(bytes)
}
function targetOutputFormat(file: File): OutputFormat {
const mime = file.type.trim().toLowerCase()
if (mime === 'image/jpeg' || mime === 'image/jpg') return 'jpeg'
if (mime === 'image/webp') return 'webp'
if (mime === 'image/avif') return 'avif'
const extension = file.name.split('.').pop()?.toLowerCase()
if (extension === 'jpg' || extension === 'jpeg') return 'jpeg'
if (extension === 'webp') return 'webp'
if (extension === 'avif') return 'avif'
return 'webp'
}
function targetResultHint(item: UploadItem): string | null {
if (!item.result || !item.targetSizeBytes) return null
const target = formatBytes(item.targetSizeBytes)
if (item.result.compressed_size * 4 < item.targetSizeBytes * 3) {
return `体积上限 ${target};当前格式的最高保真结果本身更小,不会添加无效填充。`
}
return `体积上限 ${target};结果已控制在上限内。`
}
function setCompressionMode(mode: CompressionMode) {
options.mode = mode
if (
@@ -159,7 +182,7 @@ async function runOne(item: UploadItem) {
}
const outputFormat: OutputFormat | undefined = options.outputFormat === 'auto'
? (options.mode === 'size' ? 'webp' : undefined)
? (options.mode === 'size' ? targetOutputFormat(item.file) : undefined)
: options.outputFormat
if (options.mode === 'size' && outputFormat && !targetSizeFormats.has(outputFormat)) {
@@ -188,6 +211,7 @@ async function runOne(item: UploadItem) {
auth.token,
)
item.targetSizeBytes = targetSizeBytes
item.result = result
item.status = 'done'
} catch (err) {
@@ -452,6 +476,9 @@ async function resendVerification() {
{{ item.result.saved_percent.toFixed(2) }}%
</template>
</div>
<div v-if="targetResultHint(item)" class="mt-1 text-xs text-slate-500">
{{ targetResultHint(item) }}
</div>
<div v-if="item.error" class="mt-1 text-xs text-rose-700">{{ item.error }}</div>
</div>
@@ -547,7 +574,7 @@ async function resendVerification() {
:class="options.mode === 'size' ? 'bg-indigo-600 text-white' : 'text-slate-600 hover:bg-slate-100'"
@click="setCompressionMode('size')"
>
目标大小
体积上限
</button>
</div>
</div>
@@ -573,7 +600,7 @@ async function resendVerification() {
<!-- 目标大小模式 -->
<div v-else class="space-y-1">
<div class="text-xs font-medium text-slate-600">目标大小</div>
<div class="text-xs font-medium text-slate-600">最大输出大小</div>
<div class="flex gap-2">
<input
v-model="options.targetSize"
@@ -591,7 +618,7 @@ async function resendVerification() {
</select>
</div>
<div class="text-xs text-slate-500">
仅支持 JPEG/WebP/AVIF保持原格式时会自动输出 WebP过小且无法保证清晰度的目标会被拒绝
这是体积上限不是固定输出大小系统会优先使用原格式和最高可用画质最高画质结果更小时不会填充无效数据
</div>
</div>
@@ -601,7 +628,7 @@ async function resendVerification() {
v-model="options.outputFormat"
class="w-full rounded-md border border-slate-200 bg-white px-3 py-2 text-sm text-slate-800"
>
<option value="auto">{{ options.mode === 'size' ? '自动选择 WebP推荐' : '保持原格式(推荐)' }}</option>
<option value="auto">{{ options.mode === 'size' ? '智能选择(优先原格式' : '保持原格式(推荐)' }}</option>
<option value="jpeg">JPEG</option>
<option value="png" :disabled="options.mode === 'size'">PNG</option>
<option value="webp">WebP</option>
@@ -611,7 +638,7 @@ async function resendVerification() {
<option value="tiff" :disabled="options.mode === 'size'">TIFF</option>
<option value="ico" :disabled="options.mode === 'size'">ICO</option>
</select>
<div class="text-xs text-slate-500">支持按需转码目标大小模式建议配合 JPEG/WebP/AVIF</div>
<div class="text-xs text-slate-500">支持按需转码体积上限模式仅使用 JPEG/WebP/AVIF其他输入会自动转为 WebP</div>
</label>
<div class="grid grid-cols-2 gap-3">

View File

@@ -25,7 +25,11 @@ onMounted(async () => {
try {
const resp = await verifyEmail(token.value)
message.value = resp.message
auth.markEmailVerified()
if (resp.session_invalidated) {
auth.logout()
} else {
auth.markEmailVerified()
}
} catch (err) {
if (err instanceof ApiError) {
error.value = `[${err.code}] ${err.message}`

View File

@@ -212,7 +212,7 @@ onMounted(async () => {
>
{{ subBusy ? '提交中…' : '立即开通' }}
</button>
<span class="text-xs text-slate-500">取消该用户当前有效订阅并按月数顺延</span>
<span class="text-xs text-slate-500">替换当前本地套餐存在未取消 Stripe 订阅时将拒绝操作</span>
</div>
<div v-if="subMessage" class="mt-3 rounded-lg border border-emerald-200 bg-emerald-50 p-3 text-sm text-emerald-900">

View File

@@ -9,7 +9,7 @@ const auth = useAuthStore()
const loading = ref(true)
const profileForm = ref({ email: '', username: '' })
const profileForm = ref({ email: '', username: '', currentPassword: '' })
const profileBusy = ref(false)
const profileMessage = ref<string | null>(null)
const profileError = ref<string | null>(null)
@@ -24,9 +24,13 @@ const verificationMessage = ref<string | null>(null)
const verificationError = ref<string | null>(null)
const canResendVerification = computed(() => Boolean(auth.user && !auth.user.email_verified))
const emailChangeRequested = computed(() => {
const email = profileForm.value.email.trim().toLowerCase()
return Boolean(email && auth.user && email !== auth.user.email)
})
function syncProfile(user: UserProfile) {
profileForm.value = { email: user.email ?? '', username: user.username ?? '' }
profileForm.value = { email: user.email ?? '', username: user.username ?? '', currentPassword: '' }
}
onMounted(async () => {
@@ -77,9 +81,16 @@ async function saveProfile() {
try {
const email = profileForm.value.email.trim().toLowerCase()
const username = profileForm.value.username.trim()
const payload: { email?: string; username?: string } = {}
const payload: { email?: string; username?: string; current_password?: string } = {}
if (email && email !== auth.user.email) payload.email = email
if (email && email !== auth.user.email) {
if (!profileForm.value.currentPassword) {
profileError.value = '修改邮箱需要输入当前密码'
return
}
payload.email = email
payload.current_password = profileForm.value.currentPassword
}
if (username && username !== auth.user.username) payload.username = username
if (!payload.email && !payload.username) {
@@ -88,7 +99,11 @@ async function saveProfile() {
}
const resp = await updateProfile(auth.token, payload)
auth.updateUser(resp.user)
if (resp.token) {
auth.setAuth(resp.token, resp.user)
} else {
auth.updateUser(resp.user)
}
syncProfile(resp.user)
profileMessage.value = resp.message
} catch (err) {
@@ -178,6 +193,24 @@ async function changePassword() {
</label>
</div>
<label v-if="emailChangeRequested" class="mt-3 block max-w-md space-y-1">
<div class="text-xs font-medium text-slate-600">当前密码修改邮箱必填</div>
<input
v-model="profileForm.currentPassword"
type="password"
autocomplete="current-password"
class="w-full rounded-md border border-slate-200 bg-white px-3 py-2 text-sm text-slate-800"
placeholder="用于确认是本人操作"
/>
</label>
<div
v-if="auth.user?.pending_email"
class="mt-3 rounded-lg border border-amber-200 bg-amber-50 p-3 text-sm text-amber-900"
>
待确认新邮箱{{ auth.user.pending_email }}确认前仍使用当前邮箱登录和找回密码
</div>
<div v-if="profileMessage" class="mt-4 rounded-lg border border-emerald-200 bg-emerald-50 p-3 text-sm text-emerald-900">
{{ profileMessage }}
</div>

View File

@@ -29,8 +29,14 @@ export async function sendVerification(token: string): Promise<{ message: string
return apiJson<{ message: string }>('/api/v1/auth/send-verification', undefined, token, { method: 'POST' })
}
export async function verifyEmail(verificationToken: string): Promise<{ message: string }> {
return apiJson<{ message: string }>('/api/v1/auth/verify-email', { token: verificationToken }, null)
export async function verifyEmail(
verificationToken: string,
): Promise<{ message: string; session_invalidated: boolean }> {
return apiJson<{ message: string; session_invalidated: boolean }>(
'/api/v1/auth/verify-email',
{ token: verificationToken },
null,
)
}
export async function forgotPassword(email: string): Promise<{ message: string }> {
@@ -49,9 +55,11 @@ export async function getProfile(token: string): Promise<UserProfile> {
export async function updateProfile(
token: string,
payload: { email?: string; username?: string },
): Promise<{ user: UserProfile; message: string }> {
return apiJson<{ user: UserProfile; message: string }>('/api/v1/user/profile', payload, token, { method: 'PUT' })
payload: { email?: string; username?: string; current_password?: string },
): Promise<{ user: UserProfile; message: string; token?: string }> {
return apiJson<{ user: UserProfile; message: string; token?: string }>('/api/v1/user/profile', payload, token, {
method: 'PUT',
})
}
export async function updatePassword(

View File

@@ -8,6 +8,7 @@ export interface User {
username: string
role: UserRole
email_verified: boolean
pending_email?: string | null
}
interface StoredAuth {

View File

@@ -0,0 +1,25 @@
CREATE TABLE IF NOT EXISTS email_change_requests (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
new_email VARCHAR(255) NOT NULL,
token_hash VARCHAR(64) NOT NULL,
expires_at TIMESTAMPTZ NOT NULL,
confirmed_at TIMESTAMPTZ,
canceled_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_email_change_requests_token
ON email_change_requests(token_hash);
CREATE UNIQUE INDEX IF NOT EXISTS idx_email_change_requests_pending_user
ON email_change_requests(user_id)
WHERE confirmed_at IS NULL AND canceled_at IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS idx_email_change_requests_pending_email
ON email_change_requests(new_email)
WHERE confirmed_at IS NULL AND canceled_at IS NULL;
CREATE INDEX IF NOT EXISTS idx_email_change_requests_expires
ON email_change_requests(expires_at)
WHERE confirmed_at IS NULL AND canceled_at IS NULL;

View File

@@ -0,0 +1,17 @@
ALTER TABLE tasks
ADD COLUMN IF NOT EXISTS processing_attempt BIGINT NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS lease_owner UUID,
ADD COLUMN IF NOT EXISTS lease_until TIMESTAMPTZ;
ALTER TABLE task_files
ADD COLUMN IF NOT EXISTS processing_attempt BIGINT NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS lease_owner UUID,
ADD COLUMN IF NOT EXISTS lease_until TIMESTAMPTZ;
CREATE INDEX IF NOT EXISTS idx_tasks_processing_lease
ON tasks(lease_until)
WHERE status = 'processing';
CREATE INDEX IF NOT EXISTS idx_task_files_processing_lease
ON task_files(task_id, lease_until)
WHERE status = 'processing';

View File

@@ -0,0 +1,95 @@
-- Repoint references before enforcing one row per provider subscription.
WITH ranked AS (
SELECT
id,
FIRST_VALUE(id) OVER (
PARTITION BY provider, provider_subscription_id
ORDER BY updated_at DESC, created_at DESC, id DESC
) AS keep_id,
ROW_NUMBER() OVER (
PARTITION BY provider, provider_subscription_id
ORDER BY updated_at DESC, created_at DESC, id DESC
) AS row_number
FROM subscriptions
WHERE provider_subscription_id IS NOT NULL
)
UPDATE usage_periods AS usage
SET subscription_id = ranked.keep_id
FROM ranked
WHERE ranked.row_number > 1
AND usage.subscription_id = ranked.id;
WITH ranked AS (
SELECT
id,
FIRST_VALUE(id) OVER (
PARTITION BY provider, provider_subscription_id
ORDER BY updated_at DESC, created_at DESC, id DESC
) AS keep_id,
ROW_NUMBER() OVER (
PARTITION BY provider, provider_subscription_id
ORDER BY updated_at DESC, created_at DESC, id DESC
) AS row_number
FROM subscriptions
WHERE provider_subscription_id IS NOT NULL
)
UPDATE invoices AS invoice
SET subscription_id = ranked.keep_id
FROM ranked
WHERE ranked.row_number > 1
AND invoice.subscription_id = ranked.id;
WITH ranked AS (
SELECT
id,
ROW_NUMBER() OVER (
PARTITION BY provider, provider_subscription_id
ORDER BY updated_at DESC, created_at DESC, id DESC
) AS row_number
FROM subscriptions
WHERE provider_subscription_id IS NOT NULL
)
DELETE FROM subscriptions AS subscription
USING ranked
WHERE ranked.row_number > 1
AND subscription.id = ranked.id;
CREATE UNIQUE INDEX IF NOT EXISTS idx_subscriptions_provider_object_unique
ON subscriptions(provider, provider_subscription_id);
CREATE TABLE IF NOT EXISTS provider_object_event_watermarks (
provider VARCHAR(20) NOT NULL,
object_type VARCHAR(50) NOT NULL,
provider_object_id VARCHAR(200) NOT NULL,
last_event_created BIGINT NOT NULL,
last_event_rank SMALLINT NOT NULL DEFAULT 0,
last_event_id VARCHAR(200) NOT NULL,
is_deleted BOOLEAN NOT NULL DEFAULT false,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (provider, object_type, provider_object_id),
CONSTRAINT provider_object_event_watermarks_rank_check
CHECK (last_event_rank BETWEEN 0 AND 100)
);
CREATE INDEX IF NOT EXISTS idx_provider_object_event_watermarks_updated
ON provider_object_event_watermarks(updated_at);
-- Seed rollout watermarks so a delayed pre-deployment event cannot revive an
-- already-canceled Stripe subscription before the first new event arrives.
INSERT INTO provider_object_event_watermarks (
provider, object_type, provider_object_id,
last_event_created, last_event_rank, last_event_id, is_deleted, updated_at
)
SELECT
'stripe',
'subscription',
provider_subscription_id,
EXTRACT(EPOCH FROM updated_at)::bigint,
CASE WHEN status = 'canceled' THEN 2 ELSE 1 END,
'migration:' || id::text,
status = 'canceled',
updated_at
FROM subscriptions
WHERE provider = 'stripe'
AND provider_subscription_id IS NOT NULL
ON CONFLICT (provider, object_type, provider_object_id) DO NOTHING;

View File

@@ -0,0 +1,59 @@
DO $$
BEGIN
IF EXISTS (
SELECT 1
FROM users
WHERE billing_customer_id IS NOT NULL AND billing_customer_id <> ''
GROUP BY billing_customer_id
HAVING COUNT(*) > 1
) THEN
RAISE EXCEPTION 'duplicate users.billing_customer_id values require Stripe reconciliation before migration 017';
END IF;
IF EXISTS (
SELECT 1
FROM subscriptions
WHERE provider = 'stripe' AND status <> 'canceled'
GROUP BY user_id
HAVING COUNT(*) > 1
) THEN
RAISE EXCEPTION 'multiple open Stripe subscriptions per user require reconciliation before migration 017';
END IF;
END $$;
CREATE UNIQUE INDEX IF NOT EXISTS idx_users_billing_customer_unique
ON users(billing_customer_id)
WHERE billing_customer_id IS NOT NULL AND billing_customer_id <> '';
CREATE UNIQUE INDEX IF NOT EXISTS idx_subscriptions_user_open_stripe_unique
ON subscriptions(user_id)
WHERE provider = 'stripe' AND status <> 'canceled';
CREATE TABLE IF NOT EXISTS billing_checkout_sessions (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
plan_id UUID NOT NULL REFERENCES plans(id),
stripe_customer_id VARCHAR(200),
stripe_session_id VARCHAR(200),
checkout_url TEXT,
status VARCHAR(20) NOT NULL DEFAULT 'pending',
expires_at TIMESTAMPTZ NOT NULL,
lease_owner UUID,
lease_until TIMESTAMPTZ,
error_message TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
completed_at TIMESTAMPTZ,
CONSTRAINT billing_checkout_sessions_status_check
CHECK (status IN ('pending', 'completed', 'expired', 'failed', 'canceled')),
CONSTRAINT billing_checkout_sessions_stripe_session_unique
UNIQUE (stripe_session_id)
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_billing_checkout_sessions_user_pending
ON billing_checkout_sessions(user_id)
WHERE status = 'pending';
CREATE INDEX IF NOT EXISTS idx_billing_checkout_sessions_expiry
ON billing_checkout_sessions(expires_at)
WHERE status = 'pending';

View File

@@ -0,0 +1,57 @@
ALTER TABLE provider_object_event_watermarks
ADD COLUMN IF NOT EXISTS requires_reconciliation BOOLEAN NOT NULL DEFAULT false,
ADD COLUMN IF NOT EXISTS reconciliation_reason TEXT,
ADD COLUMN IF NOT EXISTS last_snapshot_at TIMESTAMPTZ;
CREATE TABLE IF NOT EXISTS stripe_subscription_reconciliations (
provider_subscription_id VARCHAR(200) PRIMARY KEY,
reason TEXT NOT NULL,
status VARCHAR(20) NOT NULL DEFAULT 'pending',
attempts INTEGER NOT NULL DEFAULT 0,
next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_error TEXT,
lease_owner UUID,
lease_until TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
completed_at TIMESTAMPTZ,
CONSTRAINT stripe_subscription_reconciliations_status_check
CHECK (status IN ('pending', 'processing', 'completed', 'failed'))
);
WITH corrected AS (
UPDATE provider_object_event_watermarks
SET last_event_created = 0,
last_event_rank = 0,
last_event_id = 'reconcile:migration',
requires_reconciliation = true,
reconciliation_reason = 'migration_016_non_causal_seed',
updated_at = NOW()
WHERE provider = 'stripe'
AND object_type = 'subscription'
AND is_deleted = false
AND last_event_id LIKE 'migration:%'
RETURNING provider_object_id
)
INSERT INTO stripe_subscription_reconciliations (
provider_subscription_id, reason, status, next_attempt_at
)
SELECT
provider_object_id,
'migration_016_non_causal_seed',
'pending',
NOW()
FROM corrected
ON CONFLICT (provider_subscription_id) DO UPDATE
SET reason = EXCLUDED.reason,
status = 'pending',
next_attempt_at = NOW(),
last_error = NULL,
lease_owner = NULL,
lease_until = NULL,
completed_at = NULL,
updated_at = NOW();
CREATE INDEX IF NOT EXISTS idx_stripe_subscription_reconciliations_ready
ON stripe_subscription_reconciliations(next_attempt_at, updated_at)
WHERE status IN ('pending', 'failed');

View File

@@ -0,0 +1,16 @@
DO $$
BEGIN
IF EXISTS (
SELECT 1
FROM invoices
WHERE provider_invoice_id IS NOT NULL
GROUP BY provider, provider_invoice_id
HAVING COUNT(*) > 1
) THEN
RAISE EXCEPTION 'duplicate invoices(provider, provider_invoice_id) values require reconciliation before migration 019';
END IF;
END $$;
CREATE UNIQUE INDEX IF NOT EXISTS idx_invoices_provider_object_unique
ON invoices(provider, provider_invoice_id)
WHERE provider_invoice_id IS NOT NULL;

View File

@@ -0,0 +1,31 @@
-- Existing Stripe invoices predate object watermarks. Force the first later
-- event to reconcile against Stripe instead of treating it as authoritative.
INSERT INTO provider_object_event_watermarks (
provider, object_type, provider_object_id,
last_event_created, last_event_rank, last_event_id,
is_deleted, requires_reconciliation, reconciliation_reason,
updated_at
)
SELECT
'stripe', 'invoice', provider_invoice_id,
0, 0, 'reconcile:migration:020',
false, true, 'migration_020_existing_invoice',
NOW()
FROM invoices
WHERE provider = 'stripe'
AND provider_invoice_id IS NOT NULL
ON CONFLICT (provider, object_type, provider_object_id) DO NOTHING;
-- A non-paid invoice must never retain the timestamp from an older paid
-- payload. Clean historical contradictions before enforcing the invariant.
UPDATE invoices
SET paid_at = NULL
WHERE status <> 'paid'
AND paid_at IS NOT NULL;
ALTER TABLE invoices
ADD CONSTRAINT invoices_paid_at_status_check
CHECK (paid_at IS NULL OR status = 'paid') NOT VALID;
ALTER TABLE invoices
VALIDATE CONSTRAINT invoices_paid_at_status_check;

View File

@@ -0,0 +1,12 @@
ALTER TABLE tasks
ADD COLUMN zip_build_token UUID,
ADD COLUMN zip_build_lease_until TIMESTAMPTZ,
ADD COLUMN zip_build_attempt BIGINT NOT NULL DEFAULT 0;
ALTER TABLE tasks
ADD CONSTRAINT tasks_zip_build_lease_pair_check
CHECK ((zip_build_token IS NULL) = (zip_build_lease_until IS NULL));
CREATE INDEX idx_tasks_zip_build_lease
ON tasks(zip_build_lease_until)
WHERE zip_storage_key IS NULL AND zip_build_token IS NOT NULL;

View File

@@ -0,0 +1,20 @@
CREATE TABLE anonymous_single_reservations (
task_id UUID PRIMARY KEY,
session_id VARCHAR(100) NOT NULL,
client_ip INET NOT NULL,
quota_date DATE NOT NULL,
units INTEGER NOT NULL,
status VARCHAR(20) NOT NULL DEFAULT 'pending',
refund_after TIMESTAMPTZ NOT NULL DEFAULT (NOW() + INTERVAL '15 minutes'),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
settled_at TIMESTAMPTZ,
CONSTRAINT anonymous_single_reservations_units_check
CHECK (units > 0),
CONSTRAINT anonymous_single_reservations_status_check
CHECK (status IN ('pending', 'charged', 'refund_pending', 'refunded'))
);
CREATE INDEX anonymous_single_reservations_unsettled
ON anonymous_single_reservations(refund_after, created_at)
WHERE status IN ('pending', 'refund_pending');

View File

@@ -0,0 +1,65 @@
WITH ranked AS (
SELECT
id,
user_id,
provider,
status::text AS previous_status,
current_period_end,
ROW_NUMBER() OVER (
PARTITION BY user_id
ORDER BY
CASE WHEN provider = 'stripe' THEN 0 ELSE 1 END,
current_period_end DESC,
updated_at DESC,
id DESC
) AS position
FROM subscriptions
WHERE status IN ('active', 'trialing', 'past_due')
), duplicates AS (
SELECT *
FROM ranked
WHERE position > 1
)
INSERT INTO audit_logs (
user_id, action, resource_type, resource_id, details
)
SELECT
user_id,
'migration_subscription_dedup',
'subscription',
id,
jsonb_build_object(
'migration', '023_cross_provider_subscription_invariant',
'provider', provider,
'previous_status', previous_status,
'current_period_end', current_period_end,
'reason', 'cross_provider_single_effective_subscription'
)
FROM duplicates;
WITH ranked AS (
SELECT
id,
ROW_NUMBER() OVER (
PARTITION BY user_id
ORDER BY
CASE WHEN provider = 'stripe' THEN 0 ELSE 1 END,
current_period_end DESC,
updated_at DESC,
id DESC
) AS position
FROM subscriptions
WHERE status IN ('active', 'trialing', 'past_due')
)
UPDATE subscriptions AS subscription
SET status = 'canceled',
cancel_at_period_end = false,
canceled_at = COALESCE(subscription.canceled_at, NOW()),
updated_at = NOW()
FROM ranked
WHERE ranked.position > 1
AND subscription.id = ranked.id;
CREATE UNIQUE INDEX idx_subscriptions_user_effective_unique
ON subscriptions(user_id)
WHERE status IN ('active', 'trialing', 'past_due');

View File

@@ -0,0 +1,28 @@
CREATE TABLE task_queue_outbox (
task_id UUID PRIMARY KEY REFERENCES tasks(id) ON DELETE CASCADE,
status VARCHAR(20) NOT NULL DEFAULT 'pending',
attempts INTEGER NOT NULL DEFAULT 0,
next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
lease_owner UUID,
lease_until TIMESTAMPTZ,
last_error TEXT,
delivered_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT task_queue_outbox_status_check
CHECK (status IN ('pending', 'delivering', 'delivered', 'dead')),
CONSTRAINT task_queue_outbox_attempts_check
CHECK (attempts >= 0),
CONSTRAINT task_queue_outbox_lease_pair_check
CHECK ((lease_owner IS NULL) = (lease_until IS NULL))
);
CREATE INDEX task_queue_outbox_ready
ON task_queue_outbox(next_attempt_at, created_at)
WHERE status IN ('pending', 'delivering');
INSERT INTO task_queue_outbox (task_id)
SELECT id
FROM tasks
WHERE status = 'pending'
ON CONFLICT (task_id) DO NOTHING;

View File

@@ -0,0 +1,101 @@
ALTER TABLE tasks
ADD COLUMN deletion_started_at TIMESTAMPTZ,
ADD COLUMN deletion_reason VARCHAR(32);
CREATE INDEX tasks_deletion_pending
ON tasks(deletion_started_at)
WHERE deletion_started_at IS NOT NULL;
CREATE TABLE storage_objects (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
task_id UUID NOT NULL,
task_file_id UUID,
object_kind VARCHAR(32) NOT NULL,
state VARCHAR(20) NOT NULL DEFAULT 'staging',
backend VARCHAR(16) NOT NULL,
storage_endpoint_id UUID REFERENCES storage_endpoints(id) ON DELETE RESTRICT,
object_key TEXT NOT NULL,
storage_etag TEXT,
size_bytes BIGINT,
lease_owner UUID,
lease_until TIMESTAMPTZ,
delete_attempts INTEGER NOT NULL DEFAULT 0,
next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_error TEXT,
published_at TIMESTAMPTZ,
deleted_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT storage_objects_kind_check
CHECK (object_kind IN ('result', 'zip_attempt', 'input', 'input_dir', 'legacy_zip')),
CONSTRAINT storage_objects_state_check
CHECK (state IN ('staging', 'published', 'delete_pending', 'deleted')),
CONSTRAINT storage_objects_backend_check
CHECK (backend IN ('s3', 'local', 'local_dir')),
CONSTRAINT storage_objects_attempts_check
CHECK (delete_attempts >= 0),
CONSTRAINT storage_objects_size_check
CHECK (size_bytes IS NULL OR size_bytes >= 0),
CONSTRAINT storage_objects_endpoint_check
CHECK (
(backend = 's3' AND storage_endpoint_id IS NOT NULL)
OR (backend IN ('local', 'local_dir') AND storage_endpoint_id IS NULL)
),
CONSTRAINT storage_objects_lease_pair_check
CHECK ((lease_owner IS NULL) = (lease_until IS NULL))
);
CREATE UNIQUE INDEX storage_objects_locator_unique
ON storage_objects(
backend,
COALESCE(storage_endpoint_id, '00000000-0000-0000-0000-000000000000'::uuid),
object_key
);
CREATE INDEX storage_objects_cleanup_ready
ON storage_objects(next_attempt_at, created_at)
WHERE state IN ('staging', 'delete_pending');
CREATE INDEX storage_objects_task
ON storage_objects(task_id, state);
INSERT INTO storage_objects (
task_id, task_file_id, object_kind, state,
backend, storage_endpoint_id, object_key, storage_etag,
size_bytes, published_at
)
SELECT
file.task_id,
file.id,
'result',
'published',
file.storage_backend,
file.storage_endpoint_id,
COALESCE(file.storage_key, file.storage_path),
file.storage_etag,
file.compressed_size,
COALESCE(file.completed_at, file.created_at)
FROM task_files AS file
WHERE file.status = 'completed'
AND COALESCE(file.storage_key, file.storage_path) IS NOT NULL
ON CONFLICT DO NOTHING;
INSERT INTO storage_objects (
task_id, object_kind, state,
backend, storage_endpoint_id, object_key, storage_etag,
size_bytes, published_at
)
SELECT
task.id,
'zip_attempt',
'published',
task.zip_storage_backend,
task.zip_storage_endpoint_id,
task.zip_storage_key,
task.zip_storage_etag,
task.zip_size,
COALESCE(task.completed_at, task.created_at)
FROM tasks AS task
WHERE task.zip_storage_backend IS NOT NULL
AND task.zip_storage_key IS NOT NULL
ON CONFLICT DO NOTHING;

View File

@@ -0,0 +1,11 @@
ALTER TABLE idempotency_keys
ADD COLUMN lease_owner UUID,
ADD COLUMN lease_until TIMESTAMPTZ;
ALTER TABLE idempotency_keys
ADD CONSTRAINT idempotency_keys_lease_pair_check
CHECK ((lease_owner IS NULL) = (lease_until IS NULL));
CREATE INDEX idempotency_keys_stale_operations
ON idempotency_keys(lease_until)
WHERE response_status = 0;

View File

@@ -0,0 +1,6 @@
ALTER TABLE tasks
ADD COLUMN target_size_bytes BIGINT;
ALTER TABLE tasks
ADD CONSTRAINT tasks_target_size_bytes_check
CHECK (target_size_bytes IS NULL OR target_size_bytes >= 1024);

View File

@@ -0,0 +1,152 @@
#!/usr/bin/env bash
set -Eeuo pipefail
EXPECTED_EXTERNAL_TESTS="${EXPECTED_EXTERNAL_TESTS:-13}"
WORK_DIR="$(mktemp -d)"
POSTGRES_CONTAINER=""
REDIS_CONTAINER=""
MINIO_PID=""
cleanup() {
local status=$?
trap - EXIT INT TERM
set +e
if [[ -n "$MINIO_PID" ]]; then
kill "$MINIO_PID" 2>/dev/null
wait "$MINIO_PID" 2>/dev/null
fi
if [[ -n "$POSTGRES_CONTAINER" ]]; then
docker rm -f "$POSTGRES_CONTAINER" >/dev/null 2>&1
fi
if [[ -n "$REDIS_CONTAINER" ]]; then
docker rm -f "$REDIS_CONTAINER" >/dev/null 2>&1
fi
rm -rf "$WORK_DIR"
exit "$status"
}
trap cleanup EXIT INT TERM
wait_for_command() {
local description=$1
shift
for _ in $(seq 1 60); do
if "$@" >/dev/null 2>&1; then
return 0
fi
sleep 1
done
echo "Timed out waiting for ${description}" >&2
return 1
}
if [[ -z "${IMAGEFORGE_TEST_DATABASE_URL:-}" || -z "${IMAGEFORGE_TEST_REDIS_URL:-}" ]]; then
if ! command -v docker >/dev/null 2>&1; then
echo "Set IMAGEFORGE_TEST_DATABASE_URL and IMAGEFORGE_TEST_REDIS_URL, or install Docker." >&2
exit 1
fi
suffix="$$-${RANDOM}"
postgres_port="${IMAGEFORGE_TEST_POSTGRES_PORT:-25432}"
redis_port="${IMAGEFORGE_TEST_REDIS_PORT:-26379}"
POSTGRES_CONTAINER="imageforge-test-postgres-${suffix}"
REDIS_CONTAINER="imageforge-test-redis-${suffix}"
docker run -d \
--label imageforge.external-tests=true \
--name "$POSTGRES_CONTAINER" \
-e POSTGRES_PASSWORD=codex_test \
-e POSTGRES_DB=imageforge_test \
-p "127.0.0.1:${postgres_port}:5432" \
postgres:16-alpine >/dev/null
docker run -d \
--label imageforge.external-tests=true \
--name "$REDIS_CONTAINER" \
-p "127.0.0.1:${redis_port}:6379" \
redis:7-alpine >/dev/null
wait_for_command PostgreSQL docker exec "$POSTGRES_CONTAINER" \
pg_isready -U postgres -d imageforge_test
wait_for_command Redis docker exec "$REDIS_CONTAINER" redis-cli ping
export IMAGEFORGE_TEST_DATABASE_URL="postgres://postgres:codex_test@127.0.0.1:${postgres_port}/imageforge_test"
export IMAGEFORGE_TEST_REDIS_URL="redis://127.0.0.1:${redis_port}/"
fi
arch="$(uname -m)"
case "$arch" in
x86_64) minio_arch=amd64 ;;
aarch64|arm64) minio_arch=arm64 ;;
*) echo "Unsupported MinIO architecture: ${arch}" >&2; exit 1 ;;
esac
default_minio_bin="${WORK_DIR}/minio"
default_mc_bin="${WORK_DIR}/mc"
if command -v minio >/dev/null 2>&1; then
default_minio_bin="$(command -v minio)"
fi
if command -v mc >/dev/null 2>&1; then
default_mc_bin="$(command -v mc)"
fi
MINIO_BIN="${MINIO_BIN:-${default_minio_bin}}"
MC_BIN="${MC_BIN:-${default_mc_bin}}"
download_tool() {
local url=$1
local output=$2
local partial="${output}.part"
curl --fail --silent --show-error --location \
--retry 10 --retry-all-errors --retry-delay 2 --connect-timeout 20 \
--continue-at - --output "$partial" "$url"
mv "$partial" "$output"
chmod +x "$output"
}
if [[ ! -x "$MINIO_BIN" ]]; then
download_tool \
"https://dl.min.io/server/minio/release/linux-${minio_arch}/minio" \
"$MINIO_BIN"
fi
if [[ ! -x "$MC_BIN" ]]; then
download_tool \
"https://dl.min.io/client/mc/release/linux-${minio_arch}/mc" \
"$MC_BIN"
fi
export IMAGEFORGE_TEST_S3_ENDPOINT="${IMAGEFORGE_TEST_S3_ENDPOINT:-http://127.0.0.1:19000}"
export IMAGEFORGE_TEST_S3_BUCKET="${IMAGEFORGE_TEST_S3_BUCKET:-imageforge-test}"
export IMAGEFORGE_TEST_S3_ACCESS_KEY="${IMAGEFORGE_TEST_S3_ACCESS_KEY:-codexminio}"
export IMAGEFORGE_TEST_S3_SECRET_KEY="${IMAGEFORGE_TEST_S3_SECRET_KEY:-codexminio123}"
if [[ "$IMAGEFORGE_TEST_S3_ENDPOINT" == "http://127.0.0.1:19000" ]]; then
MINIO_ROOT_USER="$IMAGEFORGE_TEST_S3_ACCESS_KEY" \
MINIO_ROOT_PASSWORD="$IMAGEFORGE_TEST_S3_SECRET_KEY" \
"$MINIO_BIN" server "${WORK_DIR}/minio-data" \
--address 127.0.0.1:19000 >"${WORK_DIR}/minio.log" 2>&1 &
MINIO_PID=$!
wait_for_command MinIO curl --fail --silent \
"${IMAGEFORGE_TEST_S3_ENDPOINT}/minio/health/ready"
fi
"$MC_BIN" alias set imageforge-test \
"$IMAGEFORGE_TEST_S3_ENDPOINT" \
"$IMAGEFORGE_TEST_S3_ACCESS_KEY" \
"$IMAGEFORGE_TEST_S3_SECRET_KEY" >/dev/null
"$MC_BIN" mb --ignore-existing \
"imageforge-test/${IMAGEFORGE_TEST_S3_BUCKET}" >/dev/null
export DATABASE_URL="$IMAGEFORGE_TEST_DATABASE_URL"
export REDIS_URL="$IMAGEFORGE_TEST_REDIS_URL"
export JWT_SECRET="${JWT_SECRET:-imageforge-external-test-jwt-secret}"
export API_KEY_PEPPER="${API_KEY_PEPPER:-imageforge-external-test-api-key-pepper}"
test_log="${WORK_DIR}/external-tests.log"
cargo test --all-targets --locked -- \
--ignored --test-threads=1 --nocapture 2>&1 | tee "$test_log"
expected_summary="test result: ok. ${EXPECTED_EXTERNAL_TESTS} passed; 0 failed; 0 ignored;"
if ! grep --fixed-strings --quiet "$expected_summary" "$test_log"; then
echo "External-state test count mismatch; expected: ${expected_summary}" >&2
exit 1
fi
echo "External-state gate passed: ${EXPECTED_EXTERNAL_TESTS} passed, 0 failed, 0 ignored."

View File

@@ -556,28 +556,14 @@ async fn cancel_task(
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "开启事务失败").with_source(err))?;
sqlx::query(
r#"
UPDATE task_files
SET status = 'failed',
error_message = '任务已取消',
completed_at = NOW()
WHERE task_id = $1 AND status IN ('pending', 'processing')
"#,
)
.bind(task_id)
.execute(&mut *tx)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "更新任务文件失败").with_source(err))?;
let updated = sqlx::query(
r#"
UPDATE tasks
SET status = 'cancelled',
error_message = '管理员取消任务',
completed_at = NOW(),
completed_files = (SELECT COUNT(*) FROM task_files WHERE task_id = $1 AND status = 'completed'),
failed_files = (SELECT COUNT(*) FROM task_files WHERE task_id = $1 AND status = 'failed')
lease_owner = NULL,
lease_until = NULL
WHERE id = $1 AND status IN ('pending', 'processing')
"#,
)
@@ -590,6 +576,44 @@ async fn cancel_task(
return Err(AppError::new(ErrorCode::InvalidRequest, "任务状态无法取消"));
}
sqlx::query(
r#"
UPDATE task_files
SET status = 'failed',
error_message = '任务已取消',
completed_at = NOW(),
lease_owner = NULL,
lease_until = NULL
WHERE task_id = $1 AND status IN ('pending', 'processing')
"#,
)
.bind(task_id)
.execute(&mut *tx)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "更新任务文件失败").with_source(err))?;
sqlx::query(
r#"
UPDATE tasks
SET completed_files = (
SELECT COUNT(*) FROM task_files WHERE task_id = $1 AND status = 'completed'
),
failed_files = (
SELECT COUNT(*) FROM task_files WHERE task_id = $1 AND status = 'failed'
),
total_compressed_size = COALESCE((
SELECT SUM(compressed_size)
FROM task_files
WHERE task_id = $1 AND status = 'completed'
), 0)::bigint
WHERE id = $1 AND status = 'cancelled'
"#,
)
.bind(task_id)
.execute(&mut *tx)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "更新任务统计失败").with_source(err))?;
tx.commit()
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "提交事务失败").with_source(err))?;
@@ -935,28 +959,92 @@ async fn create_manual_subscription(
return Err(AppError::new(ErrorCode::Forbidden, "套餐不可用"));
}
let (subscription_id, period_start, period_end) = persist_manual_subscription(
&state.db,
admin_id,
user_id,
plan.id,
months,
req.note.as_deref(),
ip,
)
.await?;
Ok(Json(Envelope {
success: true,
data: ManualSubscriptionResponse {
message: "套餐已开通".to_string(),
subscription_id,
user_id,
plan_id: plan.id,
plan_name: plan.name,
period_start,
period_end,
status: "active".to_string(),
},
}))
}
async fn persist_manual_subscription(
pool: &sqlx::PgPool,
admin_id: Uuid,
user_id: Uuid,
plan_id: Uuid,
months: i32,
note: Option<&str>,
ip: IpAddr,
) -> Result<(Uuid, DateTime<Utc>, DateTime<Utc>), AppError> {
let period_start = Utc::now();
let period_end = add_months_utc8(period_start, months)?;
let mut tx = state
.db
let mut tx = pool
.begin()
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "开启事务失败").with_source(err))?;
let _ = sqlx::query(
let _: Uuid = sqlx::query_scalar("SELECT id FROM users WHERE id = $1 FOR UPDATE")
.bind(user_id)
.fetch_one(&mut *tx)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "锁定订阅用户失败").with_source(err))?;
let has_open_stripe: bool = sqlx::query_scalar(
r#"
SELECT EXISTS(
SELECT 1
FROM subscriptions
WHERE user_id = $1
AND provider = 'stripe'
AND status <> 'canceled'
)
"#,
)
.bind(user_id)
.fetch_one(&mut *tx)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "检查 Stripe 订阅失败").with_source(err))?;
if has_open_stripe {
return Err(AppError::new(
ErrorCode::Forbidden,
"用户存在未取消的 Stripe 订阅,不能直接替换为手工套餐",
));
}
sqlx::query(
r#"
UPDATE subscriptions
SET status = 'canceled',
cancel_at_period_end = false,
canceled_at = NOW(),
updated_at = NOW()
WHERE user_id = $1 AND status IN ('active', 'trialing', 'past_due')
WHERE user_id = $1
AND provider <> 'stripe'
AND status IN ('active', 'trialing', 'past_due')
"#,
)
.bind(user_id)
.execute(&mut *tx)
.await;
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "关闭原本地订阅失败").with_source(err))?;
let subscription_id: Uuid = sqlx::query_scalar(
r#"
@@ -975,7 +1063,7 @@ async fn create_manual_subscription(
"#,
)
.bind(user_id)
.bind(plan.id)
.bind(plan_id)
.bind(period_start)
.bind(period_end)
.fetch_one(&mut *tx)
@@ -1007,9 +1095,9 @@ async fn create_manual_subscription(
.bind(subscription_id)
.bind(serde_json::json!({
"target_user_id": user_id,
"plan_id": plan.id,
"plan_id": plan_id,
"months": months,
"note": req.note,
"note": note,
}))
.bind(ip.to_string())
.execute(&mut *tx)
@@ -1020,19 +1108,7 @@ async fn create_manual_subscription(
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "提交事务失败").with_source(err))?;
Ok(Json(Envelope {
success: true,
data: ManualSubscriptionResponse {
message: "套餐已开通".to_string(),
subscription_id,
user_id,
plan_id: plan.id,
plan_name: plan.name,
period_start,
period_end,
status: "active".to_string(),
},
}))
Ok((subscription_id, period_start, period_end))
}
fn add_months_utc8(start: DateTime<Utc>, months: i32) -> Result<DateTime<Utc>, AppError> {
@@ -1716,6 +1792,7 @@ async fn audit_config_action(
#[cfg(test)]
mod tests {
use super::*;
use sqlx::postgres::PgPoolOptions;
#[test]
fn secret_masking_never_splits_utf8() {
@@ -1723,4 +1800,151 @@ mod tests {
assert_eq!(mask_secret("中文密钥测试内容"), "中文密钥测试内容");
assert_eq!(mask_secret("🔑🔑🔑🔑🔑🔑🔑🔑more"), "🔑🔑🔑🔑🔑🔑🔑🔑...");
}
#[tokio::test]
#[ignore = "requires IMAGEFORGE_TEST_DATABASE_URL"]
async fn manual_subscriptions_are_serialized_and_cannot_replace_stripe() {
let database_url = std::env::var("IMAGEFORGE_TEST_DATABASE_URL")
.expect("IMAGEFORGE_TEST_DATABASE_URL is required");
let pool = PgPoolOptions::new()
.max_connections(32)
.connect(&database_url)
.await
.expect("connect test database");
sqlx::migrate!()
.run(&pool)
.await
.expect("apply test migrations");
let marker = Uuid::new_v4().simple().to_string();
let admin_id: Uuid = sqlx::query_scalar(
r#"
INSERT INTO users (email, username, password_hash, role, email_verified_at)
VALUES ($1, $2, 'test', 'admin', NOW())
RETURNING id
"#,
)
.bind(format!("admin-{marker}@example.test"))
.bind(format!("admin-{marker}"))
.fetch_one(&pool)
.await
.expect("insert test admin");
let user_id: Uuid = sqlx::query_scalar(
r#"
INSERT INTO users (email, username, password_hash, email_verified_at)
VALUES ($1, $2, 'test', NOW())
RETURNING id
"#,
)
.bind(format!("user-{marker}@example.test"))
.bind(format!("user-{marker}"))
.fetch_one(&pool)
.await
.expect("insert test user");
let plan_id: Uuid = sqlx::query_scalar("SELECT id FROM plans WHERE code = 'pro_monthly'")
.fetch_one(&pool)
.await
.expect("load test plan");
let ip: IpAddr = "127.0.0.1".parse().unwrap();
let mut joins = Vec::new();
for _ in 0..20 {
let pool = pool.clone();
joins.push(tokio::spawn(async move {
persist_manual_subscription(
&pool,
admin_id,
user_id,
plan_id,
1,
Some("concurrency-test"),
ip,
)
.await
}));
}
for join in joins {
join.await
.expect("manual subscription task panicked")
.expect("manual subscription failed");
}
let effective_manual: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)
FROM subscriptions
WHERE user_id = $1
AND provider = 'manual'
AND status IN ('active', 'trialing', 'past_due')
"#,
)
.bind(user_id)
.fetch_one(&pool)
.await
.expect("count effective manual subscriptions");
assert_eq!(effective_manual, 1);
sqlx::query(
"UPDATE subscriptions SET status = 'canceled', canceled_at = NOW() WHERE user_id = $1",
)
.bind(user_id)
.execute(&pool)
.await
.expect("cancel test manual subscription");
sqlx::query(
r#"
INSERT INTO subscriptions (
user_id, plan_id, status, current_period_start, current_period_end,
provider, provider_customer_id, provider_subscription_id
) VALUES (
$1, $2, 'active', NOW(), NOW() + INTERVAL '1 month',
'stripe', $3, $4
)
"#,
)
.bind(user_id)
.bind(plan_id)
.bind(format!("cus_{marker}"))
.bind(format!("sub_{marker}"))
.execute(&pool)
.await
.expect("insert Stripe subscription");
let error = persist_manual_subscription(
&pool,
admin_id,
user_id,
plan_id,
1,
Some("must-not-replace-stripe"),
ip,
)
.await
.expect_err("manual subscription replaced Stripe");
assert_eq!(error.code, ErrorCode::Forbidden);
let effective_subscriptions: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)
FROM subscriptions
WHERE user_id = $1
AND status IN ('active', 'trialing', 'past_due')
"#,
)
.bind(user_id)
.fetch_one(&pool)
.await
.expect("count effective subscriptions");
assert_eq!(effective_subscriptions, 1);
sqlx::query("DELETE FROM audit_logs WHERE details->>'target_user_id' = $1")
.bind(user_id.to_string())
.execute(&pool)
.await
.expect("clean test audit logs");
sqlx::query("DELETE FROM users WHERE id = ANY($1)")
.bind(vec![user_id, admin_id])
.execute(&pool)
.await
.expect("clean test users");
}
}

View File

@@ -415,12 +415,22 @@ struct VerifyEmailRequest {
token: String,
}
#[derive(Debug, Serialize)]
struct VerifyEmailResponse {
message: String,
session_invalidated: bool,
}
struct EmailChangeConfirmation {
old_email: String,
}
async fn verify_email(
State(state): State<AppState>,
ConnectInfo(addr): ConnectInfo<SocketAddr>,
headers: HeaderMap,
Json(req): Json<VerifyEmailRequest>,
) -> Result<Json<Envelope<MessageResponse>>, AppError> {
) -> Result<Json<Envelope<VerifyEmailResponse>>, AppError> {
if req.token.trim().is_empty() {
return Err(AppError::new(ErrorCode::InvalidRequest, "token 不能为空"));
}
@@ -440,6 +450,19 @@ async fn verify_email(
let token_hash = credentials::sha256_hex(&req.token);
let now = Utc::now();
if let Some(confirmation) = confirm_email_change(&state, &token_hash, now).await? {
if let Err(err) = mail::send_email_change_notice(&state, &confirmation.old_email).await {
tracing::warn!(error = ?err, "email change notice delivery failed");
}
return Ok(Json(Envelope {
success: true,
data: VerifyEmailResponse {
message: "新邮箱确认成功,请重新登录".to_string(),
session_invalidated: true,
},
}));
}
let updated = sqlx::query(
r#"
WITH v AS (
@@ -477,12 +500,149 @@ async fn verify_email(
Ok(Json(Envelope {
success: true,
data: MessageResponse {
data: VerifyEmailResponse {
message: "邮箱验证成功".to_string(),
session_invalidated: false,
},
}))
}
async fn confirm_email_change(
state: &AppState,
token_hash: &str,
now: DateTime<Utc>,
) -> Result<Option<EmailChangeConfirmation>, AppError> {
let user_id: Option<Uuid> = sqlx::query_scalar(
r#"
SELECT user_id
FROM email_change_requests
WHERE token_hash = $1
AND confirmed_at IS NULL
AND canceled_at IS NULL
AND expires_at > $2
"#,
)
.bind(token_hash)
.bind(now)
.fetch_optional(&state.db)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "查询邮箱变更请求失败").with_source(err))?;
let Some(user_id) = user_id else {
return Ok(None);
};
let mut tx = state
.db
.begin()
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "开启事务失败").with_source(err))?;
let user: Option<(String,)> =
sqlx::query_as("SELECT email FROM users WHERE id = $1 FOR UPDATE")
.bind(user_id)
.fetch_optional(&mut *tx)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "锁定用户失败").with_source(err))?;
let Some((old_email,)) = user else {
tx.rollback().await.ok();
return Ok(None);
};
let request: Option<(Uuid, String)> = sqlx::query_as(
r#"
SELECT id, new_email
FROM email_change_requests
WHERE token_hash = $1
AND user_id = $2
AND confirmed_at IS NULL
AND canceled_at IS NULL
AND expires_at > $3
FOR UPDATE
"#,
)
.bind(token_hash)
.bind(user_id)
.bind(now)
.fetch_optional(&mut *tx)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "锁定邮箱变更请求失败").with_source(err))?;
let Some((request_id, new_email)) = request else {
tx.rollback().await.ok();
return Ok(None);
};
let changed = sqlx::query(
r#"
UPDATE users u
SET email = $2,
email_verified_at = $3,
token_version = token_version + 1,
updated_at = NOW()
WHERE u.id = $1
AND NOT EXISTS (
SELECT 1 FROM users other
WHERE other.email = $2 AND other.id <> u.id
)
"#,
)
.bind(user_id)
.bind(&new_email)
.bind(now)
.execute(&mut *tx)
.await
.map_err(|err| {
if matches!(&err, sqlx::Error::Database(db) if db.code().as_deref() == Some("23505")) {
AppError::new(ErrorCode::InvalidRequest, "新邮箱已被其他账号使用")
} else {
AppError::new(ErrorCode::Internal, "确认新邮箱失败").with_source(err)
}
})?;
if changed.rows_affected() == 0 {
return Err(AppError::new(
ErrorCode::InvalidRequest,
"新邮箱已被其他账号使用",
));
}
sqlx::query(
"UPDATE email_change_requests SET confirmed_at = $2 WHERE id = $1 AND confirmed_at IS NULL AND canceled_at IS NULL",
)
.bind(request_id)
.bind(now)
.execute(&mut *tx)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "完成邮箱变更请求失败").with_source(err))?;
sqlx::query(
"UPDATE email_change_requests SET canceled_at = $2 WHERE user_id = $1 AND id <> $3 AND confirmed_at IS NULL AND canceled_at IS NULL",
)
.bind(user_id)
.bind(now)
.bind(request_id)
.execute(&mut *tx)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "撤销旧邮箱变更请求失败").with_source(err))?;
sqlx::query("UPDATE password_resets SET used_at = $2 WHERE user_id = $1 AND used_at IS NULL")
.bind(user_id)
.bind(now)
.execute(&mut *tx)
.await
.map_err(|err| {
AppError::new(ErrorCode::Internal, "撤销密码重置请求失败").with_source(err)
})?;
sqlx::query(
"UPDATE email_verifications SET verified_at = $2 WHERE user_id = $1 AND verified_at IS NULL",
)
.bind(user_id)
.bind(now)
.execute(&mut *tx)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "撤销旧邮箱验证请求失败").with_source(err))?;
tx.commit()
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "提交邮箱变更失败").with_source(err))?;
Ok(Some(EmailChangeConfirmation { old_email }))
}
#[derive(Debug, Deserialize)]
struct ForgotPasswordRequest {
email: String,
@@ -517,6 +677,12 @@ async fn forgot_password(
)
.await?;
let requested_email = req.email.to_lowercase();
let mut tx = state
.db
.begin()
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "开启事务失败").with_source(err))?;
let user = sqlx::query_as::<_, UserRow>(
r#"
SELECT
@@ -530,19 +696,20 @@ async fn forgot_password(
token_version
FROM users
WHERE email = $1
FOR UPDATE
"#,
)
.bind(req.email.to_lowercase())
.fetch_optional(&state.db)
.bind(&requested_email)
.fetch_optional(&mut *tx)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "查询用户失败").with_source(err))?;
if let Some(user) = user {
let delivery = if let Some(user) = user {
let reset_token = credentials::generate_token();
let token_hash = credentials::sha256_hex(&reset_token);
let expires_at_db = Utc::now() + Duration::hours(1);
let _ = sqlx::query(
sqlx::query(
r#"
INSERT INTO password_resets (user_id, token_hash, expires_at)
VALUES ($1, $2, $3)
@@ -551,16 +718,26 @@ async fn forgot_password(
.bind(user.id)
.bind(token_hash)
.bind(expires_at_db)
.execute(&state.db)
.await;
.execute(&mut *tx)
.await
.map_err(|err| {
AppError::new(ErrorCode::Internal, "创建密码重置记录失败").with_source(err)
})?;
let reset_url = format!(
"{}/reset-password?token={}",
state.config.public_base_url, reset_token
);
Some((user.email, user.username, reset_url))
} else {
None
};
tx.commit().await.map_err(|err| {
AppError::new(ErrorCode::Internal, "提交密码找回事务失败").with_source(err)
})?;
let _ =
mail::send_password_reset_email(&state, &user.email, &user.username, &reset_url).await;
if let Some((email, username, reset_url)) = delivery {
let _ = mail::send_password_reset_email(&state, &email, &username, &reset_url).await;
}
Ok(Json(Envelope {
@@ -611,25 +788,23 @@ async fn reset_password(
let token_hash = credentials::sha256_hex(&req.token);
let now = Utc::now();
let token_exists: bool = sqlx::query_scalar(
let reset_user_id: Option<Uuid> = sqlx::query_scalar(
r#"
SELECT EXISTS(
SELECT 1
FROM password_resets
WHERE token_hash = $1
AND used_at IS NULL
AND expires_at > $2
)
SELECT user_id
FROM password_resets
WHERE token_hash = $1
AND used_at IS NULL
AND expires_at > $2
"#,
)
.bind(&token_hash)
.bind(now)
.fetch_one(&state.db)
.fetch_optional(&state.db)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "重置密码失败").with_source(err))?;
if !token_exists {
let Some(reset_user_id) = reset_user_id else {
return Err(AppError::new(ErrorCode::InvalidToken, "Token 无效或已过期"));
}
};
let password_hash = credentials::hash_password(&req.new_password).await?;
@@ -639,6 +814,16 @@ async fn reset_password(
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "开启事务失败").with_source(err))?;
let locked_user: Option<Uuid> =
sqlx::query_scalar("SELECT id FROM users WHERE id = $1 FOR UPDATE")
.bind(reset_user_id)
.fetch_optional(&mut *tx)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "锁定用户失败").with_source(err))?;
if locked_user.is_none() {
return Err(AppError::new(ErrorCode::InvalidToken, "Token 无效或已过期"));
}
let user_id: Option<Uuid> = sqlx::query_scalar(
r#"
SELECT user_id
@@ -658,6 +843,9 @@ async fn reset_password(
let Some(user_id) = user_id else {
return Err(AppError::new(ErrorCode::InvalidToken, "Token 无效或已过期"));
};
if user_id != reset_user_id {
return Err(AppError::new(ErrorCode::InvalidToken, "Token 无效或已过期"));
}
sqlx::query(
"UPDATE users SET password_hash = $1, token_version = token_version + 1, updated_at = NOW() WHERE id = $2",
@@ -668,14 +856,7 @@ async fn reset_password(
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "更新密码失败").with_source(err))?;
sqlx::query(
"UPDATE password_resets SET used_at = $2 WHERE token_hash = $1 AND used_at IS NULL",
)
.bind(token_hash)
.bind(now)
.execute(&mut *tx)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "更新重置记录失败").with_source(err))?;
credentials::invalidate_account_recovery(&mut tx, user_id, now).await?;
tx.commit()
.await

File diff suppressed because it is too large Load Diff

View File

@@ -8,6 +8,7 @@ use crate::services::compress;
use crate::services::compress::{CompressionLevel, ImageFmt};
use crate::services::filename;
use crate::services::idempotency;
use crate::services::object_lifecycle;
use crate::services::quota;
use crate::services::storage;
use crate::state::AppState;
@@ -21,6 +22,7 @@ use chrono::{DateTime, Duration, Utc};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use sqlx::FromRow;
use std::future::Future;
use std::net::{IpAddr, SocketAddr};
use uuid::Uuid;
@@ -30,6 +32,14 @@ pub fn router() -> Router<AppState> {
.route("/compress/direct", post(compress_direct))
}
fn spawn_detached_operation<F, T>(future: F) -> tokio::task::JoinHandle<T>
where
F: Future<Output = T> + Send + 'static,
T: Send + 'static,
{
tokio::spawn(future)
}
#[derive(Debug, Serialize, Deserialize)]
struct BillingView {
units_charged: i32,
@@ -79,6 +89,10 @@ fn default_units_charged() -> i32 {
1
}
fn metered_units(charged: bool) -> i32 {
i32::from(charged)
}
fn direct_response<B: IntoResponse>(
body: B,
format: ImageFmt,
@@ -210,6 +224,7 @@ async fn compress_json(
let quota_ctx = admission.quota_ctx;
let mut idem_acquired = false;
let mut idem_owner = None;
if let (Some(scope), Some(idem_key), Some(request_hash)) = (
idempotency_scope,
idempotency_key.as_deref(),
@@ -258,20 +273,42 @@ async fn compress_json(
"请求正在处理中,请稍后重试",
));
}
idempotency::BeginResult::Acquired => {
idempotency::BeginResult::Acquired { owner } => {
idem_acquired = true;
idem_owner = Some(owner);
}
}
}
let mut anonymous_reserved = false;
let op: Result<CompressResponse, AppError> = (async {
let task_id = Uuid::new_v4();
let operation_state = state.clone();
let operation_principal = principal.clone();
let operation_quota_ctx = quota_ctx.clone();
let operation_idempotency_key = idempotency_key.clone();
let operation_request_hash = request_hash.clone();
let operation_idem_owner = idem_owner;
let operation = spawn_detached_operation(async move {
let state = operation_state;
let principal = operation_principal;
let quota_ctx = operation_quota_ctx;
let idempotency_key = operation_idempotency_key;
let request_hash = operation_request_hash;
let _idempotency_heartbeat = start_idempotency_heartbeat(
&state,
idempotency_scope,
idempotency_key.as_deref(),
request_hash.as_deref(),
operation_idem_owner,
);
let mut anonymous_reservation_date = None;
let op: Result<CompressResponse, AppError> = (async {
match &quota_ctx {
QuotaContext::User(billing) => ensure_quota_available(&state, billing, 1).await?,
QuotaContext::ApiKey(billing, _) => ensure_quota_available(&state, billing, 1).await?,
QuotaContext::Anonymous { session_id, ip } => {
quota::consume_anonymous_units(&state, session_id, *ip, 1).await?;
anonymous_reserved = true;
anonymous_reservation_date = Some(
quota::reserve_anonymous_single_unit(&state, task_id, session_id, *ip).await?,
);
}
}
@@ -299,61 +336,32 @@ async fn compress_json(
} else {
(saved_bytes as f64) * 100.0 / (original_size as f64)
};
let skip_charge = req.compression_rate == Some(100)
&& req.target_size_bytes.is_none()
&& format_in == format_out
&& req.max_width.is_none()
&& req.max_height.is_none();
let charge_units = anonymous_reserved || (!skip_charge && compressed_size < original_size);
let charge_units = quota::output_consumes_unit(
req.compression_rate,
format_in == format_out,
req.max_width.is_some() || req.max_height.is_some(),
req.target_size_bytes.is_some(),
original_size,
compressed_size,
);
let task_id = Uuid::new_v4();
let file_id = Uuid::new_v4();
let retention_hours = retention.num_hours();
let object_key =
storage::result_key(retention_hours, task_id, file_id, format_out.extension());
let stored =
storage::store_bytes(&state, &object_key, compressed, format_out.content_type())
.await?;
let tracked = object_lifecycle::store_tracked_bytes(
&state,
task_id,
Some(file_id),
"result",
&object_key,
compressed,
format_out.content_type(),
)
.await?;
let expires_at = Utc::now() + retention;
if let Err(err) = record_task_and_metering(
&state,
&principal,
ip,
task_id,
file_id,
&stored,
&req.file_name,
req.max_width,
req.max_height,
effective_level,
req.compression_rate,
format_in,
format_out,
original_size,
compressed_size,
saved_percent,
expires_at,
retention_hours,
&quota_ctx,
charge_units,
)
.await
{
let _ = storage::delete_object(
&state,
&storage::ObjectLocator {
backend: stored.backend.clone(),
endpoint_id: stored.endpoint_id,
key: stored.key.clone(),
},
)
.await;
return Err(err);
}
Ok(CompressResponse {
let response = CompressResponse {
task_id,
file_id,
format_in: format_in.as_str().to_string(),
@@ -365,43 +373,101 @@ async fn compress_json(
download_url: format!("/downloads/{file_id}"),
expires_at,
billing: BillingView {
units_charged: if charge_units { 1 } else { 0 },
units_charged: metered_units(charge_units),
},
};
let idem_completion = build_idempotency_completion(
idem_acquired,
operation_idem_owner,
idempotency_scope,
idempotency_key.as_deref(),
request_hash.as_deref(),
&response,
)?;
if let Err(err) = record_task_and_metering(
&state,
&principal,
ip,
task_id,
file_id,
&tracked,
&req.file_name,
req.max_width,
req.max_height,
effective_level,
req.compression_rate,
req.target_size_bytes,
format_in,
format_out,
original_size,
compressed_size,
saved_percent,
expires_at,
retention_hours,
&quota_ctx,
charge_units,
idem_completion.as_ref(),
)
.await
{
match sync_result_was_committed(&state, task_id, file_id, &tracked).await {
Ok(true) => {
tracing::warn!(task_id = %task_id, file_id = %file_id, error = %err, "sync result commit response was lost; recovered committed publication");
}
Ok(false) => {
if let Err(cleanup_err) = object_lifecycle::schedule_tracked_delete(
&state,
&tracked,
Some(&err),
)
.await
{
tracing::error!(task_id = %task_id, storage_object_id = %tracked.lifecycle_id, error = %cleanup_err, "failed to persist rejected sync result cleanup");
}
return Err(err);
}
Err(probe_err) => {
tracing::error!(task_id = %task_id, storage_object_id = %tracked.lifecycle_id, error = %probe_err, original_error = %err, "sync result commit state is unknown; staging lease will reconcile object");
return Err(err);
}
}
}
if anonymous_reservation_date.is_some() {
if let Err(err) =
quota::finalize_anonymous_single_reservation(&state, task_id, charge_units).await
{
// The durable reservation remains visible to maintenance, so a
// transient Redis failure must not turn a successful image into
// a failed, non-idempotent request.
tracing::warn!(task_id = %task_id, charged = charge_units, error = %err, "anonymous single reservation finalization deferred");
}
}
Ok(response)
})
})
.await;
.await;
(op, anonymous_reservation_date)
});
let (op, anonymous_reservation_date) = operation.await.map_err(|err| {
AppError::new(ErrorCode::Internal, "同步压缩后台任务异常退出").with_source(err)
})?;
match op {
Ok(resp) => {
if let (Some(scope), Some(idem_key), Some(request_hash)) = (
idempotency_scope,
idempotency_key.as_deref(),
request_hash.as_deref(),
) {
if idem_acquired {
let _ = idempotency::complete(
&state,
scope,
idem_key,
request_hash,
200,
serde_json::to_value(&resp).unwrap_or(serde_json::Value::Null),
)
.await;
}
}
Ok((
jar,
Json(Envelope {
success: true,
data: resp,
}),
))
}
Ok(resp) => Ok((
jar,
Json(Envelope {
success: true,
data: resp,
}),
)),
Err(err) => {
if anonymous_reserved {
if let QuotaContext::Anonymous { session_id, ip } = &quota_ctx {
let _ = quota::refund_anonymous_units(&state, session_id, *ip, 1).await;
if anonymous_reservation_date.is_some() {
if let Err(refund_err) =
quota::refund_anonymous_single_reservation(&state, task_id).await
{
tracing::warn!(task_id = %task_id, error = %refund_err, "anonymous single reservation refund deferred");
}
}
if let (Some(scope), Some(idem_key), Some(request_hash)) = (
@@ -410,7 +476,10 @@ async fn compress_json(
request_hash.as_deref(),
) {
if idem_acquired {
let _ = idempotency::abort(&state, scope, idem_key, request_hash).await;
if let Some(owner) = idem_owner {
let _ =
idempotency::abort(&state, scope, idem_key, request_hash, owner).await;
}
}
}
Err(err)
@@ -505,6 +574,7 @@ async fn compress_direct(
let quota_ctx = admission.quota_ctx;
let mut idem_acquired = false;
let mut idem_owner = None;
if let (Some(scope), Some(idem_key), Some(request_hash)) = (
idempotency_scope,
idempotency_key.as_deref(),
@@ -546,13 +616,33 @@ async fn compress_direct(
"请求正在处理中,请稍后重试",
));
}
idempotency::BeginResult::Acquired => {
idempotency::BeginResult::Acquired { owner } => {
idem_acquired = true;
idem_owner = Some(owner);
}
}
}
let op: Result<(axum::response::Response, DirectIdempotencyData), AppError> = (async {
let operation_state = state.clone();
let operation_principal = principal.clone();
let operation_quota_ctx = quota_ctx.clone();
let operation_idempotency_key = idempotency_key.clone();
let operation_request_hash = request_hash.clone();
let operation_idem_owner = idem_owner;
let operation = spawn_detached_operation(async move {
let state = operation_state;
let principal = operation_principal;
let quota_ctx = operation_quota_ctx;
let idempotency_key = operation_idempotency_key;
let request_hash = operation_request_hash;
let _idempotency_heartbeat = start_idempotency_heartbeat(
&state,
idempotency_scope,
idempotency_key.as_deref(),
request_hash.as_deref(),
operation_idem_owner,
);
let op: Result<(axum::response::Response, DirectIdempotencyData), AppError> = (async {
match &quota_ctx {
QuotaContext::User(billing) => ensure_quota_available(&state, billing, 1).await?,
QuotaContext::ApiKey(billing, _) => ensure_quota_available(&state, billing, 1).await?,
@@ -583,20 +673,25 @@ async fn compress_direct(
} else {
(saved_bytes as f64) * 100.0 / (original_size as f64)
};
let skip_charge = req.compression_rate == Some(100)
&& req.target_size_bytes.is_none()
&& format_in == format_out
&& req.max_width.is_none()
&& req.max_height.is_none();
let charge_units = !skip_charge && compressed_size < original_size;
let charge_units = quota::output_consumes_unit(
req.compression_rate,
format_in == format_out,
req.max_width.is_some() || req.max_height.is_some(),
req.target_size_bytes.is_some(),
original_size,
compressed_size,
);
let task_id = Uuid::new_v4();
let file_id = Uuid::new_v4();
let retention_hours = retention.num_hours();
let object_key =
storage::result_key(retention_hours, task_id, file_id, format_out.extension());
let stored = storage::store_bytes(
let tracked = object_lifecycle::store_tracked_bytes(
&state,
task_id,
Some(file_id),
"result",
&object_key,
compressed.clone(),
format_out.content_type(),
@@ -604,6 +699,23 @@ async fn compress_direct(
.await?;
let expires_at = Utc::now() + retention;
let idem_data = DirectIdempotencyData {
file_id,
format_out: format_out.as_str().to_string(),
original_size,
compressed_size,
saved_bytes,
saved_percent,
units_charged: metered_units(charge_units),
};
let idem_completion = build_idempotency_completion(
idem_acquired,
operation_idem_owner,
idempotency_scope,
idempotency_key.as_deref(),
request_hash.as_deref(),
&idem_data,
)?;
if let Err(err) = record_task_and_metering(
&state,
@@ -611,12 +723,13 @@ async fn compress_direct(
ip,
task_id,
file_id,
&stored,
&tracked,
&req.file_name,
req.max_width,
req.max_height,
effective_level,
req.compression_rate,
req.target_size_bytes,
format_in,
format_out,
original_size,
@@ -626,56 +739,45 @@ async fn compress_direct(
retention_hours,
&quota_ctx,
charge_units,
idem_completion.as_ref(),
)
.await
{
let _ = storage::delete_object(
&state,
&storage::ObjectLocator {
backend: stored.backend.clone(),
endpoint_id: stored.endpoint_id,
key: stored.key.clone(),
},
)
.await;
return Err(err);
}
let idem_data = DirectIdempotencyData {
file_id,
format_out: format_out.as_str().to_string(),
original_size,
compressed_size,
saved_bytes,
saved_percent,
units_charged: if charge_units { 1 } else { 0 },
};
let response = direct_response(compressed, format_out, &idem_data);
Ok((response, idem_data))
})
.await;
match op {
Ok((response, idem_data)) => {
if let (Some(scope), Some(idem_key), Some(request_hash)) = (
idempotency_scope,
idempotency_key.as_deref(),
request_hash.as_deref(),
) {
if idem_acquired {
let _ = idempotency::complete(
match sync_result_was_committed(&state, task_id, file_id, &tracked).await {
Ok(true) => {
tracing::warn!(task_id = %task_id, file_id = %file_id, error = %err, "direct result commit response was lost; recovered committed publication");
}
Ok(false) => {
if let Err(cleanup_err) = object_lifecycle::schedule_tracked_delete(
&state,
scope,
idem_key,
request_hash,
200,
serde_json::to_value(&idem_data).unwrap_or(serde_json::Value::Null),
&tracked,
Some(&err),
)
.await;
.await
{
tracing::error!(task_id = %task_id, storage_object_id = %tracked.lifecycle_id, error = %cleanup_err, "failed to persist rejected direct result cleanup");
}
return Err(err);
}
Err(probe_err) => {
tracing::error!(task_id = %task_id, storage_object_id = %tracked.lifecycle_id, error = %probe_err, original_error = %err, "direct result commit state is unknown; staging lease will reconcile object");
return Err(err);
}
}
Ok((jar, response))
}
let response = direct_response(compressed, format_out, &idem_data);
Ok((response, idem_data))
})
.await;
op
});
let op = operation.await.map_err(|err| {
AppError::new(ErrorCode::Internal, "直接压缩后台任务异常退出").with_source(err)
})?;
match op {
Ok((response, _idem_data)) => Ok((jar, response)),
Err(err) => {
if let (Some(scope), Some(idem_key), Some(request_hash)) = (
idempotency_scope,
@@ -683,7 +785,10 @@ async fn compress_direct(
request_hash.as_deref(),
) {
if idem_acquired {
let _ = idempotency::abort(&state, scope, idem_key, request_hash).await;
if let Some(owner) = idem_owner {
let _ =
idempotency::abort(&state, scope, idem_key, request_hash, owner).await;
}
}
}
Err(err)
@@ -691,6 +796,45 @@ async fn compress_direct(
}
}
async fn sync_result_was_committed(
state: &AppState,
task_id: Uuid,
file_id: Uuid,
tracked: &object_lifecycle::TrackedStoredObject,
) -> Result<bool, AppError> {
sqlx::query_scalar(
r#"
SELECT EXISTS(
SELECT 1
FROM tasks AS task
JOIN task_files AS file ON file.task_id = task.id
JOIN storage_objects AS object ON object.id = $3
WHERE task.id = $1
AND task.status = 'completed'
AND file.id = $2
AND file.status = 'completed'
AND file.storage_backend = $4
AND file.storage_endpoint_id IS NOT DISTINCT FROM $5
AND COALESCE(file.storage_key, file.storage_path) = $6
AND object.state = 'published'
AND object.task_id = task.id
AND object.task_file_id = file.id
)
"#,
)
.bind(task_id)
.bind(file_id)
.bind(tracked.lifecycle_id)
.bind(&tracked.stored.backend)
.bind(tracked.stored.endpoint_id)
.bind(&tracked.stored.key)
.fetch_one(&state.db)
.await
.map_err(|err| {
AppError::new(ErrorCode::StorageUnavailable, "核验同步压缩提交结果失败").with_source(err)
})
}
#[derive(Debug, FromRow)]
struct DirectReplayRow {
storage_backend: String,
@@ -720,6 +864,7 @@ async fn load_direct_replay_bytes(
FROM task_files f
JOIN tasks t ON t.id = f.task_id
WHERE f.id = $1 AND t.user_id = $2
AND t.deletion_started_at IS NULL
"#,
)
.bind(file_id)
@@ -740,6 +885,7 @@ async fn load_direct_replay_bytes(
FROM task_files f
JOIN tasks t ON t.id = f.task_id
WHERE f.id = $1 AND t.api_key_id = $2
AND t.deletion_started_at IS NULL
"#,
)
.bind(file_id)
@@ -878,7 +1024,6 @@ async fn parse_single_file_request(
"target_size_bytes 格式错误,需为正整数(字节)",
)
})?);
// 最小目标大小限制1KB
if let Some(size) = target_size_bytes {
if size < 1024 {
return Err(AppError::new(
@@ -886,6 +1031,12 @@ async fn parse_single_file_request(
"target_size_bytes 最小为 10241KB",
));
}
if i64::try_from(size).is_err() {
return Err(AppError::new(
ErrorCode::InvalidRequest,
"target_size_bytes 超出支持范围",
));
}
}
}
}
@@ -924,6 +1075,63 @@ enum QuotaContext {
ApiKey(BillingContext, Uuid),
}
struct IdempotencyCompletion {
scope: idempotency::Scope,
owner: Uuid,
key: String,
request_hash: String,
response_body: serde_json::Value,
}
fn build_idempotency_completion<T: Serialize>(
acquired: bool,
owner: Option<Uuid>,
scope: Option<idempotency::Scope>,
key: Option<&str>,
request_hash: Option<&str>,
response: &T,
) -> Result<Option<IdempotencyCompletion>, AppError> {
if !acquired {
return Ok(None);
}
let (owner, scope, key, request_hash) = match (owner, scope, key, request_hash) {
(Some(owner), Some(scope), Some(key), Some(request_hash)) => {
(owner, scope, key, request_hash)
}
_ => return Err(AppError::new(ErrorCode::Internal, "幂等请求上下文不完整")),
};
Ok(Some(IdempotencyCompletion {
scope,
owner,
key: key.to_string(),
request_hash: request_hash.to_string(),
response_body: serde_json::to_value(response).map_err(|err| {
AppError::new(ErrorCode::Internal, "序列化幂等响应失败").with_source(err)
})?,
}))
}
fn start_idempotency_heartbeat(
state: &AppState,
scope: Option<idempotency::Scope>,
key: Option<&str>,
request_hash: Option<&str>,
owner: Option<Uuid>,
) -> Option<idempotency::LeaseHeartbeat> {
match (scope, key, request_hash, owner) {
(Some(scope), Some(key), Some(request_hash), Some(owner)) => {
Some(idempotency::start_lease_heartbeat(
state.clone(),
scope,
key.to_string(),
request_hash.to_string(),
owner,
))
}
_ => None,
}
}
struct SingleAdmission {
retention: Duration,
quota_ctx: QuotaContext,
@@ -1022,12 +1230,13 @@ async fn record_task_and_metering(
client_ip: IpAddr,
task_id: Uuid,
file_id: Uuid,
stored: &storage::StoredObject,
tracked: &object_lifecycle::TrackedStoredObject,
original_name: &str,
max_width: Option<u32>,
max_height: Option<u32>,
level: CompressionLevel,
compression_rate: Option<u8>,
target_size_bytes: Option<u64>,
format_in: ImageFmt,
format_out: ImageFmt,
original_size: u64,
@@ -1037,7 +1246,9 @@ async fn record_task_and_metering(
retention_hours: i64,
quota_ctx: &QuotaContext,
charge_units: bool,
idempotency_completion: Option<&IdempotencyCompletion>,
) -> Result<(), AppError> {
let stored = &tracked.stored;
let (user_id, session_id, api_key_id, source) = match principal {
context::Principal::Anonymous { session_id } => {
(None, Some(session_id.clone()), None, "web")
@@ -1061,16 +1272,16 @@ async fn record_task_and_metering(
INSERT INTO tasks (
id, user_id, session_id, api_key_id, client_ip, source, status,
compression_level, output_format, max_width, max_height, preserve_metadata,
compression_rate,
compression_rate, target_size_bytes,
total_files, completed_files, failed_files,
total_original_size, total_compressed_size,
started_at, completed_at, expires_at, retention_hours
) VALUES (
$1, $2, $3, $4, $5::inet, $6::task_source, 'completed',
$7::compression_level, $8, $9, $10, $11, $12,
1, 1, 0,
$13, $14,
NOW(), NOW(), $15, $16
$13, 1, 1, 0,
$14, $15,
NOW(), NOW(), $16, $17
)
"#,
)
@@ -1086,6 +1297,7 @@ async fn record_task_and_metering(
.bind(max_height.map(|v| v as i32))
.bind(false)
.bind(compression_rate.map(|v| v as i16))
.bind(target_size_bytes.map(|v| v as i64))
.bind(original_size as i64)
.bind(compressed_size as i64)
.bind(expires_at)
@@ -1132,7 +1344,9 @@ async fn record_task_and_metering(
.map_err(|err| AppError::new(ErrorCode::Internal, "创建文件记录失败").with_source(err))?;
match quota_ctx {
QuotaContext::Anonymous { .. } => {}
QuotaContext::Anonymous { .. } => {
quota::mark_anonymous_single_result(&mut tx, task_id, charge_units).await?;
}
QuotaContext::User(billing) => {
if charge_units {
charge_one_unit(
@@ -1167,6 +1381,20 @@ async fn record_task_and_metering(
}
}
object_lifecycle::publish_in_tx(&mut tx, tracked).await?;
if let Some(idempotency_completion) = idempotency_completion {
idempotency::complete_in_tx(
&mut tx,
idempotency_completion.scope,
&idempotency_completion.key,
&idempotency_completion.request_hash,
idempotency_completion.owner,
200,
idempotency_completion.response_body.clone(),
)
.await?;
}
tx.commit()
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "提交事务失败").with_source(err))?;
@@ -1220,6 +1448,8 @@ async fn charge_one_unit(
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
#[test]
fn direct_response_has_consistent_compression_headers() {
@@ -1242,4 +1472,65 @@ mod tests {
assert_eq!(response.headers()["imageforge-saved-percent"], "37.50");
assert_eq!(response.headers()["imageforge-units-charged"], "1");
}
#[test]
fn anonymous_response_units_follow_actual_output_metering() {
let cases = [
(Some(100), true, false, false, 100, 50, 0),
(None, true, false, false, 100, 100, 0),
(None, true, false, false, 100, 50, 1),
(Some(100), true, true, false, 100, 50, 1),
];
for (
compression_rate,
same_format,
has_resize,
has_target_size,
original_size,
output_size,
expected_units,
) in cases
{
let charged = quota::output_consumes_unit(
compression_rate,
same_format,
has_resize,
has_target_size,
original_size,
output_size,
);
assert_eq!(metered_units(charged), expected_units);
}
}
#[tokio::test]
async fn detached_sync_operation_survives_waiter_abort() {
let started = Arc::new(tokio::sync::Notify::new());
let release = Arc::new(tokio::sync::Notify::new());
let completed = Arc::new(AtomicBool::new(false));
let request_started = started.clone();
let request_release = release.clone();
let request_completed = completed.clone();
let request = tokio::spawn(async move {
spawn_detached_operation(async move {
request_started.notify_one();
request_release.notified().await;
request_completed.store(true, Ordering::SeqCst);
})
.await
.expect("detached operation panicked");
});
started.notified().await;
request.abort();
request.await.expect_err("request waiter was not aborted");
release.notify_one();
tokio::time::timeout(std::time::Duration::from_secs(1), async {
while !completed.load(Ordering::SeqCst) {
tokio::task::yield_now().await;
}
})
.await
.expect("detached operation was canceled with its request waiter");
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -17,18 +17,33 @@ const SCRAPE_TIMEOUT: Duration = Duration::from_secs(2);
pub async fn metrics(State(state): State<AppState>) -> impl IntoResponse {
let database = tokio::time::timeout(
SCRAPE_TIMEOUT,
sqlx::query_scalar::<_, i64>(
"SELECT COUNT(*) FROM tasks WHERE status IN ('pending', 'processing')",
sqlx::query_as::<_, (i64, i64, i64, i64, i64)>(
r#"
SELECT
(SELECT COUNT(*) FROM tasks WHERE status IN ('pending', 'processing')),
(SELECT COUNT(*) FROM task_queue_outbox WHERE status IN ('pending', 'delivering')),
(SELECT COUNT(*) FROM task_queue_outbox WHERE status = 'dead'),
(SELECT COUNT(*) FROM storage_objects WHERE state = 'delete_pending'),
(SELECT COUNT(*) FROM storage_objects WHERE state = 'staging')
"#,
)
.fetch_one(&state.db),
);
let redis = tokio::time::timeout(SCRAPE_TIMEOUT, redis_queue_stats(state.redis.clone()));
let (database, redis) = tokio::join!(database, redis);
let (database_up, active_tasks) = match database {
Ok(Ok(value)) => (1, value),
_ => (0, 0),
};
let (database_up, active_tasks, outbox_pending, outbox_dead, delete_pending, staging) =
match database {
Ok(Ok((active, outbox_pending, outbox_dead, delete_pending, staging))) => (
1,
active,
outbox_pending,
outbox_dead,
delete_pending,
staging,
),
_ => (0, 0, 0, 0, 0, 0),
};
let (redis_up, queue_length, pending, dead_length, cluster) = match redis {
Ok(Ok((queue_length, pending, dead_length, cluster))) => {
(1, queue_length, pending, dead_length, cluster)
@@ -51,6 +66,28 @@ pub async fn metrics(State(state): State<AppState>) -> impl IntoResponse {
output.push_str("# HELP imageforge_active_tasks Current pending or processing tasks.\n");
output.push_str("# TYPE imageforge_active_tasks gauge\n");
let _ = writeln!(output, "imageforge_active_tasks {active_tasks}");
output.push_str("# HELP imageforge_task_outbox Current durable task delivery states.\n");
output.push_str("# TYPE imageforge_task_outbox gauge\n");
let _ = writeln!(
output,
"imageforge_task_outbox{{state=\"pending\"}} {outbox_pending}"
);
let _ = writeln!(
output,
"imageforge_task_outbox{{state=\"dead\"}} {outbox_dead}"
);
output.push_str(
"# HELP imageforge_storage_object_lifecycle Current durable object cleanup states.\n",
);
output.push_str("# TYPE imageforge_storage_object_lifecycle gauge\n");
let _ = writeln!(
output,
"imageforge_storage_object_lifecycle{{state=\"delete_pending\"}} {delete_pending}"
);
let _ = writeln!(
output,
"imageforge_storage_object_lifecycle{{state=\"staging\"}} {staging}"
);
output.push_str("# HELP imageforge_queue_messages Current Redis stream message counts.\n");
output.push_str("# TYPE imageforge_queue_messages gauge\n");
let _ = writeln!(

View File

@@ -44,7 +44,7 @@ pub async fn run(state: AppState) -> Result<(), AppError> {
.nest("/api/v1", v1)
.fallback_service(static_service)
.layer(axum::middleware::from_fn(request_context::middleware))
.with_state(state);
.with_state(state.clone());
let listener = tokio::net::TcpListener::bind(&addr)
.await
@@ -52,12 +52,22 @@ pub async fn run(state: AppState) -> Result<(), AppError> {
tracing::info!(addr = %addr, "API server listening");
axum::serve(
let reconciliation_task = tokio::spawn(webhooks::reconciliation_loop(state.clone()));
let queue_dispatch_task =
tokio::spawn(crate::services::task_queue::dispatch_loop(state.clone()));
let object_lifecycle_task = tokio::spawn(crate::services::object_lifecycle::maintenance_loop(
state.clone(),
));
let serve_result = axum::serve(
listener,
app.into_make_service_with_connect_info::<SocketAddr>(),
)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "HTTP 服务异常退出").with_source(err))
.await;
reconciliation_task.abort();
queue_dispatch_task.abort();
object_lifecycle_task.abort();
serve_result
.map_err(|err| AppError::new(ErrorCode::Internal, "HTTP 服务异常退出").with_source(err))
}
fn v1_router() -> Router<AppState> {

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -17,6 +17,9 @@ pub struct Config {
pub worker_task_concurrency: u32,
pub worker_concurrency: u32,
pub image_processing_concurrency: u32,
pub zip_build_concurrency: u32,
pub zip_max_entries: u32,
pub zip_max_uncompressed_bytes: u64,
pub jwt_secret: String,
pub jwt_expiry_hours: i64,
@@ -25,6 +28,7 @@ pub struct Config {
pub stripe_secret_key: Option<String>,
pub stripe_webhook_secret: Option<String>,
pub stripe_api_base_url: String,
pub storage_path: String,
@@ -80,6 +84,15 @@ impl Config {
.map(|v| v.get() as u32)
.unwrap_or(4)
});
let zip_build_concurrency = env_u32("ZIP_BUILD_CONCURRENCY")
.filter(|value| *value > 0)
.unwrap_or(2);
let zip_max_entries = env_u32("ZIP_MAX_ENTRIES")
.filter(|value| *value > 0)
.unwrap_or(200);
let zip_max_uncompressed_bytes = env_u64("ZIP_MAX_UNCOMPRESSED_BYTES")
.filter(|value| *value > 0)
.unwrap_or(2 * 1024 * 1024 * 1024);
let jwt_secret = env_string("JWT_SECRET")
.ok_or_else(|| AppError::new(ErrorCode::InvalidRequest, "缺少环境变量 JWT_SECRET"))?;
@@ -99,6 +112,10 @@ impl Config {
}
let stripe_secret_key = env_string("STRIPE_SECRET_KEY");
let stripe_webhook_secret = env_string("STRIPE_WEBHOOK_SECRET");
let stripe_api_base_url = env_string("STRIPE_API_BASE_URL")
.unwrap_or_else(|| "https://api.stripe.com".to_string())
.trim_end_matches('/')
.to_string();
let storage_path = env_string("STORAGE_PATH").unwrap_or_else(|| "./uploads".to_string());
@@ -135,11 +152,15 @@ impl Config {
worker_task_concurrency,
worker_concurrency,
image_processing_concurrency,
zip_build_concurrency,
zip_max_entries,
zip_max_uncompressed_bytes,
jwt_secret,
jwt_expiry_hours,
api_key_pepper,
stripe_secret_key,
stripe_webhook_secret,
stripe_api_base_url,
storage_path,
allow_anonymous_upload,
anon_max_file_size_mb,

View File

@@ -36,6 +36,9 @@ async fn main() -> Result<(), AppError> {
let image_processing_semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(
config.image_processing_concurrency as usize,
));
let zip_build_semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(
config.zip_build_concurrency as usize,
));
let state = AppState {
config,
@@ -43,6 +46,7 @@ async fn main() -> Result<(), AppError> {
redis,
mailer: std::sync::Arc::new(mailer),
image_processing_semaphore,
zip_build_semaphore,
runtime_policy_cache: crate::services::settings::RuntimePolicyCache::new(),
storage_cache: crate::services::storage::StorageCache::new(),
};

View File

@@ -33,11 +33,12 @@ const AVIF_TARGET_MIN_QUALITY: u8 = 38;
const JPEG_PERCEPTUAL_QUALITY: u8 = 72;
const WEBP_PERCEPTUAL_QUALITY: u8 = 70;
const AVIF_PERCEPTUAL_QUALITY: u8 = 55;
const JPEG_TARGET_MAX_QUALITY: u8 = 90;
const WEBP_TARGET_MAX_QUALITY: u8 = 92;
const AVIF_TARGET_MAX_QUALITY: u8 = 90;
const JPEG_TARGET_MAX_QUALITY: u8 = 100;
const WEBP_TARGET_MAX_QUALITY: u8 = 100;
const AVIF_TARGET_MAX_QUALITY: u8 = 100;
const AVIF_ENCODER_SPEED: u8 = 5;
const WEBP_TARGET_SAFETY_PERCENT: u64 = 97;
const WEBP_HIGH_EFFORT_LOSSLESS_MAX_PIXELS: u64 = 2_100_000;
const METADATA_TARGET_OVERHEAD: u64 = 1024;
#[derive(Clone)]
@@ -927,6 +928,23 @@ fn encode_webp_target(
) -> Result<Vec<u8>, AppError> {
deadline.check()?;
let pixels = prepare_target_pixels(&image);
let lossless_candidate = encode_webp_lossless_pixels(&pixels);
deadline.check()?;
match lossless_candidate {
Ok(lossless) if lossless.len() as u64 <= target_size => return Ok(lossless),
Ok(_) => {}
Err(error) => {
tracing::debug!(error = %error, "WebP 无损候选编码失败,继续尝试有损编码");
}
}
deadline.check()?;
let max_lossy = encode_webp_pixels(&pixels, WEBP_TARGET_MAX_QUALITY)?;
deadline.check()?;
if max_lossy.len() as u64 <= target_size {
return Ok(max_lossy);
}
let native_min_quality = if allow_resize {
WEBP_PERCEPTUAL_QUALITY
} else {
@@ -987,6 +1005,37 @@ fn encode_webp_native_target(
})
}
fn encode_webp_lossless_pixels(pixels: &TargetPixels) -> Result<Vec<u8>, AppError> {
let mut config = webp::WebPConfig::new()
.map_err(|_| AppError::new(ErrorCode::CompressionFailed, "初始化 WebP 无损配置失败"))?;
config.lossless = 1;
config.quality = 100.0;
config.method = webp_lossless_method(pixels);
config.alpha_compression = 1;
config.near_lossless = 100;
config.exact = 1;
config.thread_level = 0;
webp_encoder(pixels)
.encode_advanced(&config)
.map(|bytes| bytes.to_vec())
.map_err(|err| {
AppError::new(
ErrorCode::CompressionFailed,
format!("WebP 无损编码失败: {err:?}"),
)
})
}
fn webp_lossless_method(pixels: &TargetPixels) -> i32 {
let pixel_count = u64::from(pixels.width).saturating_mul(u64::from(pixels.height));
if pixel_count <= WEBP_HIGH_EFFORT_LOSSLESS_MAX_PIXELS {
6
} else {
0
}
}
fn encode_avif_target(
image: DynamicImage,
target_size: u64,
@@ -1942,6 +1991,79 @@ mod tests {
assert_eq!(detect_format(&output).unwrap(), ImageFmt::Webp);
}
#[test]
fn webp_target_prefers_lossless_when_it_fits() {
let image = DynamicImage::ImageRgb8(RgbImage::from_fn(160, 120, |x, y| {
let block = ((x / 20) + (y / 20) * 3) as u8;
Rgb([
block.wrapping_mul(31),
block.wrapping_mul(17),
block.wrapping_mul(11),
])
}));
let pixels = prepare_target_pixels(&image);
let lossless = encode_webp_lossless_pixels(&pixels).unwrap();
let output = encode_webp_target(
image.clone(),
lossless.len() as u64,
true,
&CompressionDeadline::unlimited(),
)
.unwrap();
assert_eq!(output, lossless);
assert_eq!(
image::load_from_memory(&output).unwrap().to_rgb8(),
image.to_rgb8()
);
}
#[test]
fn webp_target_uses_quality_100_when_lossless_exceeds_the_cap() {
let mut state = 0x7f4a_7c15_u32;
let image = DynamicImage::ImageRgb8(RgbImage::from_fn(160, 120, |_x, _y| {
let mut channel = || {
state ^= state << 13;
state ^= state >> 17;
state ^= state << 5;
state as u8
};
Rgb([channel(), channel(), channel()])
}));
let pixels = prepare_target_pixels(&image);
let max_lossy = encode_webp_pixels(&pixels, WEBP_TARGET_MAX_QUALITY).unwrap();
let lossless = encode_webp_lossless_pixels(&pixels).unwrap();
assert!(max_lossy.len() < lossless.len());
let output = encode_webp_target(
image,
max_lossy.len() as u64,
true,
&CompressionDeadline::unlimited(),
)
.unwrap();
assert_eq!(output, max_lossy);
}
#[test]
fn large_webp_targets_use_the_fast_lossless_probe() {
let small = TargetPixels {
bytes: Vec::new(),
width: 1920,
height: 1080,
layout: TargetPixelLayout::Rgb,
};
let large = TargetPixels {
bytes: Vec::new(),
width: 4096,
height: 3072,
layout: TargetPixelLayout::Rgb,
};
assert_eq!(webp_lossless_method(&small), 6);
assert_eq!(webp_lossless_method(&large), 0);
}
#[test]
fn jpeg_target_encoder_prefers_perceptual_downscaling() {
let image = DynamicImage::ImageRgb8(RgbImage::from_fn(800, 600, |x, y| {

View File

@@ -2,8 +2,11 @@ use crate::error::{AppError, ErrorCode};
use argon2::{Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
use chrono::{DateTime, Utc};
use rand::RngCore;
use sha2::{Digest, Sha256};
use sqlx::{Postgres, Transaction};
use uuid::Uuid;
pub fn validate_email(email: &str) -> Result<(), AppError> {
if email.trim().is_empty() || !email.contains('@') {
@@ -80,6 +83,34 @@ pub async fn consume_dummy_password_work(password: &str) -> Result<(), AppError>
.map_err(|err| AppError::new(ErrorCode::Internal, "密码校验失败").with_source(err))
}
pub async fn invalidate_account_recovery(
tx: &mut Transaction<'_, Postgres>,
user_id: Uuid,
invalidated_at: DateTime<Utc>,
) -> Result<(), AppError> {
sqlx::query(
r#"
WITH consumed_resets AS (
UPDATE password_resets
SET used_at = $2
WHERE user_id = $1 AND used_at IS NULL
RETURNING id
)
UPDATE email_change_requests
SET canceled_at = $2
WHERE user_id = $1
AND confirmed_at IS NULL
AND canceled_at IS NULL
"#,
)
.bind(user_id)
.bind(invalidated_at)
.execute(&mut **tx)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "撤销账号恢复凭据失败").with_source(err))?;
Ok(())
}
pub fn generate_token() -> String {
let mut bytes = [0u8; 32];
rand::rngs::OsRng.fill_bytes(&mut bytes);

View File

@@ -3,10 +3,11 @@ use crate::state::AppState;
use chrono::{DateTime, Duration, Utc};
use serde_json::Value as JsonValue;
use sha2::{Digest, Sha256};
use sqlx::FromRow;
use uuid::Uuid;
const OPERATION_LEASE_MINUTES: i64 = 30;
#[derive(Debug, Clone, Copy)]
pub enum Scope {
User(Uuid),
@@ -15,7 +16,7 @@ pub enum Scope {
#[derive(Debug)]
pub enum BeginResult {
Acquired,
Acquired { owner: Uuid },
Replay { response_body: JsonValue },
InProgress,
}
@@ -27,15 +28,6 @@ struct IdemRow {
response_body: Option<JsonValue>,
}
pub fn sha256_hex(parts: &[&[u8]]) -> String {
let mut hasher = Sha256::new();
for p in parts {
hasher.update(p);
hasher.update([0u8]); // separator
}
hex::encode(hasher.finalize())
}
pub async fn begin(
state: &AppState,
scope: Scope,
@@ -64,6 +56,8 @@ pub async fn begin(
let now = Utc::now();
let expires_at = now + Duration::hours(ttl_hours.max(1));
let owner = Uuid::new_v4();
let lease_until = now + Duration::minutes(OPERATION_LEASE_MINUTES);
cleanup_expired_for_key(state, scope, idempotency_key, now).await?;
@@ -74,11 +68,11 @@ pub async fn begin(
INSERT INTO idempotency_keys (
user_id, idempotency_key, request_hash,
response_status, response_body,
expires_at
expires_at, lease_owner, lease_until
) VALUES (
$1, $2, $3,
0, NULL,
$4
$4, $5, $6
)
ON CONFLICT DO NOTHING
"#,
@@ -87,6 +81,8 @@ pub async fn begin(
.bind(idempotency_key)
.bind(request_hash)
.bind(expires_at)
.bind(owner)
.bind(lease_until)
.execute(&state.db)
.await
}
@@ -96,11 +92,11 @@ pub async fn begin(
INSERT INTO idempotency_keys (
api_key_id, idempotency_key, request_hash,
response_status, response_body,
expires_at
expires_at, lease_owner, lease_until
) VALUES (
$1, $2, $3,
0, NULL,
$4
$4, $5, $6
)
ON CONFLICT DO NOTHING
"#,
@@ -109,6 +105,8 @@ pub async fn begin(
.bind(idempotency_key)
.bind(request_hash)
.bind(expires_at)
.bind(owner)
.bind(lease_until)
.execute(&state.db)
.await
}
@@ -116,12 +114,15 @@ pub async fn begin(
.map_err(|err| AppError::new(ErrorCode::Internal, "写入幂等记录失败").with_source(err))?;
if inserted.rows_affected() > 0 {
return Ok(BeginResult::Acquired);
return Ok(BeginResult::Acquired { owner });
}
let row = get_row(state, scope, idempotency_key, now).await?;
let Some(row) = row else {
return Ok(BeginResult::Acquired);
return Err(AppError::new(
ErrorCode::StorageUnavailable,
"幂等记录状态已变化,请重试",
));
};
if row.request_hash != request_hash {
@@ -132,6 +133,19 @@ pub async fn begin(
}
if row.response_status == 0 || row.response_body.is_none() {
if take_over_stale_operation(
state,
scope,
idempotency_key,
request_hash,
owner,
lease_until,
now,
)
.await?
{
return Ok(BeginResult::Acquired { owner });
}
return Ok(BeginResult::InProgress);
}
@@ -140,6 +154,152 @@ pub async fn begin(
})
}
async fn take_over_stale_operation(
state: &AppState,
scope: Scope,
idempotency_key: &str,
request_hash: &str,
owner: Uuid,
lease_until: DateTime<Utc>,
now: DateTime<Utc>,
) -> Result<bool, AppError> {
let updated = match scope {
Scope::User(user_id) => {
sqlx::query(
r#"
UPDATE idempotency_keys
SET lease_owner = $4, lease_until = $5
WHERE user_id = $1 AND idempotency_key = $2
AND request_hash = $3 AND response_status = 0
AND (lease_until IS NULL OR lease_until <= $6)
"#,
)
.bind(user_id)
.bind(idempotency_key)
.bind(request_hash)
.bind(owner)
.bind(lease_until)
.bind(now)
.execute(&state.db)
.await
}
Scope::ApiKey(api_key_id) => {
sqlx::query(
r#"
UPDATE idempotency_keys
SET lease_owner = $4, lease_until = $5
WHERE api_key_id = $1 AND idempotency_key = $2
AND request_hash = $3 AND response_status = 0
AND (lease_until IS NULL OR lease_until <= $6)
"#,
)
.bind(api_key_id)
.bind(idempotency_key)
.bind(request_hash)
.bind(owner)
.bind(lease_until)
.bind(now)
.execute(&state.db)
.await
}
}
.map_err(|err| AppError::new(ErrorCode::Internal, "接管过期幂等操作失败").with_source(err))?;
Ok(updated.rows_affected() == 1)
}
pub struct LeaseHeartbeat(Option<tokio::sync::oneshot::Sender<()>>);
impl Drop for LeaseHeartbeat {
fn drop(&mut self) {
if let Some(stop) = self.0.take() {
let _ = stop.send(());
}
}
}
pub fn start_lease_heartbeat(
state: AppState,
scope: Scope,
idempotency_key: String,
request_hash: String,
owner: Uuid,
) -> LeaseHeartbeat {
let (stop_tx, mut stop_rx) = tokio::sync::oneshot::channel();
tokio::spawn(async move {
let mut interval = tokio::time::interval(std::time::Duration::from_secs(60));
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
interval.tick().await;
loop {
tokio::select! {
_ = &mut stop_rx => break,
_ = interval.tick() => {
match renew_lease(
&state,
scope,
&idempotency_key,
&request_hash,
owner,
).await {
Ok(true) => {}
Ok(false) => break,
Err(err) => tracing::warn!(error = %err, "failed to renew idempotency operation lease"),
}
}
}
}
});
LeaseHeartbeat(Some(stop_tx))
}
async fn renew_lease(
state: &AppState,
scope: Scope,
idempotency_key: &str,
request_hash: &str,
owner: Uuid,
) -> Result<bool, AppError> {
let updated = match scope {
Scope::User(user_id) => {
sqlx::query(
r#"
UPDATE idempotency_keys
SET lease_until = NOW() + ($5 * INTERVAL '1 minute')
WHERE user_id = $1 AND idempotency_key = $2
AND request_hash = $3 AND lease_owner = $4
AND response_status = 0
"#,
)
.bind(user_id)
.bind(idempotency_key)
.bind(request_hash)
.bind(owner)
.bind(OPERATION_LEASE_MINUTES)
.execute(&state.db)
.await
}
Scope::ApiKey(api_key_id) => {
sqlx::query(
r#"
UPDATE idempotency_keys
SET lease_until = NOW() + ($5 * INTERVAL '1 minute')
WHERE api_key_id = $1 AND idempotency_key = $2
AND request_hash = $3 AND lease_owner = $4
AND response_status = 0
"#,
)
.bind(api_key_id)
.bind(idempotency_key)
.bind(request_hash)
.bind(owner)
.bind(OPERATION_LEASE_MINUTES)
.execute(&state.db)
.await
}
}
.map_err(|err| AppError::new(ErrorCode::Internal, "续租幂等操作失败").with_source(err))?;
Ok(updated.rows_affected() == 1)
}
pub async fn wait_for_replay(
state: &AppState,
scope: Scope,
@@ -175,11 +335,12 @@ pub async fn wait_for_replay(
}
}
pub async fn complete(
state: &AppState,
pub async fn complete_in_tx(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
scope: Scope,
idempotency_key: &str,
request_hash: &str,
owner: Uuid,
response_status: i32,
response_body: JsonValue,
) -> Result<(), AppError> {
@@ -189,10 +350,13 @@ pub async fn complete(
r#"
UPDATE idempotency_keys
SET response_status = $4,
response_body = $5
response_body = $5,
lease_owner = NULL,
lease_until = NULL
WHERE user_id = $1
AND idempotency_key = $2
AND request_hash = $3
AND lease_owner = $6
AND response_status = 0
"#,
)
@@ -201,7 +365,8 @@ pub async fn complete(
.bind(request_hash)
.bind(response_status)
.bind(response_body)
.execute(&state.db)
.bind(owner)
.execute(&mut **tx)
.await
}
Scope::ApiKey(api_key_id) => {
@@ -209,10 +374,13 @@ pub async fn complete(
r#"
UPDATE idempotency_keys
SET response_status = $4,
response_body = $5
response_body = $5,
lease_owner = NULL,
lease_until = NULL
WHERE api_key_id = $1
AND idempotency_key = $2
AND request_hash = $3
AND lease_owner = $6
AND response_status = 0
"#,
)
@@ -221,16 +389,19 @@ pub async fn complete(
.bind(request_hash)
.bind(response_status)
.bind(response_body)
.execute(&state.db)
.bind(owner)
.execute(&mut **tx)
.await
}
}
.map_err(|err| AppError::new(ErrorCode::Internal, "写入幂等结果失败").with_source(err))?;
.map_err(|err| AppError::new(ErrorCode::Internal, "事务内写入幂等结果失败").with_source(err))?;
if updated.rows_affected() == 0 {
tracing::warn!("idempotency record not updated (maybe already completed?)");
if updated.rows_affected() != 1 {
return Err(AppError::new(
ErrorCode::IdempotencyConflict,
"幂等请求所有权已变化,请重试",
));
}
Ok(())
}
@@ -239,25 +410,28 @@ pub async fn abort(
scope: Scope,
idempotency_key: &str,
request_hash: &str,
owner: Uuid,
) -> Result<(), AppError> {
match scope {
Scope::User(user_id) => {
let _ = sqlx::query(
"DELETE FROM idempotency_keys WHERE user_id = $1 AND idempotency_key = $2 AND request_hash = $3 AND response_status = 0",
"DELETE FROM idempotency_keys WHERE user_id = $1 AND idempotency_key = $2 AND request_hash = $3 AND lease_owner = $4 AND response_status = 0",
)
.bind(user_id)
.bind(idempotency_key)
.bind(request_hash)
.bind(owner)
.execute(&state.db)
.await;
}
Scope::ApiKey(api_key_id) => {
let _ = sqlx::query(
"DELETE FROM idempotency_keys WHERE api_key_id = $1 AND idempotency_key = $2 AND request_hash = $3 AND response_status = 0",
"DELETE FROM idempotency_keys WHERE api_key_id = $1 AND idempotency_key = $2 AND request_hash = $3 AND lease_owner = $4 AND response_status = 0",
)
.bind(api_key_id)
.bind(idempotency_key)
.bind(request_hash)
.bind(owner)
.execute(&state.db)
.await;
}
@@ -344,3 +518,149 @@ async fn get_row(
Ok(row)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::Config;
use crate::services::mail::Mailer;
use crate::services::settings::RuntimePolicyCache;
use crate::services::storage::StorageCache;
use sqlx::postgres::PgPoolOptions;
use std::sync::Arc;
use tokio::sync::Semaphore;
async fn test_state(database_url: String, redis_url: String) -> AppState {
let mut config = Config::from_env().expect("load idempotency test config");
config.database_url = database_url.clone();
config.redis_url = redis_url;
config.mail_enabled = false;
config.mail_log_links_when_disabled = false;
let db = PgPoolOptions::new()
.max_connections(8)
.connect(&database_url)
.await
.expect("connect idempotency test database");
sqlx::migrate!().run(&db).await.expect("run migrations");
let redis = redis::Client::open(config.redis_url.clone())
.expect("create idempotency test Redis client")
.get_connection_manager()
.await
.expect("connect idempotency test Redis");
AppState {
mailer: Arc::new(Mailer::new(&config).expect("create disabled test mailer")),
image_processing_semaphore: Arc::new(Semaphore::new(1)),
zip_build_semaphore: Arc::new(Semaphore::new(1)),
runtime_policy_cache: RuntimePolicyCache::new(),
storage_cache: StorageCache::new(),
config,
db,
redis,
}
}
#[tokio::test]
#[ignore = "requires IMAGEFORGE_TEST_DATABASE_URL and IMAGEFORGE_TEST_REDIS_URL"]
async fn stale_operation_is_fenced_and_replay_remains_atomic() {
let database_url = std::env::var("IMAGEFORGE_TEST_DATABASE_URL")
.expect("IMAGEFORGE_TEST_DATABASE_URL is required");
let redis_url = std::env::var("IMAGEFORGE_TEST_REDIS_URL")
.expect("IMAGEFORGE_TEST_REDIS_URL is required");
let state = test_state(database_url, redis_url).await;
let marker = Uuid::new_v4().simple().to_string();
let user_id: Uuid = sqlx::query_scalar(
r#"
INSERT INTO users (email, username, password_hash, email_verified_at)
VALUES ($1, $2, 'test', NOW())
RETURNING id
"#,
)
.bind(format!("idem-{marker}@example.test"))
.bind(format!("idem-{marker}"))
.fetch_one(&state.db)
.await
.expect("insert idempotency test user");
let scope = Scope::User(user_id);
let key = format!("idem-{marker}");
let request_hash = "a".repeat(64);
let owner_one = match begin(&state, scope, &key, &request_hash, 24)
.await
.expect("acquire first operation")
{
BeginResult::Acquired { owner } => owner,
other => panic!("unexpected first begin result: {other:?}"),
};
assert!(matches!(
begin(&state, scope, &key, &request_hash, 24)
.await
.expect("probe live operation"),
BeginResult::InProgress
));
sqlx::query(
"UPDATE idempotency_keys SET lease_until = NOW() - INTERVAL '1 second' WHERE user_id = $1 AND idempotency_key = $2",
)
.bind(user_id)
.bind(&key)
.execute(&state.db)
.await
.expect("expire first operation lease");
let owner_two = match begin(&state, scope, &key, &request_hash, 24)
.await
.expect("take over stale operation")
{
BeginResult::Acquired { owner } => owner,
other => panic!("unexpected takeover result: {other:?}"),
};
assert_ne!(owner_one, owner_two);
let mut stale_tx = state.db.begin().await.expect("begin stale completion tx");
let stale_error = complete_in_tx(
&mut stale_tx,
scope,
&key,
&request_hash,
owner_one,
200,
serde_json::json!({"owner": "stale"}),
)
.await
.expect_err("stale operation completed after takeover");
assert_eq!(stale_error.code, ErrorCode::IdempotencyConflict);
stale_tx
.rollback()
.await
.expect("rollback stale completion");
let expected = serde_json::json!({"owner": "current"});
let mut current_tx = state.db.begin().await.expect("begin current completion tx");
complete_in_tx(
&mut current_tx,
scope,
&key,
&request_hash,
owner_two,
200,
expected.clone(),
)
.await
.expect("complete current operation");
current_tx
.commit()
.await
.expect("commit current completion");
match begin(&state, scope, &key, &request_hash, 24)
.await
.expect("replay completed operation")
{
BeginResult::Replay { response_body } => assert_eq!(response_body, expected),
other => panic!("unexpected replay result: {other:?}"),
}
sqlx::query("DELETE FROM users WHERE id = $1")
.bind(user_id)
.execute(&state.db)
.await
.expect("clean idempotency test user");
}
}

View File

@@ -183,6 +183,13 @@ impl Mailer {
.await
}
pub async fn send_email_change_notice(&self, to: &str) -> Result<(), AppError> {
let text = "您的 ImageForge 账号邮箱已完成修改。若非本人操作,请立即联系管理员并重置密码。";
let html = "<h2>账号邮箱已修改</h2><p>您的 ImageForge 账号邮箱已完成修改。</p><p>若非本人操作,请立即联系管理员并重置密码。</p>";
self.send_email(to, "ImageForge 账号邮箱变更通知", text, html)
.await
}
async fn send_email(
&self,
to: &str,
@@ -336,6 +343,11 @@ pub async fn send_password_reset_email(
.await
}
pub async fn send_email_change_notice(state: &AppState, to: &str) -> Result<(), AppError> {
let mailer = resolve_mailer(state).await?;
mailer.send_email_change_notice(to).await
}
pub async fn send_test_email(state: &AppState, to: &str) -> Result<(), AppError> {
let mailer = resolve_mailer(state).await?;
let year = chrono::Utc::now().year().to_string();

View File

@@ -6,7 +6,9 @@ pub mod filename;
pub mod idempotency;
pub mod mail;
pub mod metrics;
pub mod object_lifecycle;
pub mod quota;
pub mod rate_limit;
pub mod settings;
pub mod storage;
pub mod task_queue;

File diff suppressed because it is too large Load Diff

View File

@@ -320,55 +320,257 @@ pub async fn reserve_anonymous_units(
Ok(date)
}
pub async fn refund_anonymous_units(
pub async fn reserve_anonymous_single_unit(
state: &AppState,
task_id: Uuid,
session_id: &str,
ip: IpAddr,
units: u32,
) -> Result<(), AppError> {
refund_anonymous_units_for_date(state, session_id, ip, utc8_date(), units).await
) -> Result<NaiveDate, AppError> {
reserve_anonymous_single_unit_for_date(state, task_id, session_id, ip, utc8_date()).await
}
async fn refund_anonymous_units_for_date(
async fn reserve_anonymous_single_unit_for_date(
state: &AppState,
task_id: Uuid,
session_id: &str,
ip: IpAddr,
date: NaiveDate,
units: u32,
) -> Result<(), AppError> {
if units == 0 {
return Ok(());
}
) -> Result<NaiveDate, AppError> {
sqlx::query(
r#"
INSERT INTO anonymous_single_reservations (
task_id, session_id, client_ip, quota_date, units
) VALUES ($1, $2, $3::inet, $4, 1)
"#,
)
.bind(task_id)
.bind(session_id)
.bind(ip.to_string())
.bind(date)
.execute(&state.db)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "创建匿名单文件预留失败").with_source(err))?;
let limit = crate::services::settings::runtime_policy(state)
.await?
.rate_limits
.anonymous_units_per_day as i64;
let session_key = anonymous_session_key(session_id, date);
let ip_key = anonymous_ip_key(ip, date);
let reservation_key = anonymous_single_reservation_key(task_id);
let mut conn = state.redis.clone();
let script = redis::Script::new(
r#"
local limit = tonumber(ARGV[1])
local ttl = tonumber(ARGV[2])
if redis.call('EXISTS', KEYS[3]) == 1 then
return tonumber(redis.call('GET', KEYS[1]) or '0')
end
local session_value = tonumber(redis.call('GET', KEYS[1]) or '0')
local ip_value = tonumber(redis.call('GET', KEYS[2]) or '0')
if session_value + 1 > limit or ip_value + 1 > limit then
return -1
end
session_value = redis.call('INCRBY', KEYS[1], 1)
ip_value = redis.call('INCRBY', KEYS[2], 1)
if session_value == 1 then redis.call('EXPIRE', KEYS[1], ttl) end
if ip_value == 1 then redis.call('EXPIRE', KEYS[2], ttl) end
redis.call('SET', KEYS[3], '1', 'EX', ttl)
return session_value
"#,
);
let reserved: i64 = script
.key(session_key)
.key(ip_key)
.key(reservation_key)
.arg(limit)
.arg(48 * 60 * 60)
.invoke_async(&mut conn)
.await
.map_err(|err| {
// Keep the durable pending row: the script may have committed even
// if the response was lost, and maintenance can safely reconcile it.
AppError::new(ErrorCode::Internal, "匿名配额检查失败").with_source(err)
})?;
if reserved < 0 {
sqlx::query(
r#"
UPDATE anonymous_single_reservations
SET status = 'refunded', settled_at = NOW(), updated_at = NOW()
WHERE task_id = $1 AND status = 'pending'
"#,
)
.bind(task_id)
.execute(&state.db)
.await
.map_err(|err| {
AppError::new(ErrorCode::Internal, "关闭匿名单文件预留失败").with_source(err)
})?;
return Err(AppError::new(
ErrorCode::QuotaExceeded,
format!("匿名试用次数已用完(每日 {limit} 次)"),
));
}
Ok(date)
}
pub async fn mark_anonymous_single_result(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
task_id: Uuid,
charged: bool,
) -> Result<(), AppError> {
let status = if charged { "charged" } else { "refund_pending" };
let updated = sqlx::query(
r#"
UPDATE anonymous_single_reservations
SET status = $2,
settled_at = CASE WHEN $2 = 'charged' THEN NOW() ELSE settled_at END,
updated_at = NOW()
WHERE task_id = $1 AND status = 'pending'
"#,
)
.bind(task_id)
.bind(status)
.execute(&mut **tx)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "更新匿名单文件预留失败").with_source(err))?;
if updated.rows_affected() != 1 {
return Err(AppError::new(
ErrorCode::StorageUnavailable,
"匿名单文件预留已失效,请重试",
));
}
Ok(())
}
pub async fn finalize_anonymous_single_reservation(
state: &AppState,
task_id: Uuid,
charged: bool,
) -> Result<(), AppError> {
if charged {
let mut conn = state.redis.clone();
let _: i64 = redis::cmd("DEL")
.arg(anonymous_single_reservation_key(task_id))
.query_async(&mut conn)
.await
.map_err(|err| {
AppError::new(ErrorCode::Internal, "完成匿名单文件计费失败").with_source(err)
})?;
return Ok(());
}
refund_anonymous_single_reservation(state, task_id).await
}
pub async fn refund_anonymous_single_reservation(
state: &AppState,
task_id: Uuid,
) -> Result<(), AppError> {
let row: Option<(String, String, NaiveDate, i32)> = sqlx::query_as(
r#"
SELECT session_id, host(client_ip), quota_date, units
FROM anonymous_single_reservations
WHERE task_id = $1
AND status IN ('pending', 'refund_pending', 'refunded')
"#,
)
.bind(task_id)
.fetch_optional(&state.db)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "查询匿名单文件预留失败").with_source(err))?;
let Some((session_id, ip, date, units)) = row else {
return Ok(());
};
let ip: IpAddr = ip.parse().map_err(|err| {
AppError::new(ErrorCode::Internal, "匿名单文件预留 IP 无效").with_source(err)
})?;
let units = u32::try_from(units).unwrap_or(0);
let session_key = anonymous_session_key(&session_id, date);
let ip_key = anonymous_ip_key(ip, date);
let reservation_key = anonymous_single_reservation_key(task_id);
let refund_key = format!("anon_quota_refund:{task_id}");
let mut conn = state.redis.clone();
let script = redis::Script::new(
r#"
local dec = tonumber(ARGV[1])
local ttl = tonumber(ARGV[2])
if redis.call('EXISTS', KEYS[4]) == 1 then return 0 end
local function refund(key)
local current = tonumber(redis.call('GET', key) or '0')
if current <= 0 then return 0 end
return redis.call('DECRBY', key, math.min(current, dec))
if redis.call('EXISTS', KEYS[3]) == 1 then
local function refund(key)
local current = tonumber(redis.call('GET', key) or '0')
if current <= 0 then return 0 end
return redis.call('DECRBY', key, math.min(current, dec))
end
refund(KEYS[1])
refund(KEYS[2])
redis.call('DEL', KEYS[3])
end
refund(KEYS[1])
refund(KEYS[2])
redis.call('SET', KEYS[4], '1', 'EX', ttl)
return 1
"#,
);
let _: i64 = script
.key(session_key)
.key(ip_key)
.key(reservation_key)
.key(refund_key)
.arg(units as i64)
.arg(48 * 60 * 60)
.invoke_async(&mut conn)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "退还匿名配额失败").with_source(err))?;
.map_err(|err| {
AppError::new(ErrorCode::Internal, "退还匿名单文件配额失败").with_source(err)
})?;
sqlx::query(
r#"
UPDATE anonymous_single_reservations
SET status = 'refunded', settled_at = NOW(), updated_at = NOW()
WHERE task_id = $1 AND status IN ('pending', 'refund_pending', 'refunded')
"#,
)
.bind(task_id)
.execute(&state.db)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "记录匿名单文件退款失败").with_source(err))?;
Ok(())
}
pub async fn settle_stale_anonymous_single_reservations(
state: &AppState,
limit: i64,
) -> Result<usize, AppError> {
let task_ids: Vec<Uuid> = sqlx::query_scalar(
r#"
SELECT task_id
FROM anonymous_single_reservations
WHERE status = 'refund_pending'
OR (status = 'pending' AND refund_after <= NOW())
ORDER BY refund_after ASC
LIMIT $1
"#,
)
.bind(limit)
.fetch_all(&state.db)
.await
.map_err(|err| {
AppError::new(ErrorCode::Internal, "查询待补偿匿名单文件预留失败").with_source(err)
})?;
let mut settled = 0;
for task_id in task_ids {
match refund_anonymous_single_reservation(state, task_id).await {
Ok(()) => settled += 1,
Err(err) => {
tracing::warn!(task_id = %task_id, error = %err, "anonymous single reservation refund deferred")
}
}
}
Ok(settled)
}
pub async fn refund_anonymous_reservation_once(
state: &AppState,
task_id: Uuid,
@@ -451,7 +653,8 @@ pub async fn settle_anonymous_task_reservation(
AND f.status = 'completed'
AND f.compressed_size < f.original_size
AND NOT (
tasks.compression_rate = 100
-- NULL means the caller did not request the explicit 100% passthrough.
COALESCE(tasks.compression_rate = 100, false)
AND f.original_format = f.output_format
AND tasks.max_width IS NULL
AND tasks.max_height IS NULL
@@ -514,6 +717,19 @@ pub async fn settle_anonymous_task_reservation(
Ok(Some(refundable))
}
pub(crate) fn output_consumes_unit(
compression_rate: Option<u8>,
same_format: bool,
has_resize: bool,
has_target_size: bool,
original_size: u64,
output_size: u64,
) -> bool {
let is_unmetered_passthrough =
compression_rate == Some(100) && same_format && !has_resize && !has_target_size;
!is_unmetered_passthrough && output_size < original_size
}
fn refundable_reserved_units(reserved: i32, total_files: i32, consumed_units: i32) -> u32 {
let reserved = reserved.max(0);
let total_files = total_files.max(0);
@@ -527,6 +743,10 @@ fn anonymous_session_key(session_id: &str, date: NaiveDate) -> String {
format!("anon_quota:{session_id}:{}", date.format("%Y-%m-%d"))
}
fn anonymous_single_reservation_key(task_id: Uuid) -> String {
format!("anon_quota_reservation:{task_id}")
}
pub(crate) fn anonymous_ip_scope(ip: IpAddr) -> String {
match ip {
IpAddr::V4(ip) => ip.to_string(),
@@ -556,6 +776,126 @@ fn utc8_date() -> NaiveDate {
#[cfg(test)]
mod tests {
use super::*;
use crate::config::Config;
use crate::services::mail::Mailer;
use sqlx::postgres::PgPoolOptions;
use std::sync::Arc;
use tokio::sync::Semaphore;
struct AnonymousSettlementFixture<'a> {
session_id: &'a str,
ip: IpAddr,
date: NaiveDate,
reserved_units: u32,
compression_rate: Option<i16>,
total_files: usize,
completed_files: usize,
}
async fn build_test_state(
pool: sqlx::PgPool,
database_url: String,
redis_url: String,
) -> AppState {
let mut config = Config::from_env().expect("load quota test config");
config.database_url = database_url;
config.redis_url = redis_url;
config.mail_enabled = false;
config.mail_log_links_when_disabled = false;
config.anon_daily_units = 10;
let redis = redis::Client::open(config.redis_url.clone())
.expect("create quota test Redis client")
.get_connection_manager()
.await
.expect("connect quota test Redis");
AppState {
mailer: Arc::new(Mailer::new(&config).expect("create disabled quota test mailer")),
image_processing_semaphore: Arc::new(Semaphore::new(2)),
zip_build_semaphore: Arc::new(Semaphore::new(2)),
runtime_policy_cache: crate::services::settings::RuntimePolicyCache::new(),
storage_cache: crate::services::storage::StorageCache::new(),
config,
db: pool,
redis,
}
}
async fn insert_anonymous_settlement_fixture(
pool: &sqlx::PgPool,
fixture: AnonymousSettlementFixture<'_>,
) -> Uuid {
assert!(fixture.completed_files <= fixture.total_files);
let task_id = Uuid::new_v4();
sqlx::query(
r#"
INSERT INTO tasks (
id, session_id, client_ip, status,
compression_rate, total_files, completed_files, failed_files,
anonymous_units_reserved, anonymous_quota_date
) VALUES (
$1, $2, $3::inet, 'completed',
$4, $5, $6, $7,
$8, $9
)
"#,
)
.bind(task_id)
.bind(fixture.session_id)
.bind(fixture.ip.to_string())
.bind(fixture.compression_rate)
.bind(fixture.total_files as i32)
.bind(fixture.completed_files as i32)
.bind((fixture.total_files - fixture.completed_files) as i32)
.bind(fixture.reserved_units as i32)
.bind(fixture.date)
.execute(pool)
.await
.expect("insert anonymous settlement task");
for index in 0..fixture.total_files {
let completed = index < fixture.completed_files;
sqlx::query(
r#"
INSERT INTO task_files (
id, task_id, original_name, original_format, output_format,
original_size, compressed_size, status
) VALUES (
$1, $2, $3, 'jpeg', 'jpeg',
100, $4, $5::file_status
)
"#,
)
.bind(Uuid::new_v4())
.bind(task_id)
.bind(format!("fixture-{index}.jpg"))
.bind(completed.then_some(50_i64))
.bind(if completed { "completed" } else { "failed" })
.execute(pool)
.await
.expect("insert anonymous settlement file");
}
task_id
}
async fn anonymous_quota_counts(
state: &AppState,
session_id: &str,
ip: IpAddr,
date: NaiveDate,
) -> (i64, i64) {
let mut redis = state.redis.clone();
let session_count: Option<i64> = redis::cmd("GET")
.arg(anonymous_session_key(session_id, date))
.query_async(&mut redis)
.await
.expect("read anonymous session quota");
let ip_count: Option<i64> = redis::cmd("GET")
.arg(anonymous_ip_key(ip, date))
.query_async(&mut redis)
.await
.expect("read anonymous IP quota");
(session_count.unwrap_or(0), ip_count.unwrap_or(0))
}
#[test]
fn balance_keeps_redeemed_units_separate_from_plan_usage() {
@@ -606,6 +946,29 @@ mod tests {
assert_eq!(refundable_reserved_units(10, -1, -2), 0);
}
#[test]
fn output_metering_matches_passthrough_contract() {
assert!(output_consumes_unit(None, true, false, false, 100, 50));
assert!(!output_consumes_unit(
Some(100),
true,
false,
false,
100,
50
));
assert!(output_consumes_unit(
Some(100),
false,
false,
false,
100,
50
));
assert!(output_consumes_unit(Some(100), true, true, false, 100, 50));
assert!(!output_consumes_unit(None, true, false, false, 100, 100));
}
#[test]
fn anonymous_quota_keys_use_the_reserved_date() {
let date = NaiveDate::from_ymd_opt(2026, 7, 25).unwrap();
@@ -633,4 +996,420 @@ mod tests {
"203.0.113.7"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires isolated IMAGEFORGE_TEST_DATABASE_URL and IMAGEFORGE_TEST_REDIS_URL"]
async fn anonymous_batch_settlement_charges_successes_and_refunds_only_unused_units() {
let database_url = std::env::var("IMAGEFORGE_TEST_DATABASE_URL")
.expect("IMAGEFORGE_TEST_DATABASE_URL must be set");
assert!(
database_url.to_ascii_lowercase().contains("test"),
"refusing to run destructive integration test outside a test database"
);
let redis_url = std::env::var("IMAGEFORGE_TEST_REDIS_URL")
.expect("IMAGEFORGE_TEST_REDIS_URL must be set");
let pool = PgPoolOptions::new()
.max_connections(16)
.connect(&database_url)
.await
.expect("connect quota test database");
sqlx::migrate!().run(&pool).await.expect("run migrations");
let state = build_test_state(pool.clone(), database_url, redis_url).await;
let marker = Uuid::new_v4().simple().to_string();
let mut cleanup = Vec::new();
let null_session = format!("quota-null-{marker}");
let null_ip: IpAddr = "198.51.100.11".parse().expect("parse fixture IP");
let null_date = reserve_anonymous_units(&state, &null_session, null_ip, 3)
.await
.expect("reserve NULL-rate batch quota");
let null_task = insert_anonymous_settlement_fixture(
&pool,
AnonymousSettlementFixture {
session_id: &null_session,
ip: null_ip,
date: null_date,
reserved_units: 3,
compression_rate: None,
total_files: 3,
completed_files: 3,
},
)
.await;
cleanup.push((null_task, null_session.clone(), null_ip, null_date));
assert_eq!(
settle_anonymous_task_reservation(&state, null_task)
.await
.expect("settle NULL-rate batch"),
Some(0)
);
assert_eq!(
anonymous_quota_counts(&state, &null_session, null_ip, null_date).await,
(3, 3)
);
let passthrough_session = format!("quota-passthrough-{marker}");
let passthrough_ip: IpAddr = "198.51.100.12".parse().expect("parse fixture IP");
let passthrough_date =
reserve_anonymous_units(&state, &passthrough_session, passthrough_ip, 3)
.await
.expect("reserve passthrough batch quota");
let passthrough_task = insert_anonymous_settlement_fixture(
&pool,
AnonymousSettlementFixture {
session_id: &passthrough_session,
ip: passthrough_ip,
date: passthrough_date,
reserved_units: 3,
compression_rate: Some(100),
total_files: 3,
completed_files: 3,
},
)
.await;
cleanup.push((
passthrough_task,
passthrough_session.clone(),
passthrough_ip,
passthrough_date,
));
assert_eq!(
settle_anonymous_task_reservation(&state, passthrough_task)
.await
.expect("settle passthrough batch"),
Some(3)
);
assert_eq!(
anonymous_quota_counts(
&state,
&passthrough_session,
passthrough_ip,
passthrough_date,
)
.await,
(0, 0)
);
let partial_session = format!("quota-partial-{marker}");
let partial_ip: IpAddr = "198.51.100.13".parse().expect("parse fixture IP");
let partial_date = reserve_anonymous_units(&state, &partial_session, partial_ip, 3)
.await
.expect("reserve partial batch quota");
let partial_task = insert_anonymous_settlement_fixture(
&pool,
AnonymousSettlementFixture {
session_id: &partial_session,
ip: partial_ip,
date: partial_date,
reserved_units: 3,
compression_rate: None,
total_files: 3,
completed_files: 2,
},
)
.await;
cleanup.push((
partial_task,
partial_session.clone(),
partial_ip,
partial_date,
));
assert_eq!(
settle_anonymous_task_reservation(&state, partial_task)
.await
.expect("settle partial batch"),
Some(1)
);
assert_eq!(
anonymous_quota_counts(&state, &partial_session, partial_ip, partial_date).await,
(2, 2)
);
let limit_session = format!("quota-limit-{marker}");
let limit_ip: IpAddr = "198.51.100.14".parse().expect("parse fixture IP");
let mut limit_date = None;
for batch in 0..2 {
let date = reserve_anonymous_units(&state, &limit_session, limit_ip, 5)
.await
.expect("reserve consecutive anonymous batch");
limit_date = Some(date);
let task_id = insert_anonymous_settlement_fixture(
&pool,
AnonymousSettlementFixture {
session_id: &limit_session,
ip: limit_ip,
date,
reserved_units: 5,
compression_rate: None,
total_files: 5,
completed_files: 5,
},
)
.await;
cleanup.push((task_id, limit_session.clone(), limit_ip, date));
assert_eq!(
settle_anonymous_task_reservation(&state, task_id)
.await
.expect("settle consecutive anonymous batch"),
Some(0),
"batch {batch} unexpectedly refunded consumed units"
);
}
let limit_error = reserve_anonymous_units(&state, &limit_session, limit_ip, 1)
.await
.expect_err("daily anonymous quota was bypassed");
assert_eq!(limit_error.code, ErrorCode::QuotaExceeded);
assert_eq!(
anonymous_quota_counts(
&state,
&limit_session,
limit_ip,
limit_date.expect("limit quota date"),
)
.await,
(10, 10)
);
let mut redis = state.redis.clone();
for (task_id, session_id, ip, date) in cleanup {
sqlx::query("DELETE FROM tasks WHERE id = $1")
.bind(task_id)
.execute(&pool)
.await
.expect("delete quota settlement fixture");
let _: i64 = redis::cmd("DEL")
.arg(anonymous_session_key(&session_id, date))
.arg(anonymous_ip_key(ip, date))
.arg(format!("anon_quota_refund:{task_id}"))
.query_async(&mut redis)
.await
.expect("delete quota settlement Redis keys");
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires isolated IMAGEFORGE_TEST_DATABASE_URL and IMAGEFORGE_TEST_REDIS_URL"]
async fn anonymous_single_reservations_charge_actual_work_and_refund_original_date() {
let database_url = std::env::var("IMAGEFORGE_TEST_DATABASE_URL")
.expect("IMAGEFORGE_TEST_DATABASE_URL must be set");
assert!(
database_url.to_ascii_lowercase().contains("test"),
"refusing to run destructive integration test outside a test database"
);
let redis_url = std::env::var("IMAGEFORGE_TEST_REDIS_URL")
.expect("IMAGEFORGE_TEST_REDIS_URL must be set");
let pool = PgPoolOptions::new()
.max_connections(16)
.connect(&database_url)
.await
.expect("connect anonymous single test database");
sqlx::migrate!().run(&pool).await.expect("run migrations");
let state = build_test_state(pool.clone(), database_url, redis_url).await;
let marker = Uuid::new_v4().simple().to_string();
let current_date = utc8_date();
let previous_date = current_date.pred_opt().expect("previous quota date");
let mut cleanup = Vec::new();
async fn mark_and_finalize(
state: &AppState,
task_id: Uuid,
charged: bool,
) -> Result<(), AppError> {
let mut tx = state.db.begin().await.map_err(|err| {
AppError::new(ErrorCode::Internal, "begin anonymous single test").with_source(err)
})?;
mark_anonymous_single_result(&mut tx, task_id, charged).await?;
tx.commit().await.map_err(|err| {
AppError::new(ErrorCode::Internal, "commit anonymous single test").with_source(err)
})?;
finalize_anonymous_single_reservation(state, task_id, charged).await
}
let passthrough_task = Uuid::new_v4();
let passthrough_session = format!("single-passthrough-{marker}");
let passthrough_ip: IpAddr = "198.51.100.31".parse().expect("parse passthrough IP");
reserve_anonymous_single_unit_for_date(
&state,
passthrough_task,
&passthrough_session,
passthrough_ip,
current_date,
)
.await
.expect("reserve passthrough unit");
let passthrough_charged = output_consumes_unit(Some(100), true, false, false, 100, 50);
assert!(!passthrough_charged);
mark_and_finalize(&state, passthrough_task, passthrough_charged)
.await
.expect("refund passthrough unit");
assert_eq!(
anonymous_quota_counts(&state, &passthrough_session, passthrough_ip, current_date)
.await,
(0, 0)
);
cleanup.push((
passthrough_task,
passthrough_session,
passthrough_ip,
current_date,
));
let unchanged_task = Uuid::new_v4();
let unchanged_session = format!("single-unchanged-{marker}");
let unchanged_ip: IpAddr = "198.51.100.32".parse().expect("parse unchanged IP");
reserve_anonymous_single_unit_for_date(
&state,
unchanged_task,
&unchanged_session,
unchanged_ip,
current_date,
)
.await
.expect("reserve unchanged-output unit");
let unchanged_charged = output_consumes_unit(None, true, false, false, 100, 100);
assert!(!unchanged_charged);
mark_and_finalize(&state, unchanged_task, unchanged_charged)
.await
.expect("refund unchanged-output unit");
assert_eq!(
anonymous_quota_counts(&state, &unchanged_session, unchanged_ip, current_date).await,
(0, 0)
);
cleanup.push((
unchanged_task,
unchanged_session,
unchanged_ip,
current_date,
));
let compressed_task = Uuid::new_v4();
let compressed_session = format!("single-compressed-{marker}");
let compressed_ip: IpAddr = "198.51.100.33".parse().expect("parse compressed IP");
reserve_anonymous_single_unit_for_date(
&state,
compressed_task,
&compressed_session,
compressed_ip,
current_date,
)
.await
.expect("reserve compressed unit");
let compressed_charged = output_consumes_unit(None, true, false, false, 100, 50);
assert!(compressed_charged);
mark_and_finalize(&state, compressed_task, compressed_charged)
.await
.expect("finalize compressed unit");
assert_eq!(
anonymous_quota_counts(&state, &compressed_session, compressed_ip, current_date).await,
(1, 1)
);
let charged_status: String = sqlx::query_scalar(
"SELECT status FROM anonymous_single_reservations WHERE task_id = $1",
)
.bind(compressed_task)
.fetch_one(&pool)
.await
.expect("query charged reservation");
assert_eq!(charged_status, "charged");
cleanup.push((
compressed_task,
compressed_session,
compressed_ip,
current_date,
));
let cross_day_task = Uuid::new_v4();
let cross_day_session = format!("single-cross-day-{marker}");
let cross_day_ip: IpAddr = "198.51.100.34".parse().expect("parse cross-day IP");
reserve_anonymous_units(&state, &cross_day_session, cross_day_ip, 2)
.await
.expect("seed current-day quota");
reserve_anonymous_single_unit_for_date(
&state,
cross_day_task,
&cross_day_session,
cross_day_ip,
previous_date,
)
.await
.expect("reserve previous-day unit");
refund_anonymous_single_reservation(&state, cross_day_task)
.await
.expect("refund previous-day failure");
assert_eq!(
anonymous_quota_counts(&state, &cross_day_session, cross_day_ip, previous_date).await,
(0, 0)
);
assert_eq!(
anonymous_quota_counts(&state, &cross_day_session, cross_day_ip, current_date).await,
(2, 2),
"cross-day refund changed the current quota bucket"
);
cleanup.push((
cross_day_task,
cross_day_session.clone(),
cross_day_ip,
previous_date,
));
let interrupted_task = Uuid::new_v4();
let interrupted_session = format!("single-interrupted-{marker}");
let interrupted_ip: IpAddr = "198.51.100.35".parse().expect("parse interrupted IP");
reserve_anonymous_single_unit_for_date(
&state,
interrupted_task,
&interrupted_session,
interrupted_ip,
current_date,
)
.await
.expect("reserve interrupted unit");
sqlx::query(
"UPDATE anonymous_single_reservations SET refund_after = NOW() - INTERVAL '1 second' WHERE task_id = $1",
)
.bind(interrupted_task)
.execute(&pool)
.await
.expect("expire interrupted reservation");
assert_eq!(
settle_stale_anonymous_single_reservations(&state, 10)
.await
.expect("settle interrupted reservation"),
1
);
assert_eq!(
anonymous_quota_counts(&state, &interrupted_session, interrupted_ip, current_date)
.await,
(0, 0)
);
cleanup.push((
interrupted_task,
interrupted_session,
interrupted_ip,
current_date,
));
let mut redis = state.redis.clone();
for (task_id, session_id, ip, date) in cleanup {
sqlx::query("DELETE FROM anonymous_single_reservations WHERE task_id = $1")
.bind(task_id)
.execute(&pool)
.await
.expect("delete anonymous single reservation");
let _: i64 = redis::cmd("DEL")
.arg(anonymous_session_key(&session_id, date))
.arg(anonymous_ip_key(ip, date))
.arg(anonymous_single_reservation_key(task_id))
.arg(format!("anon_quota_refund:{task_id}"))
.query_async(&mut redis)
.await
.expect("delete anonymous single Redis keys");
}
let _: i64 = redis::cmd("DEL")
.arg(anonymous_session_key(&cross_day_session, current_date))
.arg(anonymous_ip_key(cross_day_ip, current_date))
.query_async(&mut redis)
.await
.expect("delete cross-day current Redis keys");
}
}

View File

@@ -291,10 +291,30 @@ pub fn result_key(retention_hours: i64, task_id: Uuid, file_id: Uuid, extension:
)
}
pub fn archive_key(retention_hours: i64, task_id: Uuid) -> String {
pub fn result_attempt_key(
retention_hours: i64,
task_id: Uuid,
file_id: Uuid,
task_attempt: i64,
file_attempt: i64,
extension: &str,
) -> String {
let now = Utc::now();
format!(
"archives/{}/{:04}/{:02}/{task_id}.zip",
"results/{}/{:04}/{:02}/{task_id}/{file_id}-t{}-f{}.{}",
retention_prefix(retention_hours),
now.year(),
now.month(),
task_attempt.max(1),
file_attempt.max(1),
extension.trim_start_matches('.')
)
}
pub fn archive_attempt_key(retention_hours: i64, task_id: Uuid, token: Uuid) -> String {
let now = Utc::now();
format!(
"archives/{}/{:04}/{:02}/attempts/{task_id}/{token}.zip",
retention_prefix(retention_hours),
now.year(),
now.month()
@@ -310,27 +330,7 @@ fn retention_prefix(hours: i64) -> String {
}
}
pub async fn store_bytes<B>(
state: &AppState,
key: &str,
bytes: B,
content_type: &str,
) -> Result<StoredObject, AppError>
where
B: Into<Bytes>,
{
let bytes = bytes.into();
if let Some(endpoint) = active_endpoint(state).await? {
match store_bytes_s3(state, &endpoint, key, bytes.clone(), content_type).await {
Ok(stored) => return Ok(stored),
Err(err) => log_local_fallback(state, &endpoint, key, &err),
}
}
store_bytes_local(state, key, bytes.as_ref()).await
}
async fn store_bytes_s3(
pub(crate) async fn store_bytes_s3(
state: &AppState,
endpoint: &StorageEndpoint,
key: &str,
@@ -358,7 +358,7 @@ async fn store_bytes_s3(
})
}
async fn store_bytes_local(
pub(crate) async fn store_bytes_local(
state: &AppState,
key: &str,
bytes: &[u8],
@@ -382,27 +382,7 @@ async fn store_bytes_local(
})
}
pub async fn store_file(
state: &AppState,
key: &str,
path: &Path,
content_type: &str,
) -> Result<StoredObject, AppError> {
let metadata = tokio::fs::metadata(path).await.map_err(|err| {
AppError::new(ErrorCode::StorageUnavailable, "读取待上传文件失败").with_source(err)
})?;
if let Some(endpoint) = active_endpoint(state).await? {
match store_file_s3(state, &endpoint, key, path, content_type, metadata.len()).await {
Ok(stored) => return Ok(stored),
Err(err) => log_local_fallback(state, &endpoint, key, &err),
}
}
store_file_local(state, key, path, metadata.len()).await
}
async fn store_file_s3(
pub(crate) async fn store_file_s3(
state: &AppState,
endpoint: &StorageEndpoint,
key: &str,
@@ -439,7 +419,7 @@ async fn store_file_s3(
})
}
async fn store_file_local(
pub(crate) async fn store_file_local(
state: &AppState,
key: &str,
path: &Path,
@@ -464,7 +444,12 @@ async fn store_file_local(
})
}
fn log_local_fallback(state: &AppState, endpoint: &StorageEndpoint, key: &str, err: &AppError) {
pub(crate) fn log_local_fallback(
state: &AppState,
endpoint: &StorageEndpoint,
key: &str,
err: &AppError,
) {
crate::services::metrics::record_storage_fallback(state);
tracing::warn!(
storage_endpoint_id = %endpoint.id,
@@ -796,7 +781,7 @@ async fn endpoint_for_object(
get_endpoint(state, endpoint_id).await
}
fn local_path(state: &AppState, key: &str) -> Result<PathBuf, AppError> {
pub(crate) fn local_path(state: &AppState, key: &str) -> Result<PathBuf, AppError> {
if key.is_empty()
|| key.starts_with('/')
|| key.starts_with('\\')
@@ -963,7 +948,9 @@ mod tests {
let key = result_key(168, task_id, file_id, "webp");
assert!(key.starts_with("results/7d/"));
assert!(key.ends_with("/00000000-0000-0000-0000-000000000001.webp"));
assert!(archive_key(360, task_id).starts_with("archives/15d/"));
assert!(archive_attempt_key(360, task_id, Uuid::new_v4()).starts_with("archives/15d/"));
let attempt_key = result_attempt_key(24, task_id, file_id, 2, 3, "avif");
assert!(attempt_key.contains("-t2-f3.avif"));
}
#[test]

367
src/services/task_queue.rs Normal file
View File

@@ -0,0 +1,367 @@
use crate::error::{AppError, ErrorCode};
use crate::services::{metrics, object_lifecycle, quota};
use crate::state::AppState;
use chrono::Utc;
use sqlx::FromRow;
use std::time::Duration;
use uuid::Uuid;
const DISPATCH_INTERVAL: Duration = Duration::from_secs(1);
const DISPATCH_BATCH_SIZE: i64 = 50;
const DELIVERY_LEASE_SECONDS: i64 = 30;
const MAX_DELIVERY_ATTEMPTS: i32 = 20;
const MAX_RETRY_SECONDS: u64 = 60;
#[derive(Debug, FromRow)]
struct OutboxClaim {
task_id: Uuid,
attempts: i32,
}
pub async fn dispatch_loop(state: AppState) {
let mut interval = tokio::time::interval(DISPATCH_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
interval.tick().await;
if let Err(err) = dispatch_ready(&state, DISPATCH_BATCH_SIZE).await {
tracing::error!(error = %err, "task queue outbox dispatch iteration failed");
}
}
}
pub async fn dispatch_ready(state: &AppState, limit: i64) -> Result<usize, AppError> {
reconcile_non_pending_tasks(state).await?;
let lease_owner = Uuid::new_v4();
let claims = claim_ready(state, lease_owner, limit.max(1), None).await?;
let count = claims.len();
for claim in claims {
dispatch_claim(state, lease_owner, claim).await?;
}
Ok(count)
}
pub async fn dispatch_task(state: &AppState, task_id: Uuid) -> Result<bool, AppError> {
let lease_owner = Uuid::new_v4();
let mut claims = claim_ready(state, lease_owner, 1, Some(task_id)).await?;
let Some(claim) = claims.pop() else {
return Ok(false);
};
dispatch_claim(state, lease_owner, claim).await?;
Ok(true)
}
async fn claim_ready(
state: &AppState,
lease_owner: Uuid,
limit: i64,
task_id: Option<Uuid>,
) -> Result<Vec<OutboxClaim>, AppError> {
sqlx::query_as::<_, OutboxClaim>(
r#"
WITH candidate AS (
SELECT outbox.task_id
FROM task_queue_outbox AS outbox
JOIN tasks AS task ON task.id = outbox.task_id
WHERE outbox.status IN ('pending', 'delivering')
AND outbox.next_attempt_at <= NOW()
AND (outbox.lease_until IS NULL OR outbox.lease_until <= NOW())
AND task.status = 'pending'
AND task.deletion_started_at IS NULL
AND ($3::uuid IS NULL OR outbox.task_id = $3)
ORDER BY outbox.next_attempt_at, outbox.created_at
FOR UPDATE OF outbox SKIP LOCKED
LIMIT $2
)
UPDATE task_queue_outbox AS outbox
SET status = 'delivering',
attempts = outbox.attempts + 1,
lease_owner = $1,
lease_until = NOW() + ($4 * INTERVAL '1 second'),
updated_at = NOW()
FROM candidate
WHERE outbox.task_id = candidate.task_id
RETURNING outbox.task_id, outbox.attempts
"#,
)
.bind(lease_owner)
.bind(limit)
.bind(task_id)
.bind(DELIVERY_LEASE_SECONDS)
.fetch_all(&state.db)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "领取任务队列 outbox 失败").with_source(err))
}
async fn dispatch_claim(
state: &AppState,
lease_owner: Uuid,
claim: OutboxClaim,
) -> Result<(), AppError> {
match enqueue_task(state, claim.task_id).await {
Ok(()) => mark_delivered(state, claim.task_id, lease_owner).await,
Err(err) if claim.attempts >= MAX_DELIVERY_ATTEMPTS => {
dead_letter_pending_task(state, claim.task_id, lease_owner, &err).await
}
Err(err) => release_for_retry(state, claim, lease_owner, &err).await,
}
}
pub(crate) async fn enqueue_task(state: &AppState, task_id: Uuid) -> Result<(), AppError> {
let mut connection = state.redis.clone();
redis::cmd("XADD")
.arg(metrics::QUEUE_STREAM_KEY)
.arg("MAXLEN")
.arg("~")
.arg(100_000)
.arg("*")
.arg("task_id")
.arg(task_id.to_string())
.arg("created_at")
.arg(Utc::now().to_rfc3339())
.query_async::<_, redis::Value>(&mut connection)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "写入队列失败").with_source(err))?;
Ok(())
}
async fn mark_delivered(
state: &AppState,
task_id: Uuid,
lease_owner: Uuid,
) -> Result<(), AppError> {
sqlx::query(
r#"
UPDATE task_queue_outbox
SET status = 'delivered',
delivered_at = COALESCE(delivered_at, NOW()),
lease_owner = NULL,
lease_until = NULL,
last_error = NULL,
updated_at = NOW()
WHERE task_id = $1
AND status = 'delivering'
AND lease_owner = $2
"#,
)
.bind(task_id)
.bind(lease_owner)
.execute(&state.db)
.await
.map_err(|err| {
AppError::new(ErrorCode::Internal, "完成任务队列 outbox 失败").with_source(err)
})?;
Ok(())
}
async fn release_for_retry(
state: &AppState,
claim: OutboxClaim,
lease_owner: Uuid,
error: &AppError,
) -> Result<(), AppError> {
let delay = retry_delay(claim.attempts);
let message = truncate_error(error);
sqlx::query(
r#"
UPDATE task_queue_outbox
SET status = 'pending',
next_attempt_at = NOW() + ($3 * INTERVAL '1 second'),
lease_owner = NULL,
lease_until = NULL,
last_error = $4,
updated_at = NOW()
WHERE task_id = $1
AND status = 'delivering'
AND lease_owner = $2
"#,
)
.bind(claim.task_id)
.bind(lease_owner)
.bind(delay.as_secs() as i64)
.bind(&message)
.execute(&state.db)
.await
.map_err(|err| AppError::new(ErrorCode::Internal, "记录任务队列重试失败").with_source(err))?;
tracing::warn!(task_id = %claim.task_id, attempts = claim.attempts, retry_seconds = delay.as_secs(), error = %error, "task queue delivery deferred");
Ok(())
}
async fn dead_letter_pending_task(
state: &AppState,
task_id: Uuid,
lease_owner: Uuid,
error: &AppError,
) -> Result<(), AppError> {
let message = format!("队列持续不可用:{}", truncate_error(error));
let input_dir = std::path::PathBuf::from(&state.config.storage_path)
.join("orig")
.join(task_id.to_string())
.to_string_lossy()
.to_string();
let mut tx = state.db.begin().await.map_err(|err| {
AppError::new(ErrorCode::Internal, "开启 outbox 死信事务失败").with_source(err)
})?;
let task: Option<String> =
sqlx::query_scalar("SELECT status::text FROM tasks WHERE id = $1 FOR UPDATE")
.bind(task_id)
.fetch_optional(&mut *tx)
.await
.map_err(|err| {
AppError::new(ErrorCode::Internal, "锁定 outbox 死信任务失败").with_source(err)
})?;
let Some(status) = task else {
tx.rollback().await.ok();
return Ok(());
};
let owned: Option<Uuid> = sqlx::query_scalar(
r#"
SELECT task_id
FROM task_queue_outbox
WHERE task_id = $1
AND status = 'delivering'
AND lease_owner = $2
FOR UPDATE
"#,
)
.bind(task_id)
.bind(lease_owner)
.fetch_optional(&mut *tx)
.await
.map_err(|err| {
AppError::new(ErrorCode::Internal, "校验 outbox 死信租约失败").with_source(err)
})?;
if owned.is_none() {
tx.rollback().await.ok();
return Ok(());
}
let failed = status == "pending";
if failed {
sqlx::query(
r#"
UPDATE task_files
SET status = 'failed',
error_message = $2,
completed_at = NOW(),
input_path = NULL,
lease_owner = NULL,
lease_until = NULL
WHERE task_id = $1
AND status IN ('pending', 'processing')
"#,
)
.bind(task_id)
.bind(&message)
.execute(&mut *tx)
.await
.map_err(|err| {
AppError::new(ErrorCode::Internal, "收口 outbox 死信文件失败").with_source(err)
})?;
sqlx::query(
r#"
INSERT INTO storage_objects (
task_id, object_kind, state, backend, object_key
) VALUES ($1, 'input_dir', 'delete_pending', 'local_dir', $2)
ON CONFLICT DO NOTHING
"#,
)
.bind(task_id)
.bind(&input_dir)
.execute(&mut *tx)
.await
.map_err(|err| {
AppError::new(ErrorCode::Internal, "安排 outbox 死信输入清理失败").with_source(err)
})?;
sqlx::query(
r#"
UPDATE tasks
SET status = 'failed',
completed_files = 0,
failed_files = total_files,
error_message = $2,
completed_at = NOW(),
lease_owner = NULL,
lease_until = NULL
WHERE id = $1 AND status = 'pending'
"#,
)
.bind(task_id)
.bind(&message)
.execute(&mut *tx)
.await
.map_err(|err| {
AppError::new(ErrorCode::Internal, "收口 outbox 死信任务失败").with_source(err)
})?;
}
sqlx::query(
r#"
UPDATE task_queue_outbox
SET status = CASE WHEN $3 THEN 'dead' ELSE 'delivered' END,
delivered_at = CASE WHEN $3 THEN delivered_at ELSE COALESCE(delivered_at, NOW()) END,
lease_owner = NULL,
lease_until = NULL,
last_error = $4,
updated_at = NOW()
WHERE task_id = $1
AND lease_owner = $2
"#,
)
.bind(task_id)
.bind(lease_owner)
.bind(failed)
.bind(&message)
.execute(&mut *tx)
.await
.map_err(|err| {
AppError::new(ErrorCode::Internal, "提交 outbox 死信状态失败").with_source(err)
})?;
tx.commit().await.map_err(|err| {
AppError::new(ErrorCode::Internal, "提交 outbox 死信事务失败").with_source(err)
})?;
if failed {
metrics::record_dead_letter(state);
if let Err(err) = object_lifecycle::cleanup_ready_objects(state, 10, Some(task_id)).await {
tracing::warn!(task_id = %task_id, error = %err, "dead outbox input cleanup deferred");
}
quota::settle_anonymous_task_reservation(state, task_id).await?;
}
Ok(())
}
async fn reconcile_non_pending_tasks(state: &AppState) -> Result<(), AppError> {
sqlx::query(
r#"
UPDATE task_queue_outbox AS outbox
SET status = 'delivered',
delivered_at = COALESCE(outbox.delivered_at, NOW()),
lease_owner = NULL,
lease_until = NULL,
updated_at = NOW()
FROM tasks AS task
WHERE task.id = outbox.task_id
AND task.status <> 'pending'
AND outbox.status IN ('pending', 'delivering')
"#,
)
.execute(&state.db)
.await
.map_err(|err| {
AppError::new(ErrorCode::Internal, "对账任务队列 outbox 失败").with_source(err)
})?;
Ok(())
}
fn retry_delay(attempts: i32) -> Duration {
let exponent = attempts.saturating_sub(1).min(6) as u32;
Duration::from_secs(2_u64.saturating_pow(exponent).min(MAX_RETRY_SECONDS))
}
fn truncate_error(error: &AppError) -> String {
format!("{}: {}", error.code.as_str(), error.message)
.chars()
.take(2_000)
.collect()
}

View File

@@ -10,6 +10,7 @@ pub struct AppState {
pub redis: redis::aio::ConnectionManager,
pub mailer: std::sync::Arc<Mailer>,
pub image_processing_semaphore: std::sync::Arc<tokio::sync::Semaphore>,
pub zip_build_semaphore: std::sync::Arc<tokio::sync::Semaphore>,
pub runtime_policy_cache: RuntimePolicyCache,
pub storage_cache: StorageCache,
}

File diff suppressed because it is too large Load Diff