feat: improve upload resilience and release 0.1.23

2026-02-20 20:21:42 +08:00
parent 6618de1aed
commit 01384a2215
9 changed files with 435 additions and 115 deletions
--- a/backend/database.js
+++ b/backend/database.js
@@ -2512,8 +2512,9 @@ const UploadSessionDB = {
    return db.prepare('SELECT * FROM upload_sessions WHERE session_id = ?').get(sid);
  },

-  findActiveForResume(userId, targetPath, fileSize, fileHash = null) {
+  findActiveForResume(userId, storageType, targetPath, fileSize, fileHash = null) {
    const uid = Number(userId);
+    const safeStorageType = storageType === 'local' ? 'local' : 'oss';
    const safePath = typeof targetPath === 'string' ? targetPath : '';
    const size = Math.floor(Number(fileSize) || 0);
    const hash = typeof fileHash === 'string' ? fileHash.trim() : '';
@@ -2526,6 +2527,7 @@ const UploadSessionDB = {
        SELECT *
        FROM upload_sessions
        WHERE user_id = ?
+          AND storage_type = ?
          AND target_path = ?
          AND file_size = ?
          AND status = 'active'
@@ -2533,20 +2535,21 @@ const UploadSessionDB = {
          AND COALESCE(file_hash, '') = ?
        ORDER BY updated_at DESC, created_at DESC
        LIMIT 1
-      `).get(Math.floor(uid), safePath, size, hash);
+      `).get(Math.floor(uid), safeStorageType, safePath, size, hash);
    }

    return db.prepare(`
      SELECT *
      FROM upload_sessions
      WHERE user_id = ?
+        AND storage_type = ?
        AND target_path = ?
        AND file_size = ?
        AND status = 'active'
        AND expires_at > datetime('now', 'localtime')
      ORDER BY updated_at DESC, created_at DESC
      LIMIT 1
-    `).get(Math.floor(uid), safePath, size);
+    `).get(Math.floor(uid), safeStorageType, safePath, size);
  },

  updateProgress(sessionId, uploadedChunks = [], uploadedBytes = 0, expiresAt = null) {
--- a/backend/server.js
+++ b/backend/server.js
@@ -112,7 +112,7 @@ const DOWNLOAD_SIGNED_URL_EXPIRES_SECONDS = Math.max(
  10,
  Math.min(3600, Number(process.env.DOWNLOAD_SIGNED_URL_EXPIRES_SECONDS || 30))
 );
-const DEFAULT_DESKTOP_VERSION = process.env.DESKTOP_LATEST_VERSION || '0.1.22';
+const DEFAULT_DESKTOP_VERSION = process.env.DESKTOP_LATEST_VERSION || '0.1.23';
 const DEFAULT_DESKTOP_INSTALLER_URL = process.env.DESKTOP_INSTALLER_URL || '';
 const DEFAULT_DESKTOP_RELEASE_NOTES = process.env.DESKTOP_RELEASE_NOTES || '';
 const DESKTOP_INSTALLERS_DIR = path.resolve(__dirname, '../frontend/downloads');
@@ -2332,9 +2332,13 @@ function logSecurity(req, action, message, details = null, level = 'warn') {
 }

 // 文件上传配置（临时存储）
+const MULTER_UPLOAD_MAX_BYTES = Math.max(
+  1 * 1024 * 1024,
+  Number(process.env.MULTER_UPLOAD_MAX_BYTES || (50 * 1024 * 1024 * 1024))
+);
 const upload = multer({
  dest: path.join(__dirname, 'uploads'),
-  limits: { fileSize: 5 * 1024 * 1024 * 1024 } // 5GB限制
+  limits: { fileSize: MULTER_UPLOAD_MAX_BYTES }
 });

 // ===== TTL缓存类 =====
@@ -5949,11 +5953,15 @@ app.post('/api/files/instant-upload/check', authMiddleware, async (req, res) =>
 // 本地分片上传初始化（断点续传）
 app.post('/api/upload/resumable/init', authMiddleware, async (req, res) => {
  try {
-    if ((req.user.current_storage_type || 'oss') !== 'local') {
-      return res.status(400).json({
-        success: false,
-        message: '当前存储模式不支持分片上传'
-      });
+    const uploadStorageType = (req.user.current_storage_type || 'oss') === 'local' ? 'local' : 'oss';
+    if (uploadStorageType === 'oss') {
+      const hasUnifiedConfig = SettingsDB.hasUnifiedOssConfig();
+      if (!req.user.has_oss_config && !hasUnifiedConfig) {
+        return res.status(400).json({
+          success: false,
+          message: '未配置 OSS 服务，无法上传文件'
+        });
+      }
    }

    const filename = typeof req.body?.filename === 'string' ? req.body.filename.trim() : '';
@@ -6013,6 +6021,7 @@ app.post('/api/upload/resumable/init', authMiddleware, async (req, res) => {
    const totalChunks = Math.ceil(fileSize / chunkSize);
    const existingSession = UploadSessionDB.findActiveForResume(
      req.user.id,
+      uploadStorageType,
      targetPath,
      fileSize,
      fileHash || null
@@ -6038,6 +6047,7 @@ app.post('/api/upload/resumable/init', authMiddleware, async (req, res) => {
        return res.json({
          success: true,
          resumed: true,
+          storage_type: existingSession.storage_type || uploadStorageType,
          session_id: existingSession.session_id,
          target_path: existingSession.target_path,
          file_name: existingSession.file_name,
@@ -6061,7 +6071,7 @@ app.post('/api/upload/resumable/init', authMiddleware, async (req, res) => {
    const createdSession = UploadSessionDB.create({
      sessionId,
      userId: req.user.id,
-      storageType: 'local',
+      storageType: uploadStorageType,
      targetPath,
      fileName: filename,
      fileSize,
@@ -6085,6 +6095,7 @@ app.post('/api/upload/resumable/init', authMiddleware, async (req, res) => {
    res.json({
      success: true,
      resumed: false,
+      storage_type: createdSession.storage_type || uploadStorageType,
      session_id: createdSession.session_id,
      target_path: createdSession.target_path,
      file_name: createdSession.file_name,
@@ -6154,14 +6165,6 @@ app.get('/api/upload/resumable/status', authMiddleware, (req, res) => {
 app.post('/api/upload/resumable/chunk', authMiddleware, upload.single('chunk'), async (req, res) => {
  const tempChunkPath = req.file?.path;
  try {
-    if ((req.user.current_storage_type || 'oss') !== 'local') {
-      if (tempChunkPath) safeDeleteFile(tempChunkPath);
-      return res.status(400).json({
-        success: false,
-        message: '当前存储模式不支持分片上传'
-      });
-    }
-
    const sessionId = typeof req.body?.session_id === 'string' ? req.body.session_id.trim() : '';
    const chunkIndex = Math.floor(Number(req.body?.chunk_index));
    if (!sessionId) {
@@ -6287,16 +6290,10 @@ app.post('/api/upload/resumable/chunk', authMiddleware, upload.single('chunk'),
  }
 });

-// 完成分片上传（写入本地存储）
+// 完成分片上传（写入当前存储：本地或 OSS）
 app.post('/api/upload/resumable/complete', authMiddleware, async (req, res) => {
+  let storage = null;
  try {
-    if ((req.user.current_storage_type || 'oss') !== 'local') {
-      return res.status(400).json({
-        success: false,
-        message: '当前存储模式不支持分片上传'
-      });
-    }
-
    const sessionId = typeof req.body?.session_id === 'string' ? req.body.session_id.trim() : '';
    if (!sessionId) {
      return res.status(400).json({
@@ -6345,6 +6342,8 @@ app.post('/api/upload/resumable/complete', authMiddleware, async (req, res) => {
      });
    }

+    const sessionStorageType = session.storage_type === 'local' ? 'local' : 'oss';
+
    const latestUser = UserDB.findById(req.user.id);
    if (!latestUser) {
      return res.status(404).json({
@@ -6353,12 +6352,25 @@ app.post('/api/upload/resumable/complete', authMiddleware, async (req, res) => {
      });
    }

-    const localUserContext = buildStorageUserContext(latestUser, {
-      current_storage_type: 'local'
+    if (sessionStorageType === 'oss') {
+      const hasUnifiedConfig = SettingsDB.hasUnifiedOssConfig();
+      if (!latestUser.has_oss_config && !hasUnifiedConfig) {
+        return res.status(400).json({
+          success: false,
+          message: '未配置 OSS 服务，无法完成分片上传'
+        });
+      }
+    }
+
+    const storageUserContext = buildStorageUserContext(latestUser, {
+      current_storage_type: sessionStorageType
    });
-    const localStorage = new LocalStorageClient(localUserContext);
-    await localStorage.init();
-    await localStorage.put(session.temp_file_path, session.target_path);
+    const storageInterface = new StorageInterface(storageUserContext);
+    storage = await storageInterface.connect();
+    await storage.put(session.temp_file_path, session.target_path);
+    if (sessionStorageType === 'oss') {
+      clearOssUsageCache(req.user.id);
+    }

    UploadSessionDB.setStatus(session.session_id, 'completed', {
      completed: true,
@@ -6368,7 +6380,7 @@ app.post('/api/upload/resumable/complete', authMiddleware, async (req, res) => {

    await trackFileHashIndexForUpload({
      userId: req.user.id,
-      storageType: 'local',
+      storageType: sessionStorageType,
      fileHash: session.file_hash,
      fileSize,
      filePath: session.target_path
@@ -6387,6 +6399,10 @@ app.post('/api/upload/resumable/complete', authMiddleware, async (req, res) => {
      success: false,
      message: getSafeErrorMessage(error, '完成分片上传失败，请稍后重试', '完成分片上传')
    });
+  } finally {
+    if (storage) {
+      await storage.end().catch(() => {});
+    }
  }
 });

--- a/backend/storage.js
+++ b/backend/storage.js
@@ -1,4 +1,16 @@
-const { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectsCommand, ListObjectsV2Command, HeadObjectCommand, CopyObjectCommand } = require('@aws-sdk/client-s3');
+const {
+  S3Client,
+  PutObjectCommand,
+  GetObjectCommand,
+  DeleteObjectsCommand,
+  ListObjectsV2Command,
+  HeadObjectCommand,
+  CopyObjectCommand,
+  CreateMultipartUploadCommand,
+  UploadPartCommand,
+  CompleteMultipartUploadCommand,
+  AbortMultipartUploadCommand
+} = require('@aws-sdk/client-s3');
 const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
 const fs = require('fs');
 const path = require('path');
@@ -18,6 +30,25 @@ function formatFileSize(bytes) {
  return Math.round(bytes / Math.pow(k, i) * 100) / 100 + ' ' + sizes[i];
 }

+function parsePositiveInteger(value, fallback) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    return fallback;
+  }
+  return Math.floor(parsed);
+}
+
+const OSS_MULTIPART_MAX_PARTS = 10000;
+const OSS_MULTIPART_MIN_PART_SIZE = 5 * 1024 * 1024; // S3 协议要求最小 5MB（最后一片除外）
+const OSS_MULTIPART_DEFAULT_PART_SIZE = Math.max(
+  OSS_MULTIPART_MIN_PART_SIZE,
+  parsePositiveInteger(process.env.OSS_MULTIPART_PART_SIZE_BYTES, 16 * 1024 * 1024)
+);
+const OSS_SINGLE_UPLOAD_THRESHOLD = Math.max(
+  OSS_MULTIPART_MIN_PART_SIZE,
+  parsePositiveInteger(process.env.OSS_SINGLE_UPLOAD_THRESHOLD_BYTES, 64 * 1024 * 1024)
+);
+
 /**
 * 将 OSS/网络错误转换为友好的错误信息
 * @param {Error} error - 原始错误
@@ -1090,9 +1121,13 @@ class OssStorageClient {
   * @param {string} remotePath - 远程文件路径
   */
  async put(localPath, remotePath) {
+    let bucket = '';
+    let key = '';
+    let uploadId = null;
+    let fileHandle = null;
    try {
-      const key = this.getObjectKey(remotePath);
-      const bucket = this.getBucket();
+      key = this.getObjectKey(remotePath);
+      bucket = this.getBucket();

      // 检查本地文件是否存在
      if (!fs.existsSync(localPath)) {
@@ -1102,30 +1137,107 @@ class OssStorageClient {
      const fileStats = fs.statSync(localPath);
      const fileSize = fileStats.size;

-      // 检查文件大小（AWS S3 单次上传最大 5GB）
-      const MAX_SINGLE_UPLOAD_SIZE = 5 * 1024 * 1024 * 1024; // 5GB
-      if (fileSize > MAX_SINGLE_UPLOAD_SIZE) {
-        throw new Error(`文件过大 (${formatFileSize(fileSize)})，单次上传最大支持 5GB，请使用分片上传`);
+      if (fileSize <= OSS_SINGLE_UPLOAD_THRESHOLD) {
+        // 小文件保持单请求上传，兼容性更高。
+        const fileContent = fs.readFileSync(localPath);
+        const command = new PutObjectCommand({
+          Bucket: bucket,
+          Key: key,
+          Body: fileContent,
+          ContentLength: fileSize,
+          ChecksumAlgorithm: undefined
+        });
+        await this.s3Client.send(command);
+        console.log(`[OSS存储] 上传成功: ${key} (${formatFileSize(fileSize)})`);
+        return;
      }

-      // 使用Buffer上传而非流式上传，避免AWS SDK使用aws-chunked编码
-      // 这样可以确保与阿里云OSS的兼容性
-      const fileContent = fs.readFileSync(localPath);
+      // 大文件自动切换到分片上传，避免整文件读入内存和 5GB 单请求上限。
+      const minPartSizeByCount = Math.ceil(fileSize / OSS_MULTIPART_MAX_PARTS);
+      const alignedMinPartSize = Math.ceil(
+        Math.max(minPartSizeByCount, OSS_MULTIPART_MIN_PART_SIZE) / (1024 * 1024)
+      ) * 1024 * 1024;
+      const partSize = Math.max(OSS_MULTIPART_DEFAULT_PART_SIZE, alignedMinPartSize);
+      const estimatedParts = Math.ceil(fileSize / partSize);
+      if (estimatedParts > OSS_MULTIPART_MAX_PARTS) {
+        throw new Error(`文件过大，分片数量超限 (${estimatedParts}/${OSS_MULTIPART_MAX_PARTS})`);
+      }

-      // 直接上传
-      const command = new PutObjectCommand({
+      const createResp = await this.s3Client.send(new CreateMultipartUploadCommand({
+        Bucket: bucket,
+        Key: key
+      }));
+      uploadId = createResp?.UploadId || null;
+      if (!uploadId) {
+        throw new Error('创建分片上传会话失败：缺少 UploadId');
+      }
+
+      const uploadedParts = [];
+      let offset = 0;
+      let partNumber = 1;
+      fileHandle = await fs.promises.open(localPath, 'r');
+
+      while (offset < fileSize) {
+        const remaining = fileSize - offset;
+        const currentPartSize = Math.min(partSize, remaining);
+        const buffer = Buffer.allocUnsafe(currentPartSize);
+        const { bytesRead } = await fileHandle.read(buffer, 0, currentPartSize, offset);
+        if (bytesRead <= 0) {
+          throw new Error('读取本地文件失败，分片上传中断');
+        }
+
+        const body = bytesRead === currentPartSize ? buffer : buffer.subarray(0, bytesRead);
+        const partResp = await this.s3Client.send(new UploadPartCommand({
+          Bucket: bucket,
+          Key: key,
+          UploadId: uploadId,
+          PartNumber: partNumber,
+          Body: body,
+          ContentLength: bytesRead
+        }));
+
+        if (!partResp?.ETag) {
+          throw new Error(`分片 ${partNumber} 上传失败：服务端未返回 ETag`);
+        }
+        uploadedParts.push({
+          ETag: partResp.ETag,
+          PartNumber: partNumber
+        });
+
+        offset += bytesRead;
+        if (partNumber === 1 || partNumber % 10 === 0 || offset >= fileSize) {
+          console.log(
+            `[OSS存储] 上传分片进度: ${key} ${partNumber}/${estimatedParts} (${formatFileSize(offset)}/${formatFileSize(fileSize)})`
+          );
+        }
+        partNumber += 1;
+      }
+
+      await this.s3Client.send(new CompleteMultipartUploadCommand({
        Bucket: bucket,
        Key: key,
-        Body: fileContent,
-        ContentLength: fileSize, // 明确指定内容长度，避免某些服务端问题
-        // 禁用checksum算法（阿里云OSS不完全支持AWS的x-amz-content-sha256头）
-        ChecksumAlgorithm: undefined
-      });
-
-      await this.s3Client.send(command);
-      console.log(`[OSS存储] 上传成功: ${key} (${formatFileSize(fileSize)})`);
+        UploadId: uploadId,
+        MultipartUpload: { Parts: uploadedParts }
+      }));
+      uploadId = null;
+      console.log(
+        `[OSS存储] 分片上传完成: ${key} (${formatFileSize(fileSize)}, ${uploadedParts.length} 片)`
+      );

    } catch (error) {
+      if (uploadId && bucket && key) {
+        try {
+          await this.s3Client.send(new AbortMultipartUploadCommand({
+            Bucket: bucket,
+            Key: key,
+            UploadId: uploadId
+          }));
+          console.warn(`[OSS存储] 已中止失败的分片上传: ${key}`);
+        } catch (abortError) {
+          console.error(`[OSS存储] 中止分片上传失败: ${key}`, abortError.message);
+        }
+      }
+
      console.error(`[OSS存储] 上传失败: ${remotePath}`, error.message);

      // 判断错误类型并给出友好的错误信息
@@ -1139,6 +1251,10 @@ class OssStorageClient {
        throw new Error(`本地文件不存在: ${localPath}`);
      }
      throw new Error(`文件上传失败: ${error.message}`);
+    } finally {
+      if (fileHandle) {
+        await fileHandle.close().catch(() => {});
+      }
    }
  }