feat: improve upload resilience and release 0.1.23
This commit is contained in:
@@ -1,4 +1,16 @@
|
||||
const { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectsCommand, ListObjectsV2Command, HeadObjectCommand, CopyObjectCommand } = require('@aws-sdk/client-s3');
|
||||
const {
|
||||
S3Client,
|
||||
PutObjectCommand,
|
||||
GetObjectCommand,
|
||||
DeleteObjectsCommand,
|
||||
ListObjectsV2Command,
|
||||
HeadObjectCommand,
|
||||
CopyObjectCommand,
|
||||
CreateMultipartUploadCommand,
|
||||
UploadPartCommand,
|
||||
CompleteMultipartUploadCommand,
|
||||
AbortMultipartUploadCommand
|
||||
} = require('@aws-sdk/client-s3');
|
||||
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
@@ -18,6 +30,25 @@ function formatFileSize(bytes) {
|
||||
return Math.round(bytes / Math.pow(k, i) * 100) / 100 + ' ' + sizes[i];
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value, fallback) {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
return fallback;
|
||||
}
|
||||
return Math.floor(parsed);
|
||||
}
|
||||
|
||||
const OSS_MULTIPART_MAX_PARTS = 10000;
|
||||
const OSS_MULTIPART_MIN_PART_SIZE = 5 * 1024 * 1024; // S3 协议要求最小 5MB（最后一片除外）
|
||||
const OSS_MULTIPART_DEFAULT_PART_SIZE = Math.max(
|
||||
OSS_MULTIPART_MIN_PART_SIZE,
|
||||
parsePositiveInteger(process.env.OSS_MULTIPART_PART_SIZE_BYTES, 16 * 1024 * 1024)
|
||||
);
|
||||
const OSS_SINGLE_UPLOAD_THRESHOLD = Math.max(
|
||||
OSS_MULTIPART_MIN_PART_SIZE,
|
||||
parsePositiveInteger(process.env.OSS_SINGLE_UPLOAD_THRESHOLD_BYTES, 64 * 1024 * 1024)
|
||||
);
|
||||
|
||||
/**
|
||||
* 将 OSS/网络错误转换为友好的错误信息
|
||||
* @param {Error} error - 原始错误
|
||||
@@ -1090,9 +1121,13 @@ class OssStorageClient {
|
||||
* @param {string} remotePath - 远程文件路径
|
||||
*/
|
||||
async put(localPath, remotePath) {
|
||||
let bucket = '';
|
||||
let key = '';
|
||||
let uploadId = null;
|
||||
let fileHandle = null;
|
||||
try {
|
||||
const key = this.getObjectKey(remotePath);
|
||||
const bucket = this.getBucket();
|
||||
key = this.getObjectKey(remotePath);
|
||||
bucket = this.getBucket();
|
||||
|
||||
// 检查本地文件是否存在
|
||||
if (!fs.existsSync(localPath)) {
|
||||
@@ -1102,30 +1137,107 @@ class OssStorageClient {
|
||||
const fileStats = fs.statSync(localPath);
|
||||
const fileSize = fileStats.size;
|
||||
|
||||
// 检查文件大小（AWS S3 单次上传最大 5GB）
|
||||
const MAX_SINGLE_UPLOAD_SIZE = 5 * 1024 * 1024 * 1024; // 5GB
|
||||
if (fileSize > MAX_SINGLE_UPLOAD_SIZE) {
|
||||
throw new Error(`文件过大 (${formatFileSize(fileSize)})，单次上传最大支持 5GB，请使用分片上传`);
|
||||
if (fileSize <= OSS_SINGLE_UPLOAD_THRESHOLD) {
|
||||
// 小文件保持单请求上传，兼容性更高。
|
||||
const fileContent = fs.readFileSync(localPath);
|
||||
const command = new PutObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
Body: fileContent,
|
||||
ContentLength: fileSize,
|
||||
ChecksumAlgorithm: undefined
|
||||
});
|
||||
await this.s3Client.send(command);
|
||||
console.log(`[OSS存储] 上传成功: ${key} (${formatFileSize(fileSize)})`);
|
||||
return;
|
||||
}
|
||||
|
||||
// 使用Buffer上传而非流式上传，避免AWS SDK使用aws-chunked编码
|
||||
// 这样可以确保与阿里云OSS的兼容性
|
||||
const fileContent = fs.readFileSync(localPath);
|
||||
// 大文件自动切换到分片上传，避免整文件读入内存和 5GB 单请求上限。
|
||||
const minPartSizeByCount = Math.ceil(fileSize / OSS_MULTIPART_MAX_PARTS);
|
||||
const alignedMinPartSize = Math.ceil(
|
||||
Math.max(minPartSizeByCount, OSS_MULTIPART_MIN_PART_SIZE) / (1024 * 1024)
|
||||
) * 1024 * 1024;
|
||||
const partSize = Math.max(OSS_MULTIPART_DEFAULT_PART_SIZE, alignedMinPartSize);
|
||||
const estimatedParts = Math.ceil(fileSize / partSize);
|
||||
if (estimatedParts > OSS_MULTIPART_MAX_PARTS) {
|
||||
throw new Error(`文件过大，分片数量超限 (${estimatedParts}/${OSS_MULTIPART_MAX_PARTS})`);
|
||||
}
|
||||
|
||||
// 直接上传
|
||||
const command = new PutObjectCommand({
|
||||
const createResp = await this.s3Client.send(new CreateMultipartUploadCommand({
|
||||
Bucket: bucket,
|
||||
Key: key
|
||||
}));
|
||||
uploadId = createResp?.UploadId || null;
|
||||
if (!uploadId) {
|
||||
throw new Error('创建分片上传会话失败：缺少 UploadId');
|
||||
}
|
||||
|
||||
const uploadedParts = [];
|
||||
let offset = 0;
|
||||
let partNumber = 1;
|
||||
fileHandle = await fs.promises.open(localPath, 'r');
|
||||
|
||||
while (offset < fileSize) {
|
||||
const remaining = fileSize - offset;
|
||||
const currentPartSize = Math.min(partSize, remaining);
|
||||
const buffer = Buffer.allocUnsafe(currentPartSize);
|
||||
const { bytesRead } = await fileHandle.read(buffer, 0, currentPartSize, offset);
|
||||
if (bytesRead <= 0) {
|
||||
throw new Error('读取本地文件失败，分片上传中断');
|
||||
}
|
||||
|
||||
const body = bytesRead === currentPartSize ? buffer : buffer.subarray(0, bytesRead);
|
||||
const partResp = await this.s3Client.send(new UploadPartCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
PartNumber: partNumber,
|
||||
Body: body,
|
||||
ContentLength: bytesRead
|
||||
}));
|
||||
|
||||
if (!partResp?.ETag) {
|
||||
throw new Error(`分片 ${partNumber} 上传失败：服务端未返回 ETag`);
|
||||
}
|
||||
uploadedParts.push({
|
||||
ETag: partResp.ETag,
|
||||
PartNumber: partNumber
|
||||
});
|
||||
|
||||
offset += bytesRead;
|
||||
if (partNumber === 1 || partNumber % 10 === 0 || offset >= fileSize) {
|
||||
console.log(
|
||||
`[OSS存储] 上传分片进度: ${key} ${partNumber}/${estimatedParts} (${formatFileSize(offset)}/${formatFileSize(fileSize)})`
|
||||
);
|
||||
}
|
||||
partNumber += 1;
|
||||
}
|
||||
|
||||
await this.s3Client.send(new CompleteMultipartUploadCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
Body: fileContent,
|
||||
ContentLength: fileSize, // 明确指定内容长度，避免某些服务端问题
|
||||
// 禁用checksum算法（阿里云OSS不完全支持AWS的x-amz-content-sha256头）
|
||||
ChecksumAlgorithm: undefined
|
||||
});
|
||||
|
||||
await this.s3Client.send(command);
|
||||
console.log(`[OSS存储] 上传成功: ${key} (${formatFileSize(fileSize)})`);
|
||||
UploadId: uploadId,
|
||||
MultipartUpload: { Parts: uploadedParts }
|
||||
}));
|
||||
uploadId = null;
|
||||
console.log(
|
||||
`[OSS存储] 分片上传完成: ${key} (${formatFileSize(fileSize)}, ${uploadedParts.length} 片)`
|
||||
);
|
||||
|
||||
} catch (error) {
|
||||
if (uploadId && bucket && key) {
|
||||
try {
|
||||
await this.s3Client.send(new AbortMultipartUploadCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
UploadId: uploadId
|
||||
}));
|
||||
console.warn(`[OSS存储] 已中止失败的分片上传: ${key}`);
|
||||
} catch (abortError) {
|
||||
console.error(`[OSS存储] 中止分片上传失败: ${key}`, abortError.message);
|
||||
}
|
||||
}
|
||||
|
||||
console.error(`[OSS存储] 上传失败: ${remotePath}`, error.message);
|
||||
|
||||
// 判断错误类型并给出友好的错误信息
|
||||
@@ -1139,6 +1251,10 @@ class OssStorageClient {
|
||||
throw new Error(`本地文件不存在: ${localPath}`);
|
||||
}
|
||||
throw new Error(`文件上传失败: ${error.message}`);
|
||||
} finally {
|
||||
if (fileHandle) {
|
||||
await fileHandle.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user