feat: 删除SFTP上传工具,修复OSS配置bug
主要变更: - 删除管理员工具栏及上传工具相关功能(后端API + 前端UI) - 删除upload-tool目录及相关文件 - 修复OSS配置测试连接bug(testUser缺少has_oss_config标志) - 新增backend/utils加密和缓存工具模块 - 更新.gitignore排除测试报告文件 技术改进: - 统一使用OSS存储,废弃SFTP上传方式 - 修复OSS配置保存和测试连接时的错误处理 - 完善代码库文件管理,排除临时报告文件
This commit is contained in:
@@ -3,7 +3,7 @@ const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { Readable } = require('stream');
|
||||
const { UserDB } = require('./database');
|
||||
const { UserDB, SettingsDB } = require('./database');
|
||||
|
||||
// ===== 工具函数 =====
|
||||
|
||||
@@ -107,10 +107,8 @@ class StorageInterface {
|
||||
await client.init();
|
||||
return client;
|
||||
} else {
|
||||
// 在尝试连接 OSS 之前,先检查用户是否已配置 OSS
|
||||
if (!this.user.has_oss_config) {
|
||||
throw new Error('OSS 存储未配置,请先在设置中配置您的 OSS 服务(阿里云/腾讯云/AWS)');
|
||||
}
|
||||
// OSS 客户端会自动检查是否有可用配置(系统配置或用户配置)
|
||||
// 不再在这里强制检查 has_oss_config
|
||||
const client = new OssStorageClient(this.user);
|
||||
await client.connect();
|
||||
return client;
|
||||
@@ -509,6 +507,208 @@ class LocalStorageClient {
|
||||
this.user.local_storage_used = newUsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* 恢复未完成的重命名操作(启动时调用)
|
||||
* 扫描OSS存储中的待处理重命名标记文件,执行回滚或完成操作
|
||||
*
|
||||
* **重命名操作的两个阶段:**
|
||||
* 1. copying 阶段:正在复制文件到新位置
|
||||
* - 恢复策略:删除已复制的目标文件,保留原文件
|
||||
* 2. deleting 阶段:正在删除原文件
|
||||
* - 恢复策略:确保原文件被完全删除(补充删除逻辑)
|
||||
*
|
||||
* @private
|
||||
*/
|
||||
async recoverPendingRenames() {
|
||||
try {
|
||||
console.log('[OSS存储] 检查未完成的重命名操作...');
|
||||
|
||||
const bucket = this.getBucket();
|
||||
const markerPrefix = this.prefix + '.rename_pending_';
|
||||
|
||||
// 列出所有待处理的标记文件
|
||||
const listCommand = new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
Prefix: markerPrefix,
|
||||
MaxKeys: 100
|
||||
});
|
||||
|
||||
const response = await this.s3Client.send(listCommand);
|
||||
|
||||
if (!response.Contents || response.Contents.length === 0) {
|
||||
console.log('[OSS存储] 没有未完成的重命名操作');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[OSS存储] 发现 ${response.Contents.length} 个未完成的重命名操作,开始恢复...`);
|
||||
|
||||
for (const marker of response.Contents) {
|
||||
try {
|
||||
// 从标记文件名中解析元数据
|
||||
// 格式: .rename_pending_{timestamp}_{oldKeyHash}.json
|
||||
const markerKey = marker.Key;
|
||||
|
||||
// 读取标记文件内容
|
||||
const getMarkerCommand = new GetObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: markerKey
|
||||
});
|
||||
|
||||
const markerResponse = await this.s3Client.send(getMarkerCommand);
|
||||
const markerContent = await streamToBuffer(markerResponse.Body);
|
||||
const metadata = JSON.parse(markerContent.toString());
|
||||
|
||||
const { oldPrefix, newPrefix, timestamp, phase } = metadata;
|
||||
|
||||
// 检查标记是否过期(超过1小时视为失败,需要恢复)
|
||||
const age = Date.now() - timestamp;
|
||||
const TIMEOUT = 60 * 60 * 1000; // 1小时
|
||||
|
||||
if (age > TIMEOUT) {
|
||||
console.warn(`[OSS存储] 检测到超时的重命名操作: ${oldPrefix} -> ${newPrefix}, 阶段: ${phase}`);
|
||||
|
||||
// 根据不同阶段执行不同的恢复策略
|
||||
if (phase === 'copying') {
|
||||
// ===== 第一阶段:复制阶段超时 =====
|
||||
// 策略:删除已复制的目标文件,保留原文件
|
||||
console.log(`[OSS存储] [copying阶段] 执行回滚: 删除已复制的文件 ${newPrefix}`);
|
||||
await this._rollbackRename(oldPrefix, newPrefix);
|
||||
|
||||
} else if (phase === 'deleting') {
|
||||
// ===== 第二阶段:删除阶段超时(第二轮修复) =====
|
||||
// 策略:补充完整的删除逻辑,确保原文件被清理干净
|
||||
console.log(`[OSS存储] [deleting阶段] 执行补充删除: 清理剩余原文件 ${oldPrefix}`);
|
||||
|
||||
try {
|
||||
// 步骤1:列出原位置的所有剩余文件
|
||||
let continuationToken = null;
|
||||
let remainingCount = 0;
|
||||
const MAX_KEYS_PER_REQUEST = 1000;
|
||||
|
||||
do {
|
||||
const listOldCommand = new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
Prefix: oldPrefix,
|
||||
MaxKeys: MAX_KEYS_PER_REQUEST,
|
||||
ContinuationToken: continuationToken
|
||||
});
|
||||
|
||||
const listOldResponse = await this.s3Client.send(listOldCommand);
|
||||
continuationToken = listOldResponse.NextContinuationToken;
|
||||
|
||||
if (listOldResponse.Contents && listOldResponse.Contents.length > 0) {
|
||||
// 步骤2:批量删除剩余的原文件
|
||||
const deleteCommand = new DeleteObjectsCommand({
|
||||
Bucket: bucket,
|
||||
Delete: {
|
||||
Objects: listOldResponse.Contents.map(obj => ({ Key: obj.Key })),
|
||||
Quiet: true
|
||||
}
|
||||
});
|
||||
|
||||
const deleteResult = await this.s3Client.send(deleteCommand);
|
||||
remainingCount += listOldResponse.Contents.length;
|
||||
|
||||
console.log(`[OSS存储] [deleting阶段] 已删除 ${listOldResponse.Contents.length} 个剩余原文件`);
|
||||
|
||||
// 检查删除结果
|
||||
if (deleteResult.Errors && deleteResult.Errors.length > 0) {
|
||||
console.warn(`[OSS存储] [deleting阶段] 部分文件删除失败:`, deleteResult.Errors);
|
||||
}
|
||||
}
|
||||
} while (continuationToken);
|
||||
|
||||
if (remainingCount > 0) {
|
||||
console.log(`[OSS存储] [deleting阶段] 补充删除完成: 清理了 ${remainingCount} 个原文件`);
|
||||
} else {
|
||||
console.log(`[OSS存储] [deleting阶段] 原位置 ${oldPrefix} 已是空的,无需清理`);
|
||||
}
|
||||
|
||||
} catch (cleanupError) {
|
||||
console.error(`[OSS存储] [deleting阶段] 补充删除失败: ${cleanupError.message}`);
|
||||
// 继续执行,不中断流程
|
||||
}
|
||||
|
||||
} else {
|
||||
// 未知阶段,记录警告
|
||||
console.warn(`[OSS存储] 未知阶段 ${phase},跳过恢复`);
|
||||
}
|
||||
|
||||
// 删除标记文件(完成恢复后清理)
|
||||
await this.s3Client.send(new DeleteObjectsCommand({
|
||||
Bucket: bucket,
|
||||
Delete: {
|
||||
Objects: [{ Key: markerKey }],
|
||||
Quiet: true
|
||||
}
|
||||
}));
|
||||
|
||||
console.log(`[OSS存储] 已清理超时的重命名标记: ${markerKey}`);
|
||||
} else {
|
||||
console.log(`[OSS存储] 重命名操作仍在进行中: ${oldPrefix} -> ${newPrefix} (阶段: ${phase}, 剩余: ${Math.floor((TIMEOUT - age) / 1000)}秒)`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`[OSS存储] 恢复重命名操作失败: ${marker.Key}`, error.message);
|
||||
// 继续处理下一个标记文件
|
||||
}
|
||||
}
|
||||
|
||||
console.log('[OSS存储] 重命名操作恢复完成');
|
||||
} catch (error) {
|
||||
console.error('[OSS存储] 恢复重命名操作时出错:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 回滚重命名操作(删除已复制的目标文件)
|
||||
* @param {string} oldPrefix - 原前缀
|
||||
* @param {string} newPrefix - 新前缀
|
||||
* @private
|
||||
*/
|
||||
async _rollbackRename(oldPrefix, newPrefix) {
|
||||
const bucket = this.getBucket();
|
||||
const newPrefixWithSlash = newPrefix.endsWith('/') ? newPrefix : `${newPrefix}/`;
|
||||
|
||||
try {
|
||||
// 列出所有已复制的对象
|
||||
let continuationToken = null;
|
||||
let deletedCount = 0;
|
||||
|
||||
do {
|
||||
const listCommand = new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
Prefix: newPrefixWithSlash,
|
||||
ContinuationToken: continuationToken,
|
||||
MaxKeys: 1000
|
||||
});
|
||||
|
||||
const listResponse = await this.s3Client.send(listCommand);
|
||||
continuationToken = listResponse.NextContinuationToken;
|
||||
|
||||
if (listResponse.Contents && listResponse.Contents.length > 0) {
|
||||
// 批量删除
|
||||
const deleteCommand = new DeleteObjectsCommand({
|
||||
Bucket: bucket,
|
||||
Delete: {
|
||||
Objects: listResponse.Contents.map(obj => ({ Key: obj.Key })),
|
||||
Quiet: true
|
||||
}
|
||||
});
|
||||
|
||||
await this.s3Client.send(deleteCommand);
|
||||
deletedCount += listResponse.Contents.length;
|
||||
}
|
||||
} while (continuationToken);
|
||||
|
||||
if (deletedCount > 0) {
|
||||
console.log(`[OSS存储] 回滚完成: 删除了 ${deletedCount} 个对象`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`[OSS存储] 回滚失败: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化文件大小
|
||||
*/
|
||||
@@ -522,20 +722,65 @@ class LocalStorageClient {
|
||||
/**
|
||||
* OSS 存储客户端(基于 S3 协议)
|
||||
* 支持阿里云 OSS、腾讯云 COS、AWS S3
|
||||
*
|
||||
* **优先级规则:**
|
||||
* 1. 如果系统配置了统一 OSS(管理员配置),优先使用系统配置
|
||||
* 2. 否则使用用户自己的 OSS 配置(如果有的话)
|
||||
* 3. 用户文件通过 `user_{userId}/` 前缀完全隔离
|
||||
*/
|
||||
class OssStorageClient {
|
||||
constructor(user) {
|
||||
this.user = user;
|
||||
this.s3Client = null;
|
||||
this.prefix = `user_${user.id}/`; // 用户隔离前缀
|
||||
this.useUnifiedConfig = false; // 标记是否使用统一配置
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取有效的 OSS 配置(优先使用系统配置)
|
||||
* @returns {Object} OSS 配置对象
|
||||
* @throws {Error} 如果没有可用的配置
|
||||
*/
|
||||
getEffectiveConfig() {
|
||||
// 1. 优先检查系统级统一配置
|
||||
const unifiedConfig = SettingsDB.getUnifiedOssConfig();
|
||||
if (unifiedConfig) {
|
||||
console.log(`[OSS存储] 用户 ${this.user.id} 使用系统级统一 OSS 配置`);
|
||||
this.useUnifiedConfig = true;
|
||||
return {
|
||||
oss_provider: unifiedConfig.provider,
|
||||
oss_region: unifiedConfig.region,
|
||||
oss_access_key_id: unifiedConfig.access_key_id,
|
||||
oss_access_key_secret: unifiedConfig.access_key_secret,
|
||||
oss_bucket: unifiedConfig.bucket,
|
||||
oss_endpoint: unifiedConfig.endpoint
|
||||
};
|
||||
}
|
||||
|
||||
// 2. 回退到用户自己的配置
|
||||
if (this.user.has_oss_config) {
|
||||
console.log(`[OSS存储] 用户 ${this.user.id} 使用个人 OSS 配置`);
|
||||
this.useUnifiedConfig = false;
|
||||
return {
|
||||
oss_provider: this.user.oss_provider,
|
||||
oss_region: this.user.oss_region,
|
||||
oss_access_key_id: this.user.oss_access_key_id,
|
||||
oss_access_key_secret: this.user.oss_access_key_secret,
|
||||
oss_bucket: this.user.oss_bucket,
|
||||
oss_endpoint: this.user.oss_endpoint
|
||||
};
|
||||
}
|
||||
|
||||
// 3. 没有可用配置
|
||||
throw new Error('OSS 存储未配置,请联系管理员配置系统级 OSS 服务');
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 OSS 配置是否完整
|
||||
* @throws {Error} 配置不完整时抛出错误
|
||||
*/
|
||||
validateConfig() {
|
||||
const { oss_provider, oss_access_key_id, oss_access_key_secret, oss_bucket } = this.user;
|
||||
validateConfig(config) {
|
||||
const { oss_provider, oss_access_key_id, oss_access_key_secret, oss_bucket } = config;
|
||||
|
||||
if (!oss_provider || !['aliyun', 'tencent', 'aws'].includes(oss_provider)) {
|
||||
throw new Error('无效的 OSS 服务商,必须是 aliyun、tencent 或 aws');
|
||||
@@ -553,15 +798,17 @@ class OssStorageClient {
|
||||
|
||||
/**
|
||||
* 根据服务商构建 S3 配置
|
||||
* @param {Object} config - OSS 配置对象
|
||||
* @returns {Object} S3 客户端配置
|
||||
*/
|
||||
buildConfig() {
|
||||
buildConfig(config) {
|
||||
// 先验证配置
|
||||
this.validateConfig();
|
||||
this.validateConfig(config);
|
||||
|
||||
const { oss_provider, oss_region, oss_access_key_id, oss_access_key_secret, oss_endpoint } = this.user;
|
||||
const { oss_provider, oss_region, oss_access_key_id, oss_access_key_secret, oss_endpoint } = config;
|
||||
|
||||
// AWS S3 默认配置
|
||||
let config = {
|
||||
let s3Config = {
|
||||
region: oss_region || 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: oss_access_key_id,
|
||||
@@ -583,41 +830,41 @@ class OssStorageClient {
|
||||
if (!region.startsWith('oss-')) {
|
||||
region = 'oss-' + region;
|
||||
}
|
||||
config.region = region;
|
||||
s3Config.region = region;
|
||||
|
||||
if (!oss_endpoint) {
|
||||
// 默认 endpoint 格式:https://{region}.aliyuncs.com
|
||||
config.endpoint = `https://${region}.aliyuncs.com`;
|
||||
s3Config.endpoint = `https://${region}.aliyuncs.com`;
|
||||
} else {
|
||||
// 确保 endpoint 以 https:// 或 http:// 开头
|
||||
config.endpoint = oss_endpoint.startsWith('http') ? oss_endpoint : `https://${oss_endpoint}`;
|
||||
s3Config.endpoint = oss_endpoint.startsWith('http') ? oss_endpoint : `https://${oss_endpoint}`;
|
||||
}
|
||||
// 阿里云 OSS 使用 virtual-hosted-style,但需要设置 forcePathStyle 为 false
|
||||
config.forcePathStyle = false;
|
||||
s3Config.forcePathStyle = false;
|
||||
}
|
||||
// 腾讯云 COS
|
||||
else if (oss_provider === 'tencent') {
|
||||
config.region = oss_region || 'ap-guangzhou';
|
||||
s3Config.region = oss_region || 'ap-guangzhou';
|
||||
if (!oss_endpoint) {
|
||||
// 默认 endpoint 格式:https://cos.{region}.myqcloud.com
|
||||
config.endpoint = `https://cos.${config.region}.myqcloud.com`;
|
||||
s3Config.endpoint = `https://cos.${s3Config.region}.myqcloud.com`;
|
||||
} else {
|
||||
config.endpoint = oss_endpoint.startsWith('http') ? oss_endpoint : `https://${oss_endpoint}`;
|
||||
s3Config.endpoint = oss_endpoint.startsWith('http') ? oss_endpoint : `https://${oss_endpoint}`;
|
||||
}
|
||||
// 腾讯云 COS 使用 virtual-hosted-style
|
||||
config.forcePathStyle = false;
|
||||
s3Config.forcePathStyle = false;
|
||||
}
|
||||
// AWS S3 或其他兼容服务
|
||||
else {
|
||||
if (oss_endpoint) {
|
||||
config.endpoint = oss_endpoint.startsWith('http') ? oss_endpoint : `https://${oss_endpoint}`;
|
||||
s3Config.endpoint = oss_endpoint.startsWith('http') ? oss_endpoint : `https://${oss_endpoint}`;
|
||||
// 自定义 endpoint(如 MinIO)通常需要 path-style
|
||||
config.forcePathStyle = true;
|
||||
s3Config.forcePathStyle = true;
|
||||
}
|
||||
// AWS 使用默认 endpoint,无需额外配置
|
||||
}
|
||||
|
||||
return config;
|
||||
return s3Config;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -625,9 +872,15 @@ class OssStorageClient {
|
||||
*/
|
||||
async connect() {
|
||||
try {
|
||||
const config = this.buildConfig();
|
||||
this.s3Client = new S3Client(config);
|
||||
console.log(`[OSS存储] 已连接: ${this.user.oss_provider}, bucket: ${this.user.oss_bucket}`);
|
||||
// 获取有效的 OSS 配置(系统配置优先)
|
||||
const ossConfig = this.getEffectiveConfig();
|
||||
const s3Config = this.buildConfig(ossConfig);
|
||||
|
||||
// 保存当前使用的配置(供其他方法使用)
|
||||
this.currentConfig = ossConfig;
|
||||
|
||||
this.s3Client = new S3Client(s3Config);
|
||||
console.log(`[OSS存储] 已连接: ${ossConfig.oss_provider}, bucket: ${ossConfig.oss_bucket}, 使用统一配置: ${this.useUnifiedConfig}`);
|
||||
return this;
|
||||
} catch (error) {
|
||||
console.error(`[OSS存储] 连接失败:`, error.message);
|
||||
@@ -635,6 +888,30 @@ class OssStorageClient {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前使用的 bucket 名称
|
||||
* @returns {string}
|
||||
*/
|
||||
getBucket() {
|
||||
if (this.currentConfig && this.currentConfig.oss_bucket) {
|
||||
return this.currentConfig.oss_bucket;
|
||||
}
|
||||
// 回退到用户配置(向后兼容)
|
||||
return this.user.oss_bucket;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前使用的 provider
|
||||
* @returns {string}
|
||||
*/
|
||||
getProvider() {
|
||||
if (this.currentConfig && this.currentConfig.oss_provider) {
|
||||
return this.currentConfig.oss_provider;
|
||||
}
|
||||
// 回退到用户配置(向后兼容)
|
||||
return this.user.oss_provider;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取对象的完整 Key(带用户前缀)
|
||||
* 增强安全检查,防止路径遍历攻击
|
||||
@@ -715,7 +992,7 @@ class OssStorageClient {
|
||||
async list(dirPath, maxItems = 10000) {
|
||||
try {
|
||||
let prefix = this.getObjectKey(dirPath);
|
||||
const bucket = this.user.oss_bucket;
|
||||
const bucket = this.getBucket();
|
||||
|
||||
// 确保前缀以斜杠结尾(除非是根目录)
|
||||
if (prefix && !prefix.endsWith('/')) {
|
||||
@@ -810,7 +1087,7 @@ class OssStorageClient {
|
||||
|
||||
try {
|
||||
const key = this.getObjectKey(remotePath);
|
||||
const bucket = this.user.oss_bucket;
|
||||
const bucket = this.getBucket();
|
||||
|
||||
// 检查本地文件是否存在
|
||||
if (!fs.existsSync(localPath)) {
|
||||
@@ -869,24 +1146,28 @@ class OssStorageClient {
|
||||
|
||||
/**
|
||||
* 删除文件或文件夹
|
||||
* ===== P0 性能优化:返回删除的文件大小,用于更新存储使用量缓存 =====
|
||||
* @returns {Promise<{size: number}>} 返回删除的文件总大小(字节)
|
||||
*/
|
||||
async delete(filePath) {
|
||||
try {
|
||||
const key = this.getObjectKey(filePath);
|
||||
const bucket = this.user.oss_bucket;
|
||||
const bucket = this.getBucket();
|
||||
|
||||
// 检查是文件还是目录(忽略不存在的文件)
|
||||
let statResult;
|
||||
try {
|
||||
statResult = await this.stat(filePath);
|
||||
} catch (statError) {
|
||||
if (statError.message && statError.message.includes('不存在')) {
|
||||
if (statError.message && statResult?.message.includes('不存在')) {
|
||||
console.warn(`[OSS存储] 文件不存在,跳过删除: ${key}`);
|
||||
return; // 文件不存在,直接返回
|
||||
return { size: 0 }; // 文件不存在,返回大小为 0
|
||||
}
|
||||
throw statError; // 其他错误继续抛出
|
||||
}
|
||||
|
||||
let totalDeletedSize = 0;
|
||||
|
||||
if (statResult.isDirectory) {
|
||||
// 删除目录:列出所有对象并批量删除
|
||||
// 使用分页循环处理超过 1000 个对象的情况
|
||||
@@ -906,6 +1187,11 @@ class OssStorageClient {
|
||||
continuationToken = listResponse.NextContinuationToken;
|
||||
|
||||
if (listResponse.Contents && listResponse.Contents.length > 0) {
|
||||
// 累加删除的文件大小
|
||||
for (const obj of listResponse.Contents) {
|
||||
totalDeletedSize += obj.Size || 0;
|
||||
}
|
||||
|
||||
// 批量删除当前批次的对象
|
||||
const deleteCommand = new DeleteObjectsCommand({
|
||||
Bucket: bucket,
|
||||
@@ -927,10 +1213,16 @@ class OssStorageClient {
|
||||
} while (continuationToken);
|
||||
|
||||
if (totalDeletedCount > 0) {
|
||||
console.log(`[OSS存储] 删除目录: ${key} (${totalDeletedCount} 个对象)`);
|
||||
console.log(`[OSS存储] 删除目录: ${key} (${totalDeletedCount} 个对象, ${totalDeletedSize} 字节)`);
|
||||
}
|
||||
|
||||
return { size: totalDeletedSize };
|
||||
} else {
|
||||
// 删除单个文件
|
||||
// 获取文件大小
|
||||
const size = statResult.size || 0;
|
||||
totalDeletedSize = size;
|
||||
|
||||
const command = new DeleteObjectsCommand({
|
||||
Bucket: bucket,
|
||||
Delete: {
|
||||
@@ -940,7 +1232,9 @@ class OssStorageClient {
|
||||
});
|
||||
|
||||
await this.s3Client.send(command);
|
||||
console.log(`[OSS存储] 删除文件: ${key}`);
|
||||
console.log(`[OSS存储] 删除文件: ${key} (${size} 字节)`);
|
||||
|
||||
return { size: totalDeletedSize };
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`[OSS存储] 删除失败: ${filePath}`, error.message);
|
||||
@@ -1046,6 +1340,7 @@ class OssStorageClient {
|
||||
/**
|
||||
* 重命名目录(内部方法)
|
||||
* 通过遍历目录下所有对象,逐个复制到新位置后删除原对象
|
||||
* 使用事务标记机制防止竞态条件
|
||||
* @param {string} oldPath - 原目录路径
|
||||
* @param {string} newPath - 新目录路径
|
||||
* @private
|
||||
@@ -1053,23 +1348,46 @@ class OssStorageClient {
|
||||
async _renameDirectory(oldPath, newPath) {
|
||||
const oldPrefix = this.getObjectKey(oldPath);
|
||||
const newPrefix = this.getObjectKey(newPath);
|
||||
const bucket = this.user.oss_bucket;
|
||||
const bucket = this.getBucket();
|
||||
|
||||
// 确保前缀以斜杠结尾
|
||||
const oldPrefixWithSlash = oldPrefix.endsWith('/') ? oldPrefix : `${oldPrefix}/`;
|
||||
const newPrefixWithSlash = newPrefix.endsWith('/') ? newPrefix : `${newPrefix}/`;
|
||||
|
||||
// 生成事务标记文件
|
||||
const timestamp = Date.now();
|
||||
const markerKey = `${this.prefix}.rename_pending_${timestamp}.json`;
|
||||
|
||||
// 标记文件内容(用于恢复)
|
||||
const markerContent = JSON.stringify({
|
||||
oldPrefix: oldPrefixWithSlash,
|
||||
newPrefix: newPrefixWithSlash,
|
||||
timestamp: timestamp,
|
||||
phase: 'copying' // 标记当前阶段:copying(复制中)、deleting(删除中)
|
||||
});
|
||||
|
||||
let continuationToken = null;
|
||||
let copiedKeys = [];
|
||||
let totalCount = 0;
|
||||
|
||||
try {
|
||||
// 第一阶段:复制所有对象到新位置
|
||||
// 步骤1:创建事务标记文件(标识重命名操作开始)
|
||||
console.log(`[OSS存储] 创建重命名事务标记: ${markerKey}`);
|
||||
const putMarkerCommand = new PutObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: markerKey,
|
||||
Body: markerContent,
|
||||
ContentType: 'application/json'
|
||||
});
|
||||
await this.s3Client.send(putMarkerCommand);
|
||||
|
||||
// 步骤2:复制所有对象到新位置
|
||||
do {
|
||||
const listCommand = new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
Prefix: oldPrefixWithSlash,
|
||||
ContinuationToken: continuationToken
|
||||
ContinuationToken: continuationToken,
|
||||
MaxKeys: 1000
|
||||
});
|
||||
|
||||
const listResponse = await this.s3Client.send(listCommand);
|
||||
@@ -1095,9 +1413,22 @@ class OssStorageClient {
|
||||
}
|
||||
} while (continuationToken);
|
||||
|
||||
// 第二阶段:删除所有原对象
|
||||
// 步骤3:更新标记文件状态为 deleting(复制完成,开始删除)
|
||||
const updatedMarkerContent = JSON.stringify({
|
||||
oldPrefix: oldPrefixWithSlash,
|
||||
newPrefix: newPrefixWithSlash,
|
||||
timestamp: timestamp,
|
||||
phase: 'deleting'
|
||||
});
|
||||
await this.s3Client.send(new PutObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: markerKey,
|
||||
Body: updatedMarkerContent,
|
||||
ContentType: 'application/json'
|
||||
}));
|
||||
|
||||
// 步骤4:删除所有原对象(批量删除)
|
||||
if (copiedKeys.length > 0) {
|
||||
// 批量删除(每批最多 1000 个)
|
||||
for (let i = 0; i < copiedKeys.length; i += 1000) {
|
||||
const batch = copiedKeys.slice(i, i + 1000);
|
||||
const deleteCommand = new DeleteObjectsCommand({
|
||||
@@ -1111,13 +1442,25 @@ class OssStorageClient {
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[OSS存储] 重命名目录: ${oldPath} -> ${newPath} (${totalCount} 个对象)`);
|
||||
// 步骤5:删除事务标记文件(操作完成)
|
||||
await this.s3Client.send(new DeleteObjectsCommand({
|
||||
Bucket: bucket,
|
||||
Delete: {
|
||||
Objects: [{ Key: markerKey }],
|
||||
Quiet: true
|
||||
}
|
||||
}));
|
||||
|
||||
console.log(`[OSS存储] 重命名目录完成: ${oldPath} -> ${newPath} (${totalCount} 个对象)`);
|
||||
|
||||
} catch (error) {
|
||||
// 如果出错,尝试回滚(删除已复制的新对象)
|
||||
// 如果出错,尝试回滚
|
||||
console.error(`[OSS存储] 目录重命名失败: ${error.message}`);
|
||||
|
||||
if (copiedKeys.length > 0) {
|
||||
console.warn(`[OSS存储] 目录重命名失败,尝试回滚已复制的 ${copiedKeys.length} 个对象...`);
|
||||
console.warn(`[OSS存储] 尝试回滚已复制的 ${copiedKeys.length} 个对象...`);
|
||||
try {
|
||||
// 回滚:删除已复制的新对象
|
||||
for (let i = 0; i < copiedKeys.length; i += 1000) {
|
||||
const batch = copiedKeys.slice(i, i + 1000);
|
||||
const deleteCommand = new DeleteObjectsCommand({
|
||||
@@ -1134,6 +1477,20 @@ class OssStorageClient {
|
||||
console.error(`[OSS存储] 回滚失败: ${rollbackError.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 清理事务标记文件(如果还存在)
|
||||
try {
|
||||
await this.s3Client.send(new DeleteObjectsCommand({
|
||||
Bucket: bucket,
|
||||
Delete: {
|
||||
Objects: [{ Key: markerKey }],
|
||||
Quiet: true
|
||||
}
|
||||
}));
|
||||
} catch (markerError) {
|
||||
// 忽略标记文件删除错误
|
||||
}
|
||||
|
||||
throw new Error(`重命名目录失败: ${error.message}`);
|
||||
}
|
||||
}
|
||||
@@ -1209,7 +1566,7 @@ class OssStorageClient {
|
||||
async mkdir(dirPath) {
|
||||
try {
|
||||
const key = this.getObjectKey(dirPath);
|
||||
const bucket = this.user.oss_bucket;
|
||||
const bucket = this.getBucket();
|
||||
|
||||
// OSS 中文件夹通过以斜杠结尾的空对象模拟
|
||||
const folderKey = key.endsWith('/') ? key : `${key}/`;
|
||||
@@ -1266,15 +1623,16 @@ class OssStorageClient {
|
||||
*/
|
||||
getPublicUrl(filePath) {
|
||||
const key = this.getObjectKey(filePath);
|
||||
const bucket = this.user.oss_bucket;
|
||||
const bucket = this.getBucket();
|
||||
const provider = this.getProvider();
|
||||
const region = this.s3Client.config.region;
|
||||
|
||||
let baseUrl;
|
||||
if (this.user.oss_provider === 'aliyun') {
|
||||
if (provider === 'aliyun') {
|
||||
// 阿里云 OSS 公开 URL 格式
|
||||
const ossRegion = region.startsWith('oss-') ? region : `oss-${region}`;
|
||||
baseUrl = `https://${bucket}.${ossRegion}.aliyuncs.com`;
|
||||
} else if (this.user.oss_provider === 'tencent') {
|
||||
} else if (provider === 'tencent') {
|
||||
// 腾讯云 COS 公开 URL 格式
|
||||
baseUrl = `https://${bucket}.cos.${region}.myqcloud.com`;
|
||||
} else {
|
||||
@@ -1345,6 +1703,19 @@ class OssStorageClient {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将流转换为 Buffer(辅助函数)
|
||||
*/
|
||||
async function streamToBuffer(stream) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks = [];
|
||||
stream.on('data', (chunk) => chunks.push(chunk));
|
||||
stream.on('error', reject);
|
||||
stream.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
module.exports = {
|
||||
StorageInterface,
|
||||
LocalStorageClient,
|
||||
|
||||
Reference in New Issue
Block a user