b7b00fff48
- 后端: Node.js + Express + SQLite架构 - 前端: Vue 3 + Axios实现 - 功能: 用户认证、文件上传/下载、分享链接、密码重置 - 安全: 密码加密、分享链接过期机制、缓存一致性 - 部署: Docker + Nginx容器化配置 - 测试: 完整的边界测试、并发测试和状态一致性测试
353 lines
10 KiB
JavaScript
353 lines
10 KiB
JavaScript
/**
|
||
* 存储使用情况缓存管理器
|
||
* ===== P0 性能优化：解决 OSS 统计性能瓶颈 =====
|
||
*
|
||
* 问题：每次获取存储使用情况都要遍历所有 OSS 对象，极其耗时
|
||
* 解决方案：使用数据库字段 storage_used 维护缓存，上传/删除时更新
|
||
*
|
||
* @module StorageUsageCache
|
||
*/
|
||
|
||
const { UserDB } = require('../database');
|
||
|
||
/**
|
||
* 存储使用情况缓存类
|
||
*/
|
||
class StorageUsageCache {
|
||
/**
|
||
* 获取用户的存储使用情况（从缓存）
|
||
* @param {number} userId - 用户ID
|
||
* @returns {Promise<{totalSize: number, totalSizeFormatted: string, fileCount: number, cached: boolean}>}
|
||
*/
|
||
static async getUsage(userId) {
|
||
try {
|
||
const user = UserDB.findById(userId);
|
||
|
||
if (!user) {
|
||
throw new Error('用户不存在');
|
||
}
|
||
|
||
// 从数据库缓存读取
|
||
const storageUsed = user.storage_used || 0;
|
||
|
||
// 导入格式化函数
|
||
const { formatFileSize } = require('../storage');
|
||
|
||
return {
|
||
totalSize: storageUsed,
|
||
totalSizeFormatted: formatFileSize(storageUsed),
|
||
fileCount: null, // 缓存模式不统计文件数
|
||
cached: true
|
||
};
|
||
} catch (error) {
|
||
console.error('[存储缓存] 获取失败:', error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 更新用户的存储使用量
|
||
* @param {number} userId - 用户ID
|
||
* @param {number} deltaSize - 变化量（正数为增加，负数为减少）
|
||
* @returns {Promise<boolean>}
|
||
*/
|
||
static async updateUsage(userId, deltaSize) {
|
||
try {
|
||
// 使用 SQL 原子操作，避免并发问题
|
||
const result = UserDB.update(userId, {
|
||
// 使用原始 SQL，因为 update 方法不支持表达式
|
||
// 注意：这里需要在数据库层执行 UPDATE ... SET storage_used = storage_used + ?
|
||
});
|
||
|
||
// 直接执行 SQL 更新
|
||
const { db } = require('../database');
|
||
db.prepare(`
|
||
UPDATE users
|
||
SET storage_used = storage_used + ?
|
||
WHERE id = ?
|
||
`).run(deltaSize, userId);
|
||
|
||
console.log(`[存储缓存] 用户 ${userId} 存储变化: ${deltaSize > 0 ? '+' : ''}${deltaSize} 字节`);
|
||
return true;
|
||
} catch (error) {
|
||
console.error('[存储缓存] 更新失败:', error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 重置用户的存储使用量（管理员功能，用于全量统计后更新缓存）
|
||
* @param {number} userId - 用户ID
|
||
* @param {number} totalSize - 实际总大小
|
||
* @returns {Promise<boolean>}
|
||
*/
|
||
static async resetUsage(userId, totalSize) {
|
||
try {
|
||
// 使用直接SQL更新，绕过UserDB.update()的字段白名单限制
|
||
const { db } = require('../database');
|
||
db.prepare(`
|
||
UPDATE users
|
||
SET storage_used = ?
|
||
WHERE id = ?
|
||
`).run(totalSize, userId);
|
||
|
||
console.log(`[存储缓存] 用户 ${userId} 存储重置: ${totalSize} 字节`);
|
||
return true;
|
||
} catch (error) {
|
||
console.error('[存储缓存] 重置失败:', error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 验证并修复缓存（管理员功能）
|
||
* 通过全量统计对比缓存值，如果不一致则更新
|
||
* @param {number} userId - 用户ID
|
||
* @param {OssStorageClient} ossClient - OSS 客户端实例
|
||
* @returns {Promise<{actual: number, cached: number, corrected: boolean}>}
|
||
*/
|
||
static async validateAndFix(userId, ossClient) {
|
||
try {
|
||
const { ListObjectsV2Command } = require('@aws-sdk/client-s3');
|
||
const user = UserDB.findById(userId);
|
||
|
||
if (!user) {
|
||
throw new Error('用户不存在');
|
||
}
|
||
|
||
// 执行全量统计
|
||
let totalSize = 0;
|
||
let continuationToken = null;
|
||
|
||
do {
|
||
const command = new ListObjectsV2Command({
|
||
Bucket: ossClient.getBucket(), // 使用ossClient的getBucket()方法以支持系统级统一OSS配置
|
||
Prefix: `user_${userId}/`,
|
||
ContinuationToken: continuationToken
|
||
});
|
||
|
||
const response = await ossClient.s3Client.send(command);
|
||
|
||
if (response.Contents) {
|
||
for (const obj of response.Contents) {
|
||
totalSize += obj.Size || 0;
|
||
}
|
||
}
|
||
|
||
continuationToken = response.NextContinuationToken;
|
||
} while (continuationToken);
|
||
|
||
const cached = user.storage_used || 0;
|
||
const corrected = totalSize !== cached;
|
||
|
||
if (corrected) {
|
||
await this.resetUsage(userId, totalSize);
|
||
console.log(`[存储缓存] 用户 ${userId} 缓存已修复: ${cached} → ${totalSize}`);
|
||
}
|
||
|
||
return {
|
||
actual: totalSize,
|
||
cached,
|
||
corrected
|
||
};
|
||
} catch (error) {
|
||
console.error('[存储缓存] 验证修复失败:', error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 检查缓存完整性（第二轮修复：缓存一致性保障）
|
||
* 对比缓存值与实际 OSS 存储使用情况，但不自动修复
|
||
* @param {number} userId - 用户ID
|
||
* @param {OssStorageClient} ossClient - OSS 客户端实例
|
||
* @returns {Promise<{consistent: boolean, cached: number, actual: number, diff: number}>}
|
||
*/
|
||
static async checkIntegrity(userId, ossClient) {
|
||
try {
|
||
const { ListObjectsV2Command } = require('@aws-sdk/client-s3');
|
||
const user = UserDB.findById(userId);
|
||
|
||
if (!user) {
|
||
throw new Error('用户不存在');
|
||
}
|
||
|
||
// 执行全量统计
|
||
let totalSize = 0;
|
||
let fileCount = 0;
|
||
let continuationToken = null;
|
||
|
||
do {
|
||
const command = new ListObjectsV2Command({
|
||
Bucket: ossClient.getBucket(), // 使用ossClient的getBucket()方法以支持系统级统一OSS配置
|
||
Prefix: `user_${userId}/`,
|
||
ContinuationToken: continuationToken
|
||
});
|
||
|
||
const response = await ossClient.s3Client.send(command);
|
||
|
||
if (response.Contents) {
|
||
for (const obj of response.Contents) {
|
||
totalSize += obj.Size || 0;
|
||
fileCount++;
|
||
}
|
||
}
|
||
|
||
continuationToken = response.NextContinuationToken;
|
||
} while (continuationToken);
|
||
|
||
const cached = user.storage_used || 0;
|
||
const diff = totalSize - cached;
|
||
const consistent = Math.abs(diff) === 0;
|
||
|
||
console.log(`[存储缓存] 用户 ${userId} 完整性检查: 缓存=${cached}, 实际=${totalSize}, 差异=${diff}`);
|
||
|
||
return {
|
||
consistent,
|
||
cached,
|
||
actual: totalSize,
|
||
fileCount,
|
||
diff
|
||
};
|
||
} catch (error) {
|
||
console.error('[存储缓存] 完整性检查失败:', error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 重建缓存（第二轮修复：缓存一致性保障）
|
||
* 强制从 OSS 全量统计并更新缓存值，绕过一致性检查
|
||
* @param {number} userId - 用户ID
|
||
* @param {OssStorageClient} ossClient - OSS 客户端实例
|
||
* @returns {Promise<{previous: number, current: number, fileCount: number}>}
|
||
*/
|
||
static async rebuildCache(userId, ossClient) {
|
||
try {
|
||
const { ListObjectsV2Command } = require('@aws-sdk/client-s3');
|
||
const user = UserDB.findById(userId);
|
||
|
||
if (!user) {
|
||
throw new Error('用户不存在');
|
||
}
|
||
|
||
console.log(`[存储缓存] 开始重建用户 ${userId} 的缓存...`);
|
||
|
||
// 执行全量统计
|
||
let totalSize = 0;
|
||
let fileCount = 0;
|
||
let continuationToken = null;
|
||
|
||
do {
|
||
const command = new ListObjectsV2Command({
|
||
Bucket: ossClient.getBucket(), // 使用ossClient的getBucket()方法以支持系统级统一OSS配置
|
||
Prefix: `user_${userId}/`,
|
||
ContinuationToken: continuationToken
|
||
});
|
||
|
||
const response = await ossClient.s3Client.send(command);
|
||
|
||
if (response.Contents) {
|
||
for (const obj of response.Contents) {
|
||
totalSize += obj.Size || 0;
|
||
fileCount++;
|
||
}
|
||
}
|
||
|
||
continuationToken = response.NextContinuationToken;
|
||
} while (continuationToken);
|
||
|
||
const previous = user.storage_used || 0;
|
||
|
||
// 强制更新缓存
|
||
await this.resetUsage(userId, totalSize);
|
||
|
||
console.log(`[存储缓存] 用户 ${userId} 缓存重建完成: ${previous} → ${totalSize} (${fileCount} 个文件)`);
|
||
|
||
return {
|
||
previous,
|
||
current: totalSize,
|
||
fileCount
|
||
};
|
||
} catch (error) {
|
||
console.error('[存储缓存] 重建缓存失败:', error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 检查所有用户的缓存一致性（第二轮修复：批量检查）
|
||
* @param {Array} users - 用户列表
|
||
* @param {Function} getOssClient - 获取 OSS 客户端的函数
|
||
* @returns {Promise<Array>} 检查结果列表
|
||
*/
|
||
static async checkAllUsersIntegrity(users, getOssClient) {
|
||
const results = [];
|
||
|
||
for (const user of users) {
|
||
// 跳过没有配置 OSS 的用户（需要检查系统级统一配置）
|
||
const { SettingsDB } = require('../database');
|
||
const hasUnifiedConfig = SettingsDB.hasUnifiedOssConfig();
|
||
if (!user.has_oss_config && !hasUnifiedConfig) {
|
||
continue;
|
||
}
|
||
|
||
try {
|
||
const ossClient = getOssClient(user);
|
||
const checkResult = await this.checkIntegrity(user.id, ossClient);
|
||
|
||
results.push({
|
||
userId: user.id,
|
||
username: user.username,
|
||
...checkResult
|
||
});
|
||
} catch (error) {
|
||
console.error(`[存储缓存] 检查用户 ${user.id} 失败:`, error.message);
|
||
results.push({
|
||
userId: user.id,
|
||
username: user.username,
|
||
error: error.message
|
||
});
|
||
}
|
||
}
|
||
|
||
return results;
|
||
}
|
||
|
||
/**
|
||
* 自动检测并修复缓存不一致（第二轮修复：自动化保障）
|
||
* 当检测到不一致时自动修复，并记录日志
|
||
* @param {number} userId - 用户ID
|
||
* @param {OssStorageClient} ossClient - OSS 客户端实例
|
||
* @param {number} threshold - 差异阈值（字节），默认 0（任何差异都修复）
|
||
* @returns {Promise<{autoFixed: boolean, diff: number}>}
|
||
*/
|
||
static async autoDetectAndFix(userId, ossClient, threshold = 0) {
|
||
try {
|
||
const checkResult = await this.checkIntegrity(userId, ossClient);
|
||
|
||
if (!checkResult.consistent && Math.abs(checkResult.diff) > threshold) {
|
||
console.warn(`[存储缓存] 检测到用户 ${userId} 缓存不一致: 差异 ${checkResult.diff} 字节`);
|
||
|
||
// 自动修复
|
||
await this.rebuildCache(userId, ossClient);
|
||
|
||
return {
|
||
autoFixed: true,
|
||
diff: checkResult.diff
|
||
};
|
||
}
|
||
|
||
return {
|
||
autoFixed: false,
|
||
diff: checkResult.diff
|
||
};
|
||
} catch (error) {
|
||
console.error('[存储缓存] 自动检测修复失败:', error);
|
||
throw error;
|
||
}
|
||
}
|
||
}
|
||
|
||
module.exports = StorageUsageCache;
|