feat(quota): add downloadable traffic quota with local/OSS/share metering

This commit is contained in:
2026-02-17 16:52:26 +08:00
parent b0e89df5c4
commit 2629237f9e
5 changed files with 750 additions and 84 deletions

View File

@@ -80,6 +80,37 @@ const SHOULD_USE_SECURE_COOKIES =
COOKIE_SECURE_MODE === 'true' ||
(process.env.NODE_ENV === 'production' && COOKIE_SECURE_MODE !== 'false');
function getResolvedStorageRoot() {
const configuredRoot = process.env.STORAGE_ROOT;
if (!configuredRoot) {
return path.join(__dirname, 'storage');
}
return path.isAbsolute(configuredRoot)
? configuredRoot
: path.resolve(__dirname, configuredRoot);
}
function getLocalUserStorageDir(userId) {
return path.join(getResolvedStorageRoot(), `user_${userId}`);
}
function syncLocalStorageUsageFromDisk(userId, currentUsed = 0) {
const userStorageDir = getLocalUserStorageDir(userId);
if (!fs.existsSync(userStorageDir)) {
fs.mkdirSync(userStorageDir, { recursive: true, mode: 0o755 });
}
const actualUsed = getUserDirectorySize(userStorageDir);
const normalizedCurrentUsed = Number(currentUsed) || 0;
if (actualUsed !== normalizedCurrentUsed) {
UserDB.update(userId, { local_storage_used: actualUsed });
console.log(`[本地存储] 已校准用户 ${userId} 本地用量: ${normalizedCurrentUsed} -> ${actualUsed}`);
}
return actualUsed;
}
if (process.env.NODE_ENV === 'production' && COOKIE_SECURE_MODE !== 'true') {
console.warn('[安全警告] 生产环境建议设置 COOKIE_SECURE=true以避免会话Cookie在HTTP下传输');
}
@@ -597,6 +628,74 @@ function normalizeOssQuota(rawQuota) {
return parsedQuota;
}
function normalizeDownloadTrafficQuota(rawQuota) {
const parsedQuota = Number(rawQuota);
if (!Number.isFinite(parsedQuota) || parsedQuota <= 0) {
return 0; // 0 表示不限流量
}
return Math.floor(parsedQuota);
}
function normalizeDownloadTrafficUsed(rawUsed, quota = 0) {
const parsedUsed = Number(rawUsed);
const normalizedUsed = Number.isFinite(parsedUsed) && parsedUsed > 0
? Math.floor(parsedUsed)
: 0;
if (quota > 0) {
return Math.min(normalizedUsed, quota);
}
return normalizedUsed;
}
function getDownloadTrafficState(user) {
const quota = normalizeDownloadTrafficQuota(user?.download_traffic_quota);
const used = normalizeDownloadTrafficUsed(user?.download_traffic_used, quota);
return {
quota,
used,
isUnlimited: quota <= 0,
remaining: quota > 0 ? Math.max(0, quota - used) : Number.POSITIVE_INFINITY
};
}
const applyDownloadTrafficUsageTransaction = db.transaction((userId, bytesToAdd) => {
const user = UserDB.findById(userId);
if (!user) {
return null;
}
const trafficState = getDownloadTrafficState(user);
if (bytesToAdd <= 0) {
return {
quota: trafficState.quota,
usedBefore: trafficState.used,
usedAfter: trafficState.used,
added: 0
};
}
const nextUsed = trafficState.isUnlimited
? trafficState.used + bytesToAdd
: Math.min(trafficState.quota, trafficState.used + bytesToAdd);
UserDB.update(userId, { download_traffic_used: nextUsed });
return {
quota: trafficState.quota,
usedBefore: trafficState.used,
usedAfter: nextUsed,
added: Math.max(0, nextUsed - trafficState.used)
};
});
function applyDownloadTrafficUsage(userId, bytesToAdd) {
const parsedBytes = Number(bytesToAdd);
if (!Number.isFinite(parsedBytes) || parsedBytes <= 0) {
return null;
}
return applyDownloadTrafficUsageTransaction(userId, Math.floor(parsedBytes));
}
// 构建用于存储客户端的用户对象(自动尝试解密 OSS Secret
function buildStorageUserContext(user, overrides = {}) {
if (!user) {
@@ -2052,6 +2151,11 @@ app.post('/api/login',
});
}
// 本地存储用量校准:防止目录被外部清理后仍显示旧用量
if ((user.current_storage_type || 'oss') === 'local') {
user.local_storage_used = syncLocalStorageUsageFromDisk(user.id, user.local_storage_used);
}
const token = generateToken(user);
const refreshToken = generateRefreshToken(user);
@@ -2104,6 +2208,11 @@ app.post('/api/login',
local_storage_used: user.local_storage_used || 0,
oss_storage_quota: normalizeOssQuota(user.oss_storage_quota),
storage_used: user.storage_used || 0,
download_traffic_quota: normalizeDownloadTrafficQuota(user.download_traffic_quota),
download_traffic_used: normalizeDownloadTrafficUsed(
user.download_traffic_used,
normalizeDownloadTrafficQuota(user.download_traffic_quota)
),
// OSS配置来源重要用于前端判断是否使用OSS直连上传
oss_config_source: SettingsDB.hasUnifiedOssConfig() ? 'unified' : (user.has_oss_config ? 'personal' : 'none')
}
@@ -2169,8 +2278,15 @@ app.post('/api/logout', (req, res) => {
// 获取当前用户信息
app.get('/api/user/profile', authMiddleware, (req, res) => {
const userPayload = { ...req.user };
// 本地存储用量校准:避免“有占用但目录为空”的显示错乱
if ((userPayload.current_storage_type || 'oss') === 'local') {
userPayload.local_storage_used = syncLocalStorageUsageFromDisk(userPayload.id, userPayload.local_storage_used);
}
// 不返回敏感信息(密码和 OSS 密钥)
const { password, oss_access_key_secret, ...safeUser } = req.user;
const { password, oss_access_key_secret, ...safeUser } = userPayload;
// 检查是否使用统一 OSS 配置
const hasUnifiedConfig = SettingsDB.hasUnifiedOssConfig();
@@ -2800,8 +2916,19 @@ app.post('/api/user/switch-storage',
}
}
// 更新存储类型
UserDB.update(req.user.id, { current_storage_type: storage_type });
// 更新存储类型(切到本地时同步校准本地用量)
const updates = { current_storage_type: storage_type };
if (storage_type === 'local') {
const latestUser = UserDB.findById(req.user.id);
if (latestUser) {
updates.local_storage_used = syncLocalStorageUsageFromDisk(
latestUser.id,
latestUser.local_storage_used
);
}
}
UserDB.update(req.user.id, updates);
res.json({
success: true,
@@ -3580,6 +3707,23 @@ app.get('/api/files/download-url', authMiddleware, async (req, res) => {
});
}
// 开启下载流量配额后,禁止发放直连 URL避免绕过后端计量
const latestUser = UserDB.findById(req.user.id);
if (!latestUser) {
return res.status(401).json({
success: false,
message: '用户不存在'
});
}
const trafficState = getDownloadTrafficState(latestUser);
if (!trafficState.isUnlimited) {
return res.status(403).json({
success: false,
message: '当前账号已启用下载流量限制,请使用系统下载入口'
});
}
// 检查用户是否配置了 OSS包括个人配置和系统级统一配置
const hasUnifiedConfig = SettingsDB.hasUnifiedOssConfig();
if (!req.user.has_oss_config && !hasUnifiedConfig) {
@@ -3788,6 +3932,15 @@ app.get('/api/files/download', authMiddleware, async (req, res) => {
const filePath = req.query.path;
let storage;
let storageEnded = false; // 防止重复关闭
let transferFinalized = false; // 防止重复结算
let downloadedBytes = 0;
let responseBodyStartSocketBytes = 0;
// Express 会将 HEAD 映射到 GET 处理器,这里显式拒绝,避免误触发下载计量
if (req.method === 'HEAD') {
res.setHeader('Allow', 'GET');
return res.status(405).end();
}
// 安全关闭存储连接的辅助函数
const safeEndStorage = async () => {
@@ -3801,6 +3954,39 @@ app.get('/api/files/download', authMiddleware, async (req, res) => {
}
};
const finalizeTransfer = async (reason = '') => {
if (transferFinalized) {
return;
}
transferFinalized = true;
try {
const socketBytesWritten = Number(res.socket?.bytesWritten);
const socketBodyBytes = Number.isFinite(socketBytesWritten) && socketBytesWritten > responseBodyStartSocketBytes
? Math.floor(socketBytesWritten - responseBodyStartSocketBytes)
: 0;
const billableBytes = socketBodyBytes > 0
? Math.min(downloadedBytes, socketBodyBytes)
: downloadedBytes;
if (billableBytes > 0) {
const usageResult = applyDownloadTrafficUsage(req.user.id, billableBytes);
if (usageResult) {
const quotaText = usageResult.quota > 0 ? formatFileSize(usageResult.quota) : '不限';
console.log(
`[下载流量] 用户 ${req.user.id} 新增 ${formatFileSize(usageResult.added)}` +
`累计 ${formatFileSize(usageResult.usedAfter)} / ${quotaText} ` +
`(reason=${reason || 'unknown'}, streamed=${downloadedBytes}, socket=${socketBodyBytes})`
);
}
}
} catch (error) {
console.error(`[下载流量] 结算失败: user=${req.user.id}, bytes=${downloadedBytes}`, error);
}
await safeEndStorage();
};
if (!filePath) {
return res.status(400).json({
success: false,
@@ -3818,26 +4004,66 @@ app.get('/api/files/download', authMiddleware, async (req, res) => {
}
try {
const latestUser = UserDB.findById(req.user.id);
if (!latestUser) {
return res.status(401).json({
success: false,
message: '用户不存在'
});
}
const trafficState = getDownloadTrafficState(latestUser);
// 使用统一存储接口
const { StorageInterface } = require('./storage');
const storageInterface = new StorageInterface(req.user);
storage = await storageInterface.connect();
// 获取文件名
const fileName = filePath.split('/').pop();
const fileName = normalizedPath.split('/').pop() || 'download.bin';
// 先获取文件信息(获取文件大小)
const fileStats = await storage.stat(filePath);
const fileSize = fileStats.size;
const fileStats = await storage.stat(normalizedPath);
const fileSize = Math.max(0, Number(fileStats?.size) || 0);
console.log('[下载] 文件: ' + fileName + ', 大小: ' + fileSize + ' 字节');
if (!trafficState.isUnlimited && fileSize > trafficState.remaining) {
await safeEndStorage();
return res.status(403).json({
success: false,
message: `下载流量不足:文件 ${formatFileSize(fileSize)},剩余 ${formatFileSize(trafficState.remaining)}`
});
}
// 设置响应头(包含文件大小,浏览器可显示下载进度)
res.setHeader('Content-Type', 'application/octet-stream');
res.setHeader('Content-Length', fileSize);
// 关闭 Nginx 代理缓冲,避免上游提前读完整文件导致流量计量失真
res.setHeader('X-Accel-Buffering', 'no');
res.setHeader('Content-Disposition', 'attachment; filename="' + encodeURIComponent(fileName) + '"; filename*=UTF-8\'\'' + encodeURIComponent(fileName));
if (typeof res.flushHeaders === 'function') {
res.flushHeaders();
}
responseBodyStartSocketBytes = Number(res.socket?.bytesWritten) || 0;
// 创建文件流并传输(流式下载,服务器不保存临时文件)
const stream = await storage.createReadStream(filePath);
const stream = await storage.createReadStream(normalizedPath);
stream.on('data', (chunk) => {
if (!chunk) return;
downloadedBytes += Buffer.isBuffer(chunk) ? chunk.length : Buffer.byteLength(chunk);
});
res.on('finish', () => {
finalizeTransfer('finish').catch(err => {
console.error('下载完成后资源释放失败:', err);
});
});
res.on('close', () => {
finalizeTransfer('close').catch(err => {
console.error('下载连接关闭后资源释放失败:', err);
});
});
stream.on('error', (error) => {
console.error('文件流错误:', error);
@@ -3847,14 +4073,9 @@ app.get('/api/files/download', authMiddleware, async (req, res) => {
message: getSafeErrorMessage(error, '下载文件失败,请稍后重试', '下载文件')
});
}
// 发生错误时关闭存储连接
safeEndStorage();
});
// 在传输完成后关闭存储连接
stream.on('close', () => {
console.log('[下载] 文件传输完成,关闭存储连接');
safeEndStorage();
finalizeTransfer('stream_error').catch(err => {
console.error('流错误后资源释放失败:', err);
});
});
stream.pipe(res);
@@ -3863,7 +4084,7 @@ app.get('/api/files/download', authMiddleware, async (req, res) => {
console.error('下载文件失败:', error);
// 如果stream还未创建或发生错误关闭storage连接
await safeEndStorage();
await finalizeTransfer('catch_error');
if (!res.headersSent) {
res.status(500).json({
success: false,
@@ -4783,9 +5004,10 @@ app.post('/api/share/:code/download-url', shareRateLimitMiddleware, async (req,
}
const storageType = share.storage_type || 'oss';
const ownerTrafficState = getDownloadTrafficState(shareOwner);
// 本地存储模式:返回后端下载 URL短期 token避免在 URL 中传密码
if (storageType !== 'oss') {
// 本地存储,或分享者启用了下载流量上限:统一走后端下载(便于计量
if (storageType !== 'oss' || !ownerTrafficState.isUnlimited) {
let downloadUrl = `${getSecureBaseUrl(req)}/api/share/${code}/download-file?path=${encodeURIComponent(normalizedFilePath)}`;
if (share.share_password) {
@@ -4800,7 +5022,8 @@ app.post('/api/share/:code/download-url', shareRateLimitMiddleware, async (req,
return res.json({
success: true,
downloadUrl,
direct: false
direct: false,
quotaLimited: !ownerTrafficState.isUnlimited
});
}
@@ -4845,8 +5068,7 @@ app.post('/api/share/:code/download-url', shareRateLimitMiddleware, async (req,
}
});
// 分享文件下载支持本地存储公开API需要分享码和密码验证
// 注意OSS 模式请使用 /api/share/:code/download-url 获取直连下载链接
// 分享文件下载(支持本地存储和 OSS,公开 API需要分享码和密码验证
app.get('/api/share/:code/download-file', shareRateLimitMiddleware, async (req, res) => {
const { code } = req.params;
const rawFilePath = typeof req.query?.path === 'string' ? req.query.path : '';
@@ -4854,6 +5076,16 @@ app.get('/api/share/:code/download-file', shareRateLimitMiddleware, async (req,
const filePath = normalizeVirtualPath(rawFilePath);
let storage;
let storageEnded = false; // 防止重复关闭
let transferFinalized = false; // 防止重复结算
let downloadedBytes = 0;
let responseBodyStartSocketBytes = 0;
let shareOwnerId = null;
// Express 会将 HEAD 映射到 GET 处理器,这里显式拒绝,避免误触发下载计量
if (req.method === 'HEAD') {
res.setHeader('Allow', 'GET');
return res.status(405).end();
}
// 安全关闭存储连接的辅助函数
const safeEndStorage = async () => {
@@ -4867,6 +5099,39 @@ app.get('/api/share/:code/download-file', shareRateLimitMiddleware, async (req,
}
};
const finalizeTransfer = async (reason = '') => {
if (transferFinalized) {
return;
}
transferFinalized = true;
try {
if (shareOwnerId && downloadedBytes > 0) {
const socketBytesWritten = Number(res.socket?.bytesWritten);
const socketBodyBytes = Number.isFinite(socketBytesWritten) && socketBytesWritten > responseBodyStartSocketBytes
? Math.floor(socketBytesWritten - responseBodyStartSocketBytes)
: 0;
const billableBytes = socketBodyBytes > 0
? Math.min(downloadedBytes, socketBodyBytes)
: downloadedBytes;
const usageResult = applyDownloadTrafficUsage(shareOwnerId, billableBytes);
if (usageResult) {
const quotaText = usageResult.quota > 0 ? formatFileSize(usageResult.quota) : '不限';
console.log(
`[分享下载流量] 用户 ${shareOwnerId} 新增 ${formatFileSize(usageResult.added)}` +
`累计 ${formatFileSize(usageResult.usedAfter)} / ${quotaText} ` +
`(reason=${reason || 'unknown'}, streamed=${downloadedBytes}, socket=${socketBodyBytes})`
);
}
}
} catch (error) {
console.error(`[分享下载流量] 结算失败: user=${shareOwnerId}, bytes=${downloadedBytes}`, error);
}
await safeEndStorage();
};
if (!filePath) {
return res.status(400).json({
success: false,
@@ -4935,6 +5200,8 @@ app.get('/api/share/:code/download-file', shareRateLimitMiddleware, async (req,
message: '分享者不存在'
});
}
shareOwnerId = shareOwner.id;
const ownerTrafficState = getDownloadTrafficState(shareOwner);
// 使用统一存储接口根据分享的storage_type选择存储后端
// 注意:必须使用分享创建时记录的 storage_type而不是分享者当前的存储类型
@@ -4959,17 +5226,48 @@ app.get('/api/share/:code/download-file', shareRateLimitMiddleware, async (req,
const fileSize = fileStats.size;
console.log(`[分享下载] 文件: ${fileName}, 大小: ${fileSize} 字节`);
if (!ownerTrafficState.isUnlimited && fileSize > ownerTrafficState.remaining) {
await safeEndStorage();
return res.status(403).json({
success: false,
message: `分享者下载流量不足:文件 ${formatFileSize(fileSize)},剩余 ${formatFileSize(ownerTrafficState.remaining)}`
});
}
// 增加下载次数
ShareDB.incrementDownloadCount(code);
// 设置响应头(包含文件大小,浏览器可显示下载进度)
res.setHeader('Content-Type', 'application/octet-stream');
res.setHeader('Content-Length', fileSize);
// 关闭 Nginx 代理缓冲,减少代理预读导致的计量偏差
res.setHeader('X-Accel-Buffering', 'no');
res.setHeader('Content-Disposition', `attachment; filename="${encodeURIComponent(fileName)}"; filename*=UTF-8''${encodeURIComponent(fileName)}`);
if (typeof res.flushHeaders === 'function') {
res.flushHeaders();
}
responseBodyStartSocketBytes = Number(res.socket?.bytesWritten) || 0;
// 创建文件流并传输(流式下载,服务器不保存临时文件)
const stream = await storage.createReadStream(filePath);
stream.on('data', (chunk) => {
if (!chunk) return;
downloadedBytes += Buffer.isBuffer(chunk) ? chunk.length : Buffer.byteLength(chunk);
});
res.on('finish', () => {
finalizeTransfer('finish').catch(err => {
console.error('分享下载完成后资源释放失败:', err);
});
});
res.on('close', () => {
finalizeTransfer('close').catch(err => {
console.error('分享下载连接关闭后资源释放失败:', err);
});
});
stream.on('error', (error) => {
console.error('文件流错误:', error);
if (!res.headersSent) {
@@ -4978,14 +5276,9 @@ app.get('/api/share/:code/download-file', shareRateLimitMiddleware, async (req,
message: getSafeErrorMessage(error, '下载文件失败,请稍后重试', '分享下载')
});
}
// 发生错误时关闭存储连接
safeEndStorage();
});
// 在传输完成后关闭存储连接
stream.on('close', () => {
console.log('[分享下载] 文件传输完成,关闭存储连接');
safeEndStorage();
finalizeTransfer('stream_error').catch(err => {
console.error('分享下载流错误后资源释放失败:', err);
});
});
stream.pipe(res);
@@ -4998,8 +5291,8 @@ app.get('/api/share/:code/download-file', shareRateLimitMiddleware, async (req,
message: getSafeErrorMessage(error, '下载文件失败,请稍后重试', '分享下载')
});
}
// 如果发生错误,关闭存储连接
await safeEndStorage();
// 如果发生错误,结算并关闭存储连接
await finalizeTransfer('catch_error');
}
});
@@ -5423,7 +5716,7 @@ app.get('/api/admin/health-check', authMiddleware, adminMiddleware, async (req,
});
// 7. 存储目录检查
const storageRoot = process.env.STORAGE_ROOT || path.join(__dirname, 'storage');
const storageRoot = getResolvedStorageRoot();
let storageStatus = 'pass';
let storageMessage = `存储目录正常: ${storageRoot}`;
try {
@@ -5629,7 +5922,7 @@ app.post('/api/admin/wal-checkpoint',
app.get('/api/admin/storage-stats', authMiddleware, adminMiddleware, async (req, res) => {
try {
// 获取本地存储目录(与 storage.js 保持一致)
const localStorageDir = process.env.STORAGE_ROOT || path.join(__dirname, 'storage');
const localStorageDir = getResolvedStorageRoot();
// 确保存储目录存在
if (!fs.existsSync(localStorageDir)) {
@@ -5740,7 +6033,12 @@ app.get('/api/admin/users', authMiddleware, adminMiddleware, (req, res) => {
local_storage_quota: u.local_storage_quota || DEFAULT_LOCAL_STORAGE_QUOTA_BYTES,
local_storage_used: u.local_storage_used || 0,
oss_storage_quota: normalizeOssQuota(u.oss_storage_quota),
storage_used: u.storage_used || 0
storage_used: u.storage_used || 0,
download_traffic_quota: normalizeDownloadTrafficQuota(u.download_traffic_quota),
download_traffic_used: normalizeDownloadTrafficUsed(
u.download_traffic_used,
normalizeDownloadTrafficQuota(u.download_traffic_quota)
)
}))
});
} catch (error) {
@@ -6209,7 +6507,7 @@ app.delete('/api/admin/users/:id',
// 1. 删除本地存储文件(如果用户使用了本地存储)
const storagePermission = user.storage_permission || 'oss_only';
if (storagePermission === 'local_only' || storagePermission === 'user_choice') {
const storageRoot = process.env.STORAGE_ROOT || path.join(__dirname, 'storage');
const storageRoot = getResolvedStorageRoot();
const userStorageDir = path.join(storageRoot, `user_${userId}`);
if (fs.existsSync(userStorageDir)) {
@@ -6295,6 +6593,10 @@ app.delete('/api/admin/users/:id',
// 辅助函数:计算目录大小
function getUserDirectorySize(dirPath) {
if (!dirPath || !fs.existsSync(dirPath)) {
return 0;
}
let totalSize = 0;
function calculateSize(currentPath) {
@@ -6325,7 +6627,8 @@ app.post('/api/admin/users/:id/storage-permission',
[
body('storage_permission').isIn(['local_only', 'oss_only', 'user_choice']).withMessage('无效的存储权限'),
body('local_storage_quota').optional({ nullable: true }).isInt({ min: 1048576, max: 10995116277760 }).withMessage('本地配额必须在 1MB 到 10TB 之间'),
body('oss_storage_quota').optional({ nullable: true }).isInt({ min: 1048576, max: 10995116277760 }).withMessage('OSS配额必须在 1MB 到 10TB 之间')
body('oss_storage_quota').optional({ nullable: true }).isInt({ min: 1048576, max: 10995116277760 }).withMessage('OSS配额必须在 1MB 到 10TB 之间'),
body('download_traffic_quota').optional({ nullable: true }).isInt({ min: 0, max: 10995116277760 }).withMessage('下载流量配额必须在 0 到 10TB 之间0表示不限')
],
(req, res) => {
const errors = validationResult(req);
@@ -6338,7 +6641,7 @@ app.post('/api/admin/users/:id/storage-permission',
try {
const { id } = req.params;
const { storage_permission, local_storage_quota, oss_storage_quota } = req.body;
const { storage_permission, local_storage_quota, oss_storage_quota, download_traffic_quota } = req.body;
// 参数验证:验证 ID 格式
const userId = parseInt(id, 10);
@@ -6361,6 +6664,11 @@ app.post('/api/admin/users/:id/storage-permission',
updates.oss_storage_quota = parseInt(oss_storage_quota, 10);
}
// 如果提供了下载流量配额更新配额单位字节0 表示不限)
if (download_traffic_quota !== undefined && download_traffic_quota !== null) {
updates.download_traffic_quota = parseInt(download_traffic_quota, 10);
}
// 根据权限设置自动调整存储类型
const user = UserDB.findById(userId);
if (!user) {
@@ -6370,6 +6678,17 @@ app.post('/api/admin/users/:id/storage-permission',
});
}
// 如果管理员把配额调小,已用流量不能超过新配额
if (
updates.download_traffic_quota !== undefined &&
updates.download_traffic_quota > 0
) {
updates.download_traffic_used = normalizeDownloadTrafficUsed(
user.download_traffic_used,
updates.download_traffic_quota
);
}
if (storage_permission === 'local_only') {
updates.current_storage_type = 'local';
} else if (storage_permission === 'oss_only') {