fix: 内存溢出与任务调度优化

This commit is contained in:
2025-12-13 17:36:02 +08:00
parent 9e761140c1
commit d7d878dc08
5 changed files with 1128 additions and 480 deletions

View File

@@ -1623,6 +1623,15 @@ class EmailQueue:
def _process_task(self, task: Dict):
"""处理邮件任务"""
try:
func = task.get('callable')
if callable(func):
args = task.get('args') or ()
kwargs = task.get('kwargs') or {}
result = func(*args, **kwargs)
if task.get('callback'):
task['callback'](result)
return
result = send_email(
to_email=task['to_email'],
subject=task['subject'],
@@ -1670,6 +1679,20 @@ class EmailQueue:
print("[邮件服务] 邮件队列已满")
return False
def enqueue_callable(self, func: Callable, args=None, kwargs=None, callback: Callable = None) -> bool:
"""将可调用任务加入队列(用于复杂邮件/打包等逻辑异步化)"""
try:
self.queue.put({
'callable': func,
'args': args or (),
'kwargs': kwargs or {},
'callback': callback
}, timeout=5)
return True
except queue.Full:
print("[邮件服务] 邮件队列已满")
return False
@property
def pending_count(self) -> int:
"""队列中待处理的任务数"""
@@ -1958,14 +1981,14 @@ def send_task_complete_email_async(
log_callback: Callable = None
):
"""异步发送任务完成通知邮件"""
import threading
thread = threading.Thread(
target=send_task_complete_email,
queue = get_email_queue()
ok = queue.enqueue_callable(
send_task_complete_email,
args=(user_id, email, username, account_name, browse_type,
total_items, total_attachments, screenshot_path, log_callback),
daemon=True
)
thread.start()
if (not ok) and log_callback:
log_callback("[邮件] 邮件队列已满,任务通知未发送")
def send_batch_task_complete_email(
@@ -2058,32 +2081,55 @@ def send_batch_task_complete_email(
</html>
"""
# 收集所有截图文件
screenshot_files = []
# 收集可用截图文件路径(避免把所有图片一次性读入内存)
screenshot_paths = []
for s in screenshots:
if s.get('path') and os.path.exists(s['path']):
try:
with open(s['path'], 'rb') as f:
screenshot_files.append({
'filename': f"{s.get('account_name', 'screenshot')}_{os.path.basename(s['path'])}",
'data': f.read()
})
except Exception as e:
print(f"[邮件] 读取截图文件失败: {e}")
path = s.get('path')
if path and os.path.exists(path):
arcname = f"{s.get('account_name', 'screenshot')}_{os.path.basename(path)}"
screenshot_paths.append((path, arcname))
# 如果有截图,打包ZIP
# 如果有截图,优先落盘打包ZIP,再按大小决定是否附加(降低内存峰值)
zip_data = None
zip_filename = None
if screenshot_files:
attachment_note = ""
if screenshot_paths:
import tempfile
zip_path = None
try:
zip_buffer = BytesIO()
with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zf:
for sf in screenshot_files:
zf.writestr(sf['filename'], sf['data'])
zip_data = zip_buffer.getvalue()
zip_filename = f"screenshots_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip"
with tempfile.NamedTemporaryFile(prefix="screenshots_", suffix=".zip", delete=False) as tmp:
zip_path = tmp.name
with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf:
for file_path, arcname in screenshot_paths:
try:
zf.write(file_path, arcname=arcname)
except Exception as e:
print(f"[邮件] 写入ZIP失败: {e}")
zip_size = os.path.getsize(zip_path) if zip_path and os.path.exists(zip_path) else 0
if zip_size <= 0:
attachment_note = "本次无可用截图文件(可能截图失败或文件不存在)。"
elif zip_size > MAX_ATTACHMENT_SIZE:
attachment_note = f"截图打包文件过大({zip_size} bytes本次不附加附件。"
else:
with open(zip_path, 'rb') as f:
zip_data = f.read()
zip_filename = f"screenshots_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip"
attachment_note = "截图已打包为ZIP附件请查收。"
except Exception as e:
print(f"[邮件] 打包截图失败: {e}")
attachment_note = "截图打包失败,本次不附加附件。"
finally:
if zip_path and os.path.exists(zip_path):
try:
os.remove(zip_path)
except Exception:
pass
else:
attachment_note = "本次无可用截图文件(可能截图失败或未启用截图)。"
# 将附件说明写入邮件内容
html_content = html_content.replace("截图已打包为ZIP附件请查收。", attachment_note)
# 发送邮件
attachments = []
@@ -2132,13 +2178,13 @@ def send_batch_task_complete_email_async(
screenshots: List[Dict[str, Any]]
):
"""异步发送批次任务完成通知邮件"""
import threading
thread = threading.Thread(
target=send_batch_task_complete_email,
queue = get_email_queue()
ok = queue.enqueue_callable(
send_batch_task_complete_email,
args=(user_id, email, username, schedule_name, browse_type, screenshots),
daemon=True
)
thread.start()
if not ok:
print("[邮件] 邮件队列已满,批次任务邮件未发送")
# ============ 初始化 ============