fix: 内存溢出与任务调度优化

This commit is contained in:
2025-12-13 17:36:02 +08:00
parent 9e761140c1
commit d7d878dc08
5 changed files with 1128 additions and 480 deletions

View File

@@ -12,6 +12,7 @@ import time
import json
import threading
import atexit
import weakref
from typing import Optional, Callable
from dataclasses import dataclass
from app_config import get_config
@@ -28,6 +29,20 @@ else:
# 获取配置
config = get_config()
_playwright_automation_instances: "weakref.WeakSet[PlaywrightAutomation]" = weakref.WeakSet()
def _cleanup_playwright_automation_instances():
"""进程退出时清理残留的自动化实例弱引用不阻止GC"""
for inst in list(_playwright_automation_instances):
try:
inst._force_cleanup()
except Exception:
pass
atexit.register(_cleanup_playwright_automation_instances)
@dataclass
class BrowseResult:
@@ -151,8 +166,7 @@ class PlaywrightAutomation:
self._closed = False # 防止重复关闭
self._lock = threading.Lock() # Bug #13 fix: 保护浏览器资源访问
# 注册退出清理函数,确保进程异常退出时也能关闭浏览器
atexit.register(self._cleanup_on_exit)
_playwright_automation_instances.add(self)
def log(self, message: str):
"""记录日志"""
@@ -215,6 +229,52 @@ class PlaywrightAutomation:
except Exception as e:
self.log(f"加载cookies失败: {e}")
return False
def load_cookies_into_current_browser(self, username: str) -> bool:
"""在“已连接的现有 browser”上加载 cookies 创建 context用于浏览器池复用"""
import os
if not self.browser or not self.browser.is_connected():
return False
cookies_path = self.get_cookies_path(username)
if not os.path.exists(cookies_path):
return False
try:
# 检查cookies文件是否过期24小时
import time as time_module
file_age = time_module.time() - os.path.getmtime(cookies_path)
if file_age > 24 * 3600:
self.log("Cookies已过期需要重新登录")
os.remove(cookies_path)
return False
with open(cookies_path, 'r', encoding='utf-8') as f:
storage = json.load(f)
context_options = {
'viewport': {'width': 1920, 'height': 1080},
'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
'device_scale_factor': 2,
'storage_state': storage
}
self.context = self.browser.new_context(**context_options)
self.context.set_default_timeout(config.DEFAULT_TIMEOUT)
self.context.set_default_navigation_timeout(config.PAGE_LOAD_TIMEOUT)
self.page = self.context.new_page()
self.main_page = self.page
return True
except Exception as e:
self.log(f"加载cookies失败: {e}")
try:
if self.context:
self.context.close()
except Exception:
pass
self.context = None
self.page = None
self.main_page = None
return False
def check_login_state(self) -> bool:
"""检查当前是否处于登录状态"""
@@ -235,9 +295,24 @@ class PlaywrightAutomation:
def quick_login(self, username: str, password: str, remember: bool = True):
"""快速登录 - 使用池中浏览器时直接登录否则尝试cookies"""
# 如果已有浏览器实例(从池中获取),直接使用该浏览器登录
# 不尝试加载cookies因为load_cookies会创建新浏览器覆盖池中的
# 如果已有浏览器实例(从池中获取),优先尝试复用cookies避免重复登录/减少耗时)
if self.browser and self.browser.is_connected():
if self.load_cookies_into_current_browser(username):
self.log("使用池中浏览器,尝试使用已保存的登录态...")
if self.check_login_state():
self.log("✓ 登录态有效,跳过登录")
return {"success": True, "message": "使用已保存的登录态", "used_cookies": True}
else:
self.log("登录态已失效,重新登录")
try:
if self.context:
self.context.close()
except Exception:
pass
self.context = None
self.page = None
self.main_page = None
self.log("使用池中浏览器,直接登录")
result = self.login(username, password, remember)
if result.get('success'):