896 lines
30 KiB
Python
896 lines
30 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import threading
|
||
import time
|
||
|
||
import database
|
||
import email_service
|
||
from api_browser import APIBrowser
|
||
from app_config import get_config
|
||
from app_logger import LoggerAdapter, get_logger
|
||
from services.checkpoints import get_checkpoint_mgr
|
||
from services.client_log import log_to_client
|
||
from services.proxy import get_proxy_from_api
|
||
from services.runtime import get_socketio
|
||
from services.screenshots import take_screenshot_for_account
|
||
from services.state import (
|
||
safe_get_account,
|
||
safe_remove_task,
|
||
safe_remove_task_status,
|
||
safe_set_task_status,
|
||
safe_update_task_status,
|
||
)
|
||
from services.task_batches import _batch_task_record_result, _get_batch_id_from_source
|
||
from services.task_scheduler import TaskScheduler
|
||
from task_checkpoint import TaskStage
|
||
|
||
logger = get_logger("app")
|
||
config = get_config()
|
||
|
||
# 并发默认值（启动后会由系统配置覆盖并调用 update_limits）
|
||
max_concurrent_per_account = config.MAX_CONCURRENT_PER_ACCOUNT
|
||
max_concurrent_global = config.MAX_CONCURRENT_GLOBAL
|
||
_SOURCE_UNSET = object()
|
||
|
||
|
||
def _emit(event: str, data: object, *, room: str | None = None) -> None:
|
||
try:
|
||
socketio = get_socketio()
|
||
socketio.emit(event, data, room=room)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
_task_scheduler = None
|
||
_task_scheduler_lock = threading.Lock()
|
||
|
||
|
||
def get_task_scheduler() -> TaskScheduler:
|
||
"""获取全局任务调度器（单例）"""
|
||
global _task_scheduler
|
||
with _task_scheduler_lock:
|
||
if _task_scheduler is None:
|
||
try:
|
||
max_queue_size = int(os.environ.get("TASK_QUEUE_MAXSIZE", "1000"))
|
||
except Exception:
|
||
max_queue_size = 1000
|
||
_task_scheduler = TaskScheduler(
|
||
max_global=max_concurrent_global,
|
||
max_per_user=max_concurrent_per_account,
|
||
max_queue_size=max_queue_size,
|
||
run_task_fn=run_task,
|
||
)
|
||
return _task_scheduler
|
||
|
||
|
||
def submit_account_task(
|
||
user_id: int,
|
||
account_id: str,
|
||
browse_type: str,
|
||
enable_screenshot: bool = True,
|
||
source: str = "manual",
|
||
retry_count: int = 0,
|
||
done_callback=None,
|
||
):
|
||
"""统一入口：提交账号任务进入队列"""
|
||
account = safe_get_account(user_id, account_id)
|
||
if not account:
|
||
return False, "账号不存在"
|
||
if getattr(account, "is_running", False):
|
||
return False, "任务已在运行中"
|
||
|
||
try:
|
||
is_vip_user = bool(database.is_user_vip(user_id))
|
||
except Exception:
|
||
is_vip_user = False
|
||
|
||
account.is_running = True
|
||
account.should_stop = False
|
||
account.status = "排队中" + (" (VIP)" if is_vip_user else "")
|
||
|
||
safe_set_task_status(
|
||
account_id,
|
||
{
|
||
"user_id": user_id,
|
||
"username": account.username,
|
||
"status": "排队中",
|
||
"detail_status": "等待资源" + (" [VIP优先]" if is_vip_user else ""),
|
||
"browse_type": browse_type,
|
||
"start_time": time.time(),
|
||
"source": source,
|
||
"progress": {"items": 0, "attachments": 0},
|
||
"is_vip": is_vip_user,
|
||
},
|
||
)
|
||
_emit("account_update", account.to_dict(), room=f"user_{user_id}")
|
||
|
||
scheduler = get_task_scheduler()
|
||
ok, message = scheduler.submit_task(
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
browse_type=browse_type,
|
||
enable_screenshot=enable_screenshot,
|
||
source=source,
|
||
retry_count=retry_count,
|
||
is_vip=is_vip_user,
|
||
done_callback=done_callback,
|
||
)
|
||
if not ok:
|
||
account.is_running = False
|
||
account.status = "未开始"
|
||
safe_remove_task_status(account_id)
|
||
_emit("account_update", account.to_dict(), room=f"user_{user_id}")
|
||
return False, message
|
||
|
||
log_to_client(message + (" [VIP优先]" if is_vip_user else ""), user_id, account_id)
|
||
return True, message
|
||
|
||
|
||
def _account_display_name(account) -> str:
|
||
return account.remark if account.remark else account.username
|
||
|
||
|
||
def _record_batch_result(batch_id, account, total_items: int, total_attachments: int) -> None:
|
||
if not batch_id:
|
||
return
|
||
_batch_task_record_result(
|
||
batch_id=batch_id,
|
||
account_name=_account_display_name(account),
|
||
screenshot_path=None,
|
||
total_items=total_items,
|
||
total_attachments=total_attachments,
|
||
)
|
||
|
||
|
||
def _close_account_automation(account, *, on_error=None) -> bool:
|
||
automation = getattr(account, "automation", None)
|
||
if not automation:
|
||
return False
|
||
|
||
closed = False
|
||
try:
|
||
automation.close()
|
||
closed = True
|
||
except Exception as e:
|
||
if on_error:
|
||
try:
|
||
on_error(e)
|
||
except Exception:
|
||
pass
|
||
finally:
|
||
account.automation = None
|
||
|
||
return closed
|
||
|
||
|
||
def _create_task_log(
|
||
*,
|
||
user_id: int,
|
||
account_id: str,
|
||
account,
|
||
browse_type: str,
|
||
status: str,
|
||
total_items: int,
|
||
total_attachments: int,
|
||
error_message: str,
|
||
duration: int,
|
||
source=_SOURCE_UNSET,
|
||
) -> None:
|
||
payload = {
|
||
"user_id": user_id,
|
||
"account_id": account_id,
|
||
"username": account.username,
|
||
"browse_type": browse_type,
|
||
"status": status,
|
||
"total_items": total_items,
|
||
"total_attachments": total_attachments,
|
||
"error_message": error_message,
|
||
"duration": duration,
|
||
}
|
||
if source is not _SOURCE_UNSET:
|
||
payload["source"] = source
|
||
database.create_task_log(**payload)
|
||
|
||
|
||
def _handle_stop_requested(
|
||
*,
|
||
account,
|
||
user_id: int,
|
||
account_id: str,
|
||
batch_id,
|
||
remove_task_status: bool = False,
|
||
record_batch: bool = False,
|
||
) -> bool:
|
||
if not account.should_stop:
|
||
return False
|
||
|
||
log_to_client("任务已取消", user_id, account_id)
|
||
account.status = "已停止"
|
||
account.is_running = False
|
||
if remove_task_status:
|
||
safe_remove_task_status(account_id)
|
||
_emit("account_update", account.to_dict(), room=f"user_{user_id}")
|
||
|
||
if record_batch and batch_id:
|
||
_record_batch_result(batch_id=batch_id, account=account, total_items=0, total_attachments=0)
|
||
return True
|
||
|
||
|
||
def _resolve_proxy_config(user_id: int, account_id: str, account):
|
||
proxy_config = None
|
||
system_config = database.get_system_config()
|
||
if system_config.get("proxy_enabled") != 1:
|
||
return proxy_config
|
||
|
||
proxy_api_url = system_config.get("proxy_api_url", "").strip()
|
||
if not proxy_api_url:
|
||
log_to_client("⚠ 代理已启用但未配置API地址", user_id, account_id)
|
||
return proxy_config
|
||
|
||
log_to_client("正在获取代理IP...", user_id, account_id)
|
||
proxy_server = get_proxy_from_api(proxy_api_url, max_retries=3)
|
||
if proxy_server:
|
||
proxy_config = {"server": proxy_server}
|
||
log_to_client(f"[OK] 将使用代理: {proxy_server}", user_id, account_id)
|
||
account.proxy_config = proxy_config
|
||
else:
|
||
log_to_client("✗ 代理获取失败，将不使用代理继续", user_id, account_id)
|
||
return proxy_config
|
||
|
||
|
||
def _refresh_account_remark(api_browser, account, user_id: int, account_id: str) -> None:
|
||
if account.remark:
|
||
return
|
||
try:
|
||
real_name = api_browser.get_real_name()
|
||
if not real_name:
|
||
return
|
||
account.remark = real_name
|
||
database.update_account_remark(account_id, real_name)
|
||
_emit("account_update", account.to_dict(), room=f"user_{user_id}")
|
||
logger.info(f"[自动备注] 账号 {account.username} 自动设置备注为: {real_name}")
|
||
except Exception as e:
|
||
logger.warning(f"[自动备注] 获取姓名失败: {e}")
|
||
|
||
|
||
def _handle_login_failed(
|
||
*,
|
||
account,
|
||
user_id: int,
|
||
account_id: str,
|
||
browse_type: str,
|
||
source: str,
|
||
task_start_time: float,
|
||
task_id: str,
|
||
checkpoint_mgr,
|
||
time_module,
|
||
) -> None:
|
||
error_message = "登录失败"
|
||
log_to_client(f"❌ {error_message}", user_id, account_id)
|
||
|
||
is_suspended = database.increment_account_login_fail(account_id, error_message)
|
||
if is_suspended:
|
||
log_to_client("⚠ 该账号连续3次密码错误，已自动暂停", user_id, account_id)
|
||
log_to_client("请在前台修改密码后才能继续使用", user_id, account_id)
|
||
|
||
retry_action = checkpoint_mgr.record_error(task_id, error_message)
|
||
if retry_action == "paused":
|
||
logger.warning(f"[断点] 任务 {task_id} 已暂停(登录失败)")
|
||
|
||
account.status = "登录失败"
|
||
account.is_running = False
|
||
_create_task_log(
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
account=account,
|
||
browse_type=browse_type,
|
||
status="failed",
|
||
total_items=0,
|
||
total_attachments=0,
|
||
error_message=error_message,
|
||
duration=int(time_module.time() - task_start_time),
|
||
source=source,
|
||
)
|
||
_emit("account_update", account.to_dict(), room=f"user_{user_id}")
|
||
|
||
|
||
def _execute_single_attempt(
|
||
*,
|
||
account,
|
||
user_id: int,
|
||
account_id: str,
|
||
browse_type: str,
|
||
source: str,
|
||
checkpoint_mgr,
|
||
task_id: str,
|
||
task_start_time: float,
|
||
time_module,
|
||
):
|
||
proxy_config = _resolve_proxy_config(user_id=user_id, account_id=account_id, account=account)
|
||
|
||
checkpoint_mgr.update_stage(task_id, TaskStage.STARTING, progress_percent=10)
|
||
|
||
def custom_log(message: str):
|
||
log_to_client(message, user_id, account_id)
|
||
|
||
log_to_client("开始登录...", user_id, account_id)
|
||
safe_update_task_status(account_id, {"detail_status": "正在登录"})
|
||
checkpoint_mgr.update_stage(task_id, TaskStage.LOGGING_IN, progress_percent=25)
|
||
|
||
with APIBrowser(log_callback=custom_log, proxy_config=proxy_config) as api_browser:
|
||
if not api_browser.login(account.username, account.password):
|
||
_handle_login_failed(
|
||
account=account,
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
browse_type=browse_type,
|
||
source=source,
|
||
task_start_time=task_start_time,
|
||
task_id=task_id,
|
||
checkpoint_mgr=checkpoint_mgr,
|
||
time_module=time_module,
|
||
)
|
||
return "login_failed", None
|
||
|
||
log_to_client("[OK] 首次登录成功，刷新登录时间...", user_id, account_id)
|
||
|
||
if api_browser.login(account.username, account.password):
|
||
log_to_client("[OK] 二次登录成功！", user_id, account_id)
|
||
else:
|
||
log_to_client("⚠ 二次登录失败，继续使用首次登录状态", user_id, account_id)
|
||
|
||
api_browser.save_cookies_for_screenshot(account.username)
|
||
database.reset_account_login_status(account_id)
|
||
_refresh_account_remark(api_browser, account, user_id, account_id)
|
||
|
||
safe_update_task_status(account_id, {"detail_status": "正在浏览"})
|
||
log_to_client(f"开始浏览 '{browse_type}' 内容...", user_id, account_id)
|
||
account.total_items = 0
|
||
safe_update_task_status(account_id, {"progress": {"items": 0, "attachments": 0}})
|
||
|
||
def should_stop():
|
||
return account.should_stop
|
||
|
||
def on_browse_progress(progress: dict):
|
||
try:
|
||
total_items = int(progress.get("total_items") or 0)
|
||
browsed_items = int(progress.get("browsed_items") or 0)
|
||
if total_items > 0:
|
||
account.total_items = total_items
|
||
safe_update_task_status(account_id, {"progress": {"items": browsed_items, "attachments": 0}})
|
||
except Exception:
|
||
pass
|
||
|
||
checkpoint_mgr.update_stage(task_id, TaskStage.BROWSING, progress_percent=50)
|
||
result = api_browser.browse_content(
|
||
browse_type=browse_type,
|
||
should_stop_callback=should_stop,
|
||
progress_callback=on_browse_progress,
|
||
)
|
||
return "ok", result
|
||
|
||
|
||
def _record_success_without_screenshot(
|
||
*,
|
||
account,
|
||
user_id: int,
|
||
account_id: str,
|
||
browse_type: str,
|
||
source: str,
|
||
task_start_time: float,
|
||
result,
|
||
batch_id,
|
||
time_module,
|
||
) -> bool:
|
||
_create_task_log(
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
account=account,
|
||
browse_type=browse_type,
|
||
status="success",
|
||
total_items=result.total_items,
|
||
total_attachments=result.total_attachments,
|
||
error_message="",
|
||
duration=int(time_module.time() - task_start_time),
|
||
source=source,
|
||
)
|
||
if batch_id:
|
||
_record_batch_result(
|
||
batch_id=batch_id,
|
||
account=account,
|
||
total_items=result.total_items,
|
||
total_attachments=result.total_attachments,
|
||
)
|
||
return True
|
||
|
||
if source and source.startswith("user_scheduled"):
|
||
try:
|
||
user_info = database.get_user_by_id(user_id)
|
||
if user_info and user_info.get("email") and database.get_user_email_notify(user_id):
|
||
email_service.send_task_complete_email_async(
|
||
user_id=user_id,
|
||
email=user_info["email"],
|
||
username=user_info["username"],
|
||
account_name=_account_display_name(account),
|
||
browse_type=browse_type,
|
||
total_items=result.total_items,
|
||
total_attachments=result.total_attachments,
|
||
screenshot_path=None,
|
||
log_callback=lambda msg: log_to_client(msg, user_id, account_id),
|
||
)
|
||
except Exception as email_error:
|
||
logger.warning(f"发送任务完成邮件失败: {email_error}")
|
||
|
||
return False
|
||
|
||
|
||
def _is_timeout_error(error_msg: str) -> bool:
|
||
return "Timeout" in error_msg or "timeout" in error_msg
|
||
|
||
|
||
def _record_failed_task_log(
|
||
*,
|
||
account,
|
||
user_id: int,
|
||
account_id: str,
|
||
browse_type: str,
|
||
source: str,
|
||
total_items: int,
|
||
total_attachments: int,
|
||
error_message: str,
|
||
task_start_time: float,
|
||
time_module,
|
||
) -> None:
|
||
account.status = "出错"
|
||
_create_task_log(
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
account=account,
|
||
browse_type=browse_type,
|
||
status="failed",
|
||
total_items=total_items,
|
||
total_attachments=total_attachments,
|
||
error_message=error_message,
|
||
duration=int(time_module.time() - task_start_time),
|
||
source=source,
|
||
)
|
||
|
||
|
||
def _handle_failed_browse_result(
|
||
*,
|
||
account,
|
||
result,
|
||
error_msg: str,
|
||
user_id: int,
|
||
account_id: str,
|
||
browse_type: str,
|
||
source: str,
|
||
attempt: int,
|
||
max_attempts: int,
|
||
task_start_time: float,
|
||
time_module,
|
||
) -> str:
|
||
if _is_timeout_error(error_msg):
|
||
log_to_client(f"⚠ 检测到超时错误: {error_msg}", user_id, account_id)
|
||
|
||
close_ok = _close_account_automation(
|
||
account,
|
||
on_error=lambda e: logger.debug(f"关闭超时浏览器实例失败: {e}"),
|
||
)
|
||
if close_ok:
|
||
log_to_client("已关闭超时的浏览器实例", user_id, account_id)
|
||
|
||
if attempt < max_attempts:
|
||
log_to_client(f"⚠ 代理可能速度过慢，将换新IP重试 ({attempt}/{max_attempts})", user_id, account_id)
|
||
time_module.sleep(2)
|
||
return "continue"
|
||
|
||
log_to_client(f"❌ 已达到最大重试次数({max_attempts})，任务失败", user_id, account_id)
|
||
_record_failed_task_log(
|
||
account=account,
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
browse_type=browse_type,
|
||
source=source,
|
||
total_items=result.total_items,
|
||
total_attachments=result.total_attachments,
|
||
error_message=f"重试{max_attempts}次后仍失败: {error_msg}",
|
||
task_start_time=task_start_time,
|
||
time_module=time_module,
|
||
)
|
||
return "break"
|
||
|
||
log_to_client(f"浏览出错: {error_msg}", user_id, account_id)
|
||
_record_failed_task_log(
|
||
account=account,
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
browse_type=browse_type,
|
||
source=source,
|
||
total_items=result.total_items,
|
||
total_attachments=result.total_attachments,
|
||
error_message=error_msg,
|
||
task_start_time=task_start_time,
|
||
time_module=time_module,
|
||
)
|
||
return "break"
|
||
|
||
|
||
def _handle_attempt_exception(
|
||
*,
|
||
account,
|
||
error_msg: str,
|
||
user_id: int,
|
||
account_id: str,
|
||
browse_type: str,
|
||
source: str,
|
||
attempt: int,
|
||
max_attempts: int,
|
||
task_start_time: float,
|
||
time_module,
|
||
) -> str:
|
||
_close_account_automation(
|
||
account,
|
||
on_error=lambda e: logger.debug(f"关闭浏览器实例失败: {e}"),
|
||
)
|
||
|
||
if _is_timeout_error(error_msg):
|
||
log_to_client(f"⚠ 执行超时: {error_msg}", user_id, account_id)
|
||
if attempt < max_attempts:
|
||
log_to_client(f"⚠ 将换新IP重试 ({attempt}/{max_attempts})", user_id, account_id)
|
||
time_module.sleep(2)
|
||
return "continue"
|
||
|
||
log_to_client(f"❌ 已达到最大重试次数({max_attempts})，任务失败", user_id, account_id)
|
||
_record_failed_task_log(
|
||
account=account,
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
browse_type=browse_type,
|
||
source=source,
|
||
total_items=account.total_items,
|
||
total_attachments=account.total_attachments,
|
||
error_message=f"重试{max_attempts}次后仍失败: {error_msg}",
|
||
task_start_time=task_start_time,
|
||
time_module=time_module,
|
||
)
|
||
return "break"
|
||
|
||
log_to_client(f"任务执行异常: {error_msg}", user_id, account_id)
|
||
_record_failed_task_log(
|
||
account=account,
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
browse_type=browse_type,
|
||
source=source,
|
||
total_items=account.total_items,
|
||
total_attachments=account.total_attachments,
|
||
error_message=error_msg,
|
||
task_start_time=task_start_time,
|
||
time_module=time_module,
|
||
)
|
||
return "break"
|
||
|
||
|
||
def _schedule_auto_retry(
|
||
*,
|
||
user_id: int,
|
||
account_id: str,
|
||
browse_type: str,
|
||
enable_screenshot: bool,
|
||
source: str,
|
||
retry_count: int,
|
||
) -> None:
|
||
def delayed_retry_submit():
|
||
fresh_account = safe_get_account(user_id, account_id)
|
||
if not fresh_account:
|
||
log_to_client("自动重试取消: 账户不存在", user_id, account_id)
|
||
return
|
||
if fresh_account.should_stop:
|
||
log_to_client("自动重试取消: 任务已被停止", user_id, account_id)
|
||
return
|
||
|
||
log_to_client(f"🔄 开始第 {retry_count + 1} 次自动重试...", user_id, account_id)
|
||
ok, msg = submit_account_task(
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
browse_type=browse_type,
|
||
enable_screenshot=enable_screenshot,
|
||
source=source,
|
||
retry_count=retry_count + 1,
|
||
)
|
||
if not ok:
|
||
log_to_client(f"自动重试提交失败: {msg}", user_id, account_id)
|
||
|
||
try:
|
||
threading.Timer(5, delayed_retry_submit).start()
|
||
except Exception:
|
||
delayed_retry_submit()
|
||
|
||
|
||
def _finalize_task_run(
|
||
*,
|
||
account,
|
||
user_id: int,
|
||
account_id: str,
|
||
browse_type: str,
|
||
source: str,
|
||
enable_screenshot: bool,
|
||
retry_count: int,
|
||
max_auto_retry: int,
|
||
task_start_time: float,
|
||
result,
|
||
batch_id,
|
||
batch_recorded: bool,
|
||
) -> None:
|
||
final_status = str(account.status or "")
|
||
account.is_running = False
|
||
screenshot_submitted = False
|
||
|
||
_close_account_automation(
|
||
account,
|
||
on_error=lambda e: log_to_client(f"关闭主任务浏览器时出错: {str(e)}", user_id, account_id),
|
||
)
|
||
|
||
safe_remove_task(account_id)
|
||
safe_remove_task_status(account_id)
|
||
|
||
if final_status == "已完成" and not account.should_stop:
|
||
account.status = "已完成"
|
||
_emit("account_update", account.to_dict(), room=f"user_{user_id}")
|
||
|
||
if enable_screenshot:
|
||
log_to_client("等待2秒后开始截图...", user_id, account_id)
|
||
account.status = "等待截图"
|
||
_emit("account_update", account.to_dict(), room=f"user_{user_id}")
|
||
|
||
safe_set_task_status(
|
||
account_id,
|
||
{
|
||
"user_id": user_id,
|
||
"username": account.username,
|
||
"status": "排队中",
|
||
"detail_status": "等待截图资源",
|
||
"browse_type": browse_type,
|
||
"start_time": time.time(),
|
||
"source": source,
|
||
"progress": {
|
||
"items": result.total_items if result else 0,
|
||
"attachments": result.total_attachments if result else 0,
|
||
},
|
||
},
|
||
)
|
||
|
||
browse_result_dict = {
|
||
"total_items": result.total_items if result else 0,
|
||
"total_attachments": result.total_attachments if result else 0,
|
||
}
|
||
screenshot_submitted = True
|
||
threading.Thread(
|
||
target=take_screenshot_for_account,
|
||
args=(user_id, account_id, browse_type, source, task_start_time, browse_result_dict),
|
||
daemon=True,
|
||
).start()
|
||
else:
|
||
account.status = "未开始"
|
||
_emit("account_update", account.to_dict(), room=f"user_{user_id}")
|
||
log_to_client("截图功能已禁用,跳过截图", user_id, account_id)
|
||
else:
|
||
if final_status == "出错" and retry_count < max_auto_retry:
|
||
log_to_client(f"⚠ 任务执行失败，5秒后自动重试 ({retry_count + 1}/{max_auto_retry})...", user_id, account_id)
|
||
account.status = "等待重试"
|
||
_emit("account_update", account.to_dict(), room=f"user_{user_id}")
|
||
_schedule_auto_retry(
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
browse_type=browse_type,
|
||
enable_screenshot=enable_screenshot,
|
||
source=source,
|
||
retry_count=retry_count,
|
||
)
|
||
elif final_status in ["登录失败", "出错"]:
|
||
account.status = final_status
|
||
_emit("account_update", account.to_dict(), room=f"user_{user_id}")
|
||
else:
|
||
account.status = "未开始"
|
||
_emit("account_update", account.to_dict(), room=f"user_{user_id}")
|
||
|
||
if batch_id and (not screenshot_submitted) and (not batch_recorded) and account.status != "等待重试":
|
||
_record_batch_result(
|
||
batch_id=batch_id,
|
||
account=account,
|
||
total_items=getattr(account, "total_items", 0) or 0,
|
||
total_attachments=getattr(account, "total_attachments", 0) or 0,
|
||
)
|
||
|
||
|
||
def run_task(user_id, account_id, browse_type, enable_screenshot=True, source="manual", retry_count=0):
|
||
"""运行自动化任务
|
||
|
||
Args:
|
||
retry_count: 当前重试次数，用于自动重试机制（最多重试2次）
|
||
"""
|
||
MAX_AUTO_RETRY = 2 # 最大自动重试次数
|
||
LoggerAdapter("app", {"user_id": user_id, "account_id": account_id, "source": source}).debug(
|
||
f"run_task enable_screenshot={enable_screenshot} ({type(enable_screenshot).__name__}), retry={retry_count}"
|
||
)
|
||
|
||
account = safe_get_account(user_id, account_id)
|
||
if not account:
|
||
return
|
||
batch_id = _get_batch_id_from_source(source)
|
||
batch_recorded = False
|
||
|
||
checkpoint_mgr = get_checkpoint_mgr()
|
||
|
||
import time as time_module
|
||
|
||
task_start_time = time_module.time()
|
||
result = None
|
||
|
||
try:
|
||
if _handle_stop_requested(
|
||
account=account,
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
batch_id=batch_id,
|
||
remove_task_status=True,
|
||
record_batch=True,
|
||
):
|
||
return
|
||
|
||
try:
|
||
if _handle_stop_requested(
|
||
account=account,
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
batch_id=batch_id,
|
||
):
|
||
return
|
||
|
||
task_id = checkpoint_mgr.create_checkpoint(
|
||
user_id=user_id, account_id=account_id, username=account.username, browse_type=browse_type
|
||
)
|
||
logger.info(f"[断点] 任务 {task_id} 已创建")
|
||
|
||
task_start_time = time_module.time()
|
||
|
||
account.status = "运行中"
|
||
_emit("account_update", account.to_dict(), room=f"user_{user_id}")
|
||
account.last_browse_type = browse_type
|
||
|
||
safe_update_task_status(
|
||
account_id, {"status": "运行中", "detail_status": "初始化", "start_time": task_start_time}
|
||
)
|
||
|
||
max_attempts = 3
|
||
|
||
for attempt in range(1, max_attempts + 1):
|
||
try:
|
||
if attempt > 1:
|
||
log_to_client(f"🔄 第 {attempt} 次尝试（共{max_attempts}次）...", user_id, account_id)
|
||
|
||
attempt_status, result = _execute_single_attempt(
|
||
account=account,
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
browse_type=browse_type,
|
||
source=source,
|
||
checkpoint_mgr=checkpoint_mgr,
|
||
task_id=task_id,
|
||
task_start_time=task_start_time,
|
||
time_module=time_module,
|
||
)
|
||
if attempt_status == "login_failed":
|
||
return
|
||
|
||
account.total_items = result.total_items
|
||
account.total_attachments = result.total_attachments
|
||
|
||
if result.success:
|
||
log_to_client(
|
||
f"浏览完成! 共 {result.total_items} 条内容，{result.total_attachments} 个附件",
|
||
user_id,
|
||
account_id,
|
||
)
|
||
safe_update_task_status(
|
||
account_id,
|
||
{
|
||
"detail_status": "浏览完成",
|
||
"progress": {"items": result.total_items, "attachments": result.total_attachments},
|
||
},
|
||
)
|
||
account.status = "已完成"
|
||
checkpoint_mgr.update_stage(task_id, TaskStage.COMPLETING, progress_percent=95)
|
||
checkpoint_mgr.complete_task(task_id, success=True)
|
||
logger.info(f"[断点] 任务 {task_id} 已完成")
|
||
|
||
if not enable_screenshot:
|
||
batch_recorded = _record_success_without_screenshot(
|
||
account=account,
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
browse_type=browse_type,
|
||
source=source,
|
||
task_start_time=task_start_time,
|
||
result=result,
|
||
batch_id=batch_id,
|
||
time_module=time_module,
|
||
)
|
||
break
|
||
|
||
error_msg = result.error_message
|
||
action = _handle_failed_browse_result(
|
||
account=account,
|
||
result=result,
|
||
error_msg=error_msg,
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
browse_type=browse_type,
|
||
source=source,
|
||
attempt=attempt,
|
||
max_attempts=max_attempts,
|
||
task_start_time=task_start_time,
|
||
time_module=time_module,
|
||
)
|
||
if action == "continue":
|
||
continue
|
||
break
|
||
|
||
except Exception as retry_error:
|
||
error_msg = str(retry_error)
|
||
action = _handle_attempt_exception(
|
||
account=account,
|
||
error_msg=error_msg,
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
browse_type=browse_type,
|
||
source=source,
|
||
attempt=attempt,
|
||
max_attempts=max_attempts,
|
||
task_start_time=task_start_time,
|
||
time_module=time_module,
|
||
)
|
||
if action == "continue":
|
||
continue
|
||
break
|
||
|
||
except Exception as e:
|
||
error_msg = str(e)
|
||
log_to_client(f"任务执行出错: {error_msg}", user_id, account_id)
|
||
account.status = "出错"
|
||
_create_task_log(
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
account=account,
|
||
browse_type=browse_type,
|
||
status="failed",
|
||
total_items=account.total_items,
|
||
total_attachments=account.total_attachments,
|
||
error_message=error_msg,
|
||
duration=int(time_module.time() - task_start_time),
|
||
source=source,
|
||
)
|
||
|
||
finally:
|
||
_finalize_task_run(
|
||
account=account,
|
||
user_id=user_id,
|
||
account_id=account_id,
|
||
browse_type=browse_type,
|
||
source=source,
|
||
enable_screenshot=enable_screenshot,
|
||
retry_count=retry_count,
|
||
max_auto_retry=MAX_AUTO_RETRY,
|
||
task_start_time=task_start_time,
|
||
result=result,
|
||
batch_id=batch_id,
|
||
batch_recorded=batch_recorded,
|
||
)
|
||
|
||
finally:
|
||
pass
|