fix(api): 超时按单条处理避免中途结束

This commit is contained in:
2025-12-17 15:49:05 +08:00
parent 6827d11f40
commit 3f667dd21b

View File

@@ -233,9 +233,11 @@ class APIBrowser:
if not self.logged_in: if not self.logged_in:
return [], 0, None return [], 0, None
try:
if base_url and page > 1: if base_url and page > 1:
url = re.sub(r'page=\d+', f'page={page}', base_url) url = re.sub(r'page=\d+', f'page={page}', base_url)
elif page > 1:
# 兼容兜底:若没有 next_url极少数情况下页面不提供“下一页”链接尝试直接拼 page 参数
url = f"{BASE_URL}/admin/center.aspx?bz={bz}&page={page}"
else: else:
url = f"{BASE_URL}/admin/center.aspx?bz={bz}" url = f"{BASE_URL}/admin/center.aspx?bz={bz}"
@@ -290,13 +292,8 @@ class APIBrowser:
self.last_total_records = 0 self.last_total_records = 0
return articles, total_pages, next_page_url return articles, total_pages, next_page_url
except Exception as e:
self.log(f"[API] 获取列表失败: {str(e)}")
return [], 0, None
def get_article_attachments(self, article_href: str): def get_article_attachments(self, article_href: str):
"""获取文章的附件列表""" """获取文章的附件列表"""
try:
if not article_href.startswith('http'): if not article_href.startswith('http'):
url = f"{BASE_URL}/admin/{article_href}" url = f"{BASE_URL}/admin/{article_href}"
else: else:
@@ -330,9 +327,6 @@ class APIBrowser:
return attachments return attachments
except Exception as e:
return []
def mark_read(self, attach_id: str, channel_id: str = '1') -> bool: def mark_read(self, attach_id: str, channel_id: str = '1') -> bool:
"""通过访问下载链接标记已读""" """通过访问下载链接标记已读"""
download_url = f"{BASE_URL}/tools/download.ashx?site=main&id={attach_id}&channel_id={channel_id}" download_url = f"{BASE_URL}/tools/download.ashx?site=main&id={attach_id}&channel_id={channel_id}"
@@ -383,9 +377,19 @@ class APIBrowser:
total_attachments = 0 total_attachments = 0
page = 1 page = 1
base_url = None base_url = None
skipped_items = 0
skipped_pages = 0
consecutive_failures = 0
max_consecutive_failures = 3
# 获取第一页 # 获取第一页
try:
articles, total_pages, next_url = self.get_article_list_page(bz, page) articles, total_pages, next_url = self.get_article_list_page(bz, page)
consecutive_failures = 0
except Exception as e:
result.error_message = str(e)
self.log(f"[API] 获取第1页列表失败: {str(e)}")
return result
if not articles: if not articles:
self.log(f"[API] '{browse_type}' 没有待处理内容") self.log(f"[API] '{browse_type}' 没有待处理内容")
@@ -396,6 +400,8 @@ class APIBrowser:
if next_url: if next_url:
base_url = next_url base_url = next_url
elif total_pages > 1:
base_url = f"{BASE_URL}/admin/center.aspx?bz={bz}&page=2"
total_records = int(getattr(self, "last_total_records", 0) or 0) total_records = int(getattr(self, "last_total_records", 0) or 0)
last_report_ts = 0.0 last_report_ts = 0.0
@@ -416,22 +422,51 @@ class APIBrowser:
report_progress(force=True) report_progress(force=True)
# 处理所有页面 # 处理所有页面
while True: while page <= total_pages:
if should_stop_callback and should_stop_callback(): if should_stop_callback and should_stop_callback():
self.log("[API] 收到停止信号") self.log("[API] 收到停止信号")
break break
# page==1 已取过,后续页在这里获取
if page > 1:
try:
articles, _, next_url = self.get_article_list_page(bz, page, base_url)
consecutive_failures = 0
if next_url:
base_url = next_url
except Exception as e:
skipped_pages += 1
consecutive_failures += 1
self.log(
f"[API] 获取第{page}页列表失败,跳过本页(连续失败{consecutive_failures}/{max_consecutive_failures}: {str(e)}"
)
if consecutive_failures >= max_consecutive_failures:
raise
page += 1
continue
for article in articles: for article in articles:
if should_stop_callback and should_stop_callback(): if should_stop_callback and should_stop_callback():
break break
title = article['title'][:30] title = article['title'][:30]
# 获取附件(文章详情页)
try:
attachments = self.get_article_attachments(article['href'])
consecutive_failures = 0
except Exception as e:
skipped_items += 1
consecutive_failures += 1
self.log(
f"[API] 获取文章失败,跳过(连续失败{consecutive_failures}/{max_consecutive_failures}: {title} | {str(e)}"
)
if consecutive_failures >= max_consecutive_failures:
raise
continue
total_items += 1 total_items += 1
report_progress() report_progress()
# 获取附件
attachments = self.get_article_attachments(article['href'])
if attachments: if attachments:
for attach in attachments: for attach in attachments:
if self.mark_read(attach['id'], attach['channel_id']): if self.mark_read(attach['id'], attach['channel_id']):
@@ -441,21 +476,15 @@ class APIBrowser:
time.sleep(0.1) time.sleep(0.1)
# 下一页
page += 1 page += 1
if page > total_pages:
break
articles, _, next_url = self.get_article_list_page(bz, page, base_url)
if not articles:
break
if next_url:
base_url = next_url
time.sleep(0.2) time.sleep(0.2)
report_progress(force=True) report_progress(force=True)
if skipped_items or skipped_pages:
self.log(
f"[API] 浏览完成: {total_items} 条内容,{total_attachments} 个附件(跳过 {skipped_items} 条内容,{skipped_pages} 页列表)"
)
else:
self.log(f"[API] 浏览完成: {total_items} 条内容,{total_attachments} 个附件") self.log(f"[API] 浏览完成: {total_items} 条内容,{total_attachments} 个附件")
result.success = True result.success = True