Browse Source

fix(ai_seo): 修复 Kimi 和 Metaso 爬虫问题

- 在 Kimi爬虫中添加错误处理逻辑,当 API 返回错误时终止爬取
- 更新 Metaso 爬虫中的复制按钮选择器,以适应网页结构变化
master
zzx 3 months ago
parent
commit
0fc1a7ab89
  1. 9
      spiders/ai_seo/kimi.py
  2. 2
      spiders/ai_seo/metaso.py

9
spiders/ai_seo/kimi.py

@ -9,6 +9,8 @@ from domain.ai_seo import AiAnswer, AiSearchResult
from utils import create_logger
from glom import glom, Coalesce
from utils.ai_seo_api_utils import AiSeoApis
logger = create_logger(__name__)
@ -52,6 +54,7 @@ class KimiSpider(AbstractAiSeoSpider):
# 报错检查
if self.fail_status:
await AiSeoApis.update_spider_session(self.session_info['id'], 2)
raise self.fail_exception
# 关闭侧边栏
@ -85,6 +88,12 @@ class KimiSpider(AbstractAiSeoSpider):
if json_data['items']:
logger.debug(json_data)
detail = json_data['items'][-1]
if 'error' in detail:
logger.error(f"kimi回复错误: {detail['error']['detail']}")
self.fail_status = True
self.fail_exception = Exception(detail['error']['detail'])
self.completed_event.set()
return
content = detail['content']
if self.think:
self.ai_answer.search_result = self.get_search_list_enable_think(detail)

2
spiders/ai_seo/metaso.py

@ -55,7 +55,7 @@ class MetasoSpider(AbstractAiSeoSpider):
# await self.completed_event.wait()
# 等待指定元素
#//*[@id="search-content-container-8626530479804592128"]/div[2]/button
copy_button = await self.browser_page.wait_for_selector('//*[starts-with(@id, "search-content-container-")]/div[2]/button', timeout=600000)
copy_button = await self.browser_page.wait_for_selector('//*[starts-with(@id, "search-content-container-")]/div[2]/div[3]/button', timeout=600000)
# 点击复制按钮
await copy_button.click()
# 读取剪贴板

Loading…
Cancel
Save