# -*- coding: utf-8 -*- """ CollectRuntime — 采集执行器 (复用各平台 Client + CookiePool) 从 task_worker.py 提取, 适配 task_worker_v2 包结构。 """ from __future__ import annotations import asyncio import json import random import re import traceback from typing import Any, Optional import requests as req_lib from platforms.deepseek import DeepSeekClient, get_ds_session_id from platforms.kimi import KimiClient, extract_kimi_from_browser from platforms.qianwen import QianwenClient, extract_qianwen_cookies, get_qw_session_id from platforms.wenxin import WenxinClient, extract_wenxin_cookies, get_wenxin_share_url from platforms.doubao import DoubaoAdapter from platforms.base import NO_PROXY, CDP_URL, ensure_chrome_running from config import COOKIE_PLATFORM_IDS, COOKIE_DOMAINS, API_TO_INTERNAL, QUOTA_KEYWORDS, normalize_platform_name from cookie_pool import CookiePool, parse_cookie_string, cookie_str_to_playwright from collector import save_raw_and_parse # ---------- 停止检测回调 ---------- # 由 main.py 注入 (避免循环导入); 默认永不停止 _is_stopping = lambda: False # noqa: E731 def set_stop_check(fn): """让调用方 (main.py) 注入 is_stopping 函数""" global _is_stopping _is_stopping = fn def is_stopping() -> bool: return _is_stopping() # ---------- 豆包 cookie 域名列表 ---------- DOUBAO_COOKIE_DOMAINS = [".doubao.com", "www.doubao.com", "doubao.com"] # ---------- cookie 灌入工具函数 ---------- def apply_cookie_str_to_qianwen(client, cookie_str: str): """把 cookie 串灌进 QianwenClient""" client._cookie_str = cookie_str if "=" in cookie_str and "Bearer" not in cookie_str[:20] else cookie_str jar = parse_cookie_string(cookie_str) if jar: client._cookie_str = "; ".join(f"{k}={v}" for k, v in jar.items()) client._xsrf_token = jar.get("XSRF-TOKEN", "") or next( (v for k, v in jar.items() if k.upper() == "XSRF-TOKEN"), "" ) return client def apply_cookie_str_to_wenxin(client, cookie_str: str): jar = parse_cookie_string(cookie_str) if jar: client._cookie_str = "; ".join(f"{k}={v}" for k, v in jar.items()) else: client._cookie_str = cookie_str return client # ============================================================ # 采集执行器 (复用平台 Client + CookiePool) # ============================================================ class CollectRuntime: def __init__( self, cookie_pool: Optional[CookiePool] = None, *, cfg: Optional[dict] = None, wenxin_browser_share: bool = False, browser_fallback_auth: bool = False, doubao_cfg: Optional[dict] = None, ): if cookie_pool is None: raise ValueError("CollectRuntime 需要 CookiePool 实例") self.pool = cookie_pool # 若传入完整 cfg, 从中提取 browser / doubao 子配置 if cfg is not None: br_cfg = cfg.get("browser") or {} dc = cfg.get("doubao") or {} wenxin_browser_share = wenxin_browser_share or bool(br_cfg.get("wenxin_share", False)) browser_fallback_auth = browser_fallback_auth or bool(br_cfg.get("fallback_auth", False)) if doubao_cfg is None: doubao_cfg = dc # 文心短链是否走浏览器点击 (默认否 → 不启 Chrome) self.wenxin_browser_share = bool(wenxin_browser_share) # cookie 池失败时是否允许浏览器抠登录态 (默认否) self.browser_fallback_auth = bool(browser_fallback_auth) # 豆包浏览器自动化参数 dc = doubao_cfg or {} self.doubao_wait_timeout = int(dc.get("wait_timeout") or 180) self.doubao_share = bool(dc.get("share", True)) self.doubao_cookie_rotate = bool(dc.get("cookie_rotate_on_fail", True)) self.doubao_max_retry = int(dc.get("max_retry") or 2) self.doubao_use_cookie_pool = bool(dc.get("use_cookie_pool", True)) self.ds_client: Optional[DeepSeekClient] = None self.kimi_client: Optional[KimiClient] = None self.qw_client: Optional[QianwenClient] = None self.wx_client: Optional[WenxinClient] = None self.doubao_adapter: Optional[DoubaoAdapter] = None self.wx_share_page = None self._pw = None self._browser = None self._ctx = None self._cdp = None self._doubao_page = None self._doubao_dt_exhausted = False self._ds_lock = asyncio.Lock() self._kimi_lock = asyncio.Lock() self._qw_lock = asyncio.Lock() self._wx_lock = asyncio.Lock() self._db_lock = asyncio.Lock() # 豆包页面串行 self._session_ids: dict[str, Any] = {} self._doubao_cookie_str = "" self._chrome_ready = False def _needs_chrome(self, needed_internal: set[str]) -> bool: """ 仅在真正依赖浏览器时启动 Chrome: - doubao: CDP 页面自动化 (必须) - wenxin + wenxin_browser_share: 分享短链 (可选) deepseek / kimi / tongyi 走 cookie 池 HTTP, 默认不启浏览器。 """ if "doubao" in needed_internal: return True if "wenxin" in needed_internal and self.wenxin_browser_share: return True return False async def setup(self, needed_internal: set[str]): """按需初始化: 优先 cookie 池; 浏览器仅 doubao 或文心短链分享""" need_chrome = self._needs_chrome(needed_internal) if need_chrome: reasons = [] if "doubao" in needed_internal: reasons.append("doubao采集") if "wenxin" in needed_internal and self.wenxin_browser_share: reasons.append("文心短链分享") print(f"=== 启动浏览器 (原因: {', '.join(reasons)}) ===") if not ensure_chrome_running(): if "doubao" in needed_internal: raise RuntimeError("Chrome 未就绪, 豆包无法采集") print("[WARN] Chrome 未就绪, 文心短链分享将降级为页面链接") need_chrome = False else: self._chrome_ready = True else: print("=== 跳过浏览器 (当前开启平台均走 HTTP/cookie 池) ===") print(" 提示: 仅 doubao=true 或 browser.wenxin_share=true 时才会启动 Chrome") print("=== Cookie 池认证 (ToolsLoad 对齐) ===") for name, pid in COOKIE_PLATFORM_IDS.items(): if name in needed_internal: print(f" 平台 {name} → platform_id={pid}") if "ds" in needed_internal: await self._ensure_ds_client() if "kimi" in needed_internal: await self._ensure_kimi_client() if "qianwen" in needed_internal: await self._ensure_qianwen_client() if "wenxin" in needed_internal: await self._ensure_wenxin_client() if need_chrome and self._chrome_ready: from playwright.async_api import async_playwright self._pw = await async_playwright().start() self._browser = await self._pw.chromium.connect_over_cdp(CDP_URL) self._ctx = self._browser.contexts[0] if "doubao" in needed_internal: await self._setup_doubao() if "wenxin" in needed_internal and self.wenxin_browser_share: wx_page = next((p for p in self._ctx.pages if "baidu.com" in p.url), None) if not wx_page: wx_page = await self._ctx.new_page() await wx_page.goto("https://chat.baidu.com", wait_until="domcontentloaded", timeout=20000) await asyncio.sleep(2) cookie_str = (self.wx_client._cookie_str if self.wx_client else "") or "" if cookie_str: try: await self._ctx.add_cookies( cookie_str_to_playwright(cookie_str, COOKIE_DOMAINS["wenxin"]) ) await wx_page.goto("https://chat.baidu.com", wait_until="domcontentloaded", timeout=20000) await asyncio.sleep(1) print(" [auth] 文心 cookie 已注入浏览器分享页") except Exception as e: print(f" [auth] 文心 cookie 注入失败: {e}") self.wx_share_page = wx_page print(" [auth] 文心分享页就绪") elif "wenxin" in needed_internal: print(" [auth] 文心分享: 不使用浏览器, 提交时用 chat.baidu.com/search/{lid}") async def _inject_doubao_cookies(self, cookie_str: str) -> int: """注入豆包 cookie 到多个 domain, 返回注入条数""" if not self._ctx or not cookie_str: return 0 cookies = cookie_str_to_playwright(cookie_str, DOUBAO_COOKIE_DOMAINS) if not cookies: return 0 await self._ctx.add_cookies(cookies) return len(cookies) async def _check_doubao_login(self, page) -> bool: """判断豆包页是否已登录 (有输入框且非登录墙)""" try: url = page.url or "" if "login" in url or "passport" in url: return False # 常见登录入口文案 for sel in ("text=登录", "text=手机号登录", "text=扫码登录"): try: loc = page.locator(sel).first if await loc.is_visible(timeout=800): # 登录按钮在未登录态常驻; 再确认是否有发消息框 pass except Exception: pass ta = page.locator("textarea").first await ta.wait_for(state="visible", timeout=8000) # 若页面提示登录才能使用 content = "" try: content = await page.inner_text("body", timeout=3000) except Exception: pass if any(k in (content or "") for k in ("登录后即可", "请先登录", "立即登录", "未登录")): # 仍可能有 textarea 占位, 保守判断 if "登录" in (content or "")[:500] and "发消息" not in (content or ""): return False return True except Exception as e: print(f" [auth] 豆包登录检测异常: {e}") return False async def _bind_doubao_page(self, page) -> None: """为页面建立 CDP 拦截 + DoubaoAdapter""" if self._cdp: try: await self._cdp.detach() except Exception: pass self._cdp = None self._doubao_page = page self._cdp = await self._ctx.new_cdp_session(page) await self._cdp.send("Network.enable") self.doubao_adapter = DoubaoAdapter(page, self._cdp) async def _setup_doubao(self): """ 豆包浏览器自动化初始化: 1) (可选) cookie 池注入 2) 打开 /chat, 校验登录 3) 失败则换号再注入; 仍失败则依赖 Chrome 用户态 4) 挂载 CDP Network 拦截 chat/completion """ print(" [auth] 豆包自动化初始化 (Playwright CDP)...") logged_in = False if self.doubao_use_cookie_pool: for attempt in range(max(1, self.doubao_max_retry + 1)): sess = self.pool.get_for_internal("doubao") if not sess or not sess.get("cookie"): print(" [auth] 豆包 cookie 池无数据, 改用浏览器已有登录态") break self._doubao_cookie_str = sess["cookie"] self._session_ids["doubao"] = sess.get("id") try: n = await self._inject_doubao_cookies(sess["cookie"]) print(f" [auth] 豆包 cookie 已注入 id={sess.get('id')} entries={n}") except Exception as e: print(f" [auth] 豆包 cookie 注入失败: {e}") self._drop_session("doubao", "invalid") continue page = next((p for p in self._ctx.pages if "doubao.com" in (p.url or "")), None) if not page: page = await self._ctx.new_page() try: await page.goto("https://www.doubao.com/chat", wait_until="domcontentloaded", timeout=60000) await asyncio.sleep(2) except Exception as e: print(f" [auth] 豆包打开页面失败: {e}") self._drop_session("doubao", "invalid") continue if await self._check_doubao_login(page): await self._bind_doubao_page(page) logged_in = True print(" [auth] 豆包登录态 OK (cookie 池)") break print(f" [auth] 豆包 cookie 无效/未登录, 换号 (attempt {attempt + 1})") self._drop_session("doubao", "invalid") try: await self._ctx.clear_cookies() except Exception: pass if not logged_in: # 回退: Chrome user-data 已登录态 (ensure_chrome 带的 profile) page = next((p for p in self._ctx.pages if "doubao.com" in (p.url or "")), None) if not page: page = await self._ctx.new_page() await page.goto("https://www.doubao.com/chat", wait_until="domcontentloaded", timeout=60000) await asyncio.sleep(2) if await self._check_doubao_login(page): await self._bind_doubao_page(page) logged_in = True print(" [auth] 豆包登录态 OK (Chrome 已有会话)") else: await self._bind_doubao_page(page) # 仍绑定, 采集时再报错 print(" [auth] ⚠ 豆包未检测到登录, 请在调试 Chrome 中手动登录 doubao.com") print(f" [auth] 豆包浏览器就绪 share={self.doubao_share} wait={self.doubao_wait_timeout}s") # ------ 平台客户端初始化 ------ async def _ensure_ds_client(self) -> Optional[DeepSeekClient]: async with self._ds_lock: if self.ds_client: return self.ds_client sess = self.pool.get_for_internal("ds") if not sess: print(" [auth] DeepSeek cookie 池无可用 session") return None self.ds_client = DeepSeekClient(sess["cookie"], str(sess.get("id", ""))) self._session_ids["ds"] = sess.get("id") print(f" [auth] DeepSeek 就绪 (pool id={sess.get('id')})") return self.ds_client async def _ensure_kimi_client(self) -> Optional[KimiClient]: async with self._kimi_lock: if self.kimi_client: return self.kimi_client sess = self.pool.get_for_internal("kimi") token = sess.get("cookie") if sess else None sid = sess.get("id") if sess else None if not token and self.browser_fallback_auth and self._chrome_ready: print(" [auth] Kimi cookie 池无数据, 尝试浏览器提取...") br = await extract_kimi_from_browser() if br: token, sid = br.get("token"), br.get("id", "-1") if not token: print(" [auth] Kimi Token 不可用 (cookie 池无数据" + ("; 已关闭 browser.fallback_auth" if not self.browser_fallback_auth else "") + ")") return None try: self.kimi_client = KimiClient(token) self._session_ids["kimi"] = sid self.kimi_client.cookie_id = sid print(f" [auth] Kimi 就绪 (pool id={sid})") return self.kimi_client except Exception as e: print(f" [auth] Kimi 初始化失败: {e}") if sid: self.pool.invalidate(sid, "invalid") return None async def _ensure_qianwen_client(self) -> Optional[QianwenClient]: async with self._qw_lock: if self.qw_client and self.qw_client._cookie_str: return self.qw_client client = QianwenClient() sess = self.pool.get_for_internal("qianwen") if sess and sess.get("cookie"): apply_cookie_str_to_qianwen(client, sess["cookie"]) self._session_ids["qianwen"] = sess.get("id") print(f" [auth] 千问 cookie 池就绪 id={sess.get('id')}") elif self.browser_fallback_auth and self._chrome_ready: cookies = await extract_qianwen_cookies() if cookies: client.set_cookies(cookies) print(" [auth] 千问浏览器 cookie 就绪") else: print(" [auth] 千问无 cookie (HMAC 匿名仍可聊, 分享不可用)") else: print(" [auth] 千问 cookie 池无数据, 将尝试 HMAC 匿名采集") self.qw_client = client return client async def _ensure_wenxin_client(self) -> Optional[WenxinClient]: async with self._wx_lock: if self.wx_client and self.wx_client._cookie_str: return self.wx_client client = WenxinClient() sess = self.pool.get_for_internal("wenxin") if sess and sess.get("cookie"): apply_cookie_str_to_wenxin(client, sess["cookie"]) self._session_ids["wenxin"] = sess.get("id") print(f" [auth] 文心 cookie 池就绪 id={sess.get('id')}") elif self.browser_fallback_auth and self._chrome_ready: cookies = await extract_wenxin_cookies() if cookies: client._cookie_str = "; ".join(f'{c["name"]}={c["value"]}' for c in cookies) print(" [auth] 文心浏览器 cookie 就绪") else: print(" [auth] 文心无 cookie, 可能被反爬") else: print(" [auth] 文心 cookie 池无数据 (未启用 browser.fallback_auth)") self.wx_client = client return client # ------ 会话管理 ------ def _drop_session(self, internal: str, reason: str = "invalid"): """丢弃本地 client 并回写 cookie 池状态""" sid = self._session_ids.pop(internal, None) if sid is not None: self.pool.invalidate(sid, reason) if internal == "ds": self.ds_client = None elif internal == "kimi": self.kimi_client = None elif internal == "qianwen": self.qw_client = None elif internal == "wenxin": self.wx_client = None elif internal == "doubao": self._doubao_cookie_str = "" # ------ 解析 + 配额检查 ------ def _parse_and_check(self, plat_key: str, raw_body, raw_req, prompt: str, deep: bool) -> Optional[dict]: if isinstance(raw_body, bytes): raw_str = raw_body.decode("utf-8", errors="replace") else: raw_str = raw_body or "" if not raw_str: return None parsed = save_raw_and_parse(plat_key, raw_str, raw_req) if not parsed: return None answer = (parsed.get("result") or {}).get("answer") or "" if not answer.strip(): print(" ✗ 解析答案为空") return None if any(re.search(kw, answer) for kw in QUOTA_KEYWORDS): print(" ⚠ 命中限流/配额话术, 丢弃") # 回写池: 配额类 (plat_key 即内部 key: ds/kimi/qianwen/wenxin/doubao) internal = plat_key if plat_key in COOKIE_PLATFORM_IDS else None if internal: self._drop_session(internal, "quota") return None parsed["deep_thinking"] = 1 if deep else 0 parsed["question"] = prompt parsed["_raw_req"] = raw_req or "" print(f" ✓ 答案 {len(answer)} 字 | 思考 {len((parsed['result'].get('thinking_process') or ''))} 字 " f"| 信源 {len(parsed.get('sources') or [])}") return parsed # ------ Kimi 诊断工具 ------ def _kimi_response_diag(self, raw_body) -> str: """诊断 Kimi 原始响应: unauthenticated / overloaded / quota / 正常帧""" if isinstance(raw_body, bytes): raw_str = raw_body.decode("utf-8", errors="replace") else: raw_str = raw_body or "" if "unauthenticated" in raw_str or "请登录" in raw_str: return "unauthenticated" # 算力过载 / 服务端限流: 平台繁忙, 同号等待重试即可 if "OVERLOADED" in raw_str or "Kimi有点累了" in raw_str or "聊的人太多了" in raw_str or "resource_exhausted" in raw_str: return "overloaded" # 真正的配额用完: 需要换号 if any(kw in raw_str for kw in ( "算力不足", "前往升级", "对话次数已达上限", "免费次数用完", "次数已用完", )): return "quota" if "account_abnormal" in raw_str or "账号状态异常" in raw_str: return "account_abnormal" if '"error"' in raw_str: m = re.search(r'"code"\s*:\s*"([^"]+)"', raw_str) return f"error:{m.group(1)}" if m else "error" if len(raw_str.strip()) < 80 and not raw_str.strip(): return "empty" return "ok" def _kimi_probe_account(self, client) -> str: """ 用 /api/user 探测账号态 (比 Chat 错误更明确)。 返回: ok / account_abnormal / unauthenticated / unknown """ try: # 与 run.KimiClient._build_headers 对齐 (version 2.0.0) headers = { "authorization": client.token if str(client.token).startswith("Bearer") else f"Bearer {client.token}", "origin": "https://www.kimi.com", "referer": "https://www.kimi.com/", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/149.0.0.0 Safari/537.36", "x-msh-device-id": str(getattr(client, "device_id", "") or ""), "x-msh-session-id": str(getattr(client, "session_id", "") or ""), "x-msh-platform": "web", "x-msh-version": "2.0.0", "x-traffic-id": str(getattr(client, "traffic_id", "") or ""), "x-language": "zh-CN", "r-timezone": "Asia/Shanghai", } resp = client.requests.get( "https://www.kimi.com/api/user", headers=headers, timeout=15, impersonate="chrome146", ) text = resp.text or "" if "account_abnormal" in text or "账号状态异常" in text: return "account_abnormal" if resp.status_code in (401, 403) or "unauthenticated" in text or "请登录" in text: return "unauthenticated" if resp.status_code == 200 and ("id" in text or "name" in text or "email" in text or "user" in text.lower()): return "ok" return f"http_{resp.status_code}" except Exception as e: return f"probe_err:{e}" # ------ 单条采集入口 ------ async def collect_one(self, task: dict) -> Optional[dict]: """执行单条任务采集, 返回 parsed 结果 dict。失败返回 None。""" api_plat = (task.get("platform") or "").strip().lower() internal = API_TO_INTERNAL.get(api_plat) if not internal: print(f" ✗ 不支持的平台: {api_plat}") return None prompt = (task.get("prompt") or "").strip() if not prompt: print(" ✗ 题目为空") return None deep = bool(int(task.get("deepThinking") or 0)) print(f" → 采集 {api_plat}/{internal} DT={int(deep)} pool_id={COOKIE_PLATFORM_IDS.get(internal)} | {prompt[:40]}...") try: if internal == "ds": return await self._collect_ds(prompt, deep, task) if internal == "kimi": return await self._collect_kimi(prompt, task, deep=deep) if internal == "qianwen": return await self._collect_qianwen(prompt, deep, task) if internal == "wenxin": return await self._collect_wenxin(prompt, deep, task) if internal == "doubao": return await self._collect_doubao(prompt, deep, task) except Exception as e: print(f" ✗ 采集异常: {e}") traceback.print_exc() return None return None # ------ 各平台采集 ------ async def _collect_ds(self, prompt, deep, task) -> Optional[dict]: client = await self._ensure_ds_client() if not client: print(" ✗ DeepSeek 客户端不可用") return None try: raw_body, raw_req = await asyncio.to_thread(client.chat, prompt, deep) except Exception as e: err = str(e) if "Authorization Failed" in err or "invalid token" in err: async with self._ds_lock: self._drop_session("ds", "invalid") elif "user is muted" in err: async with self._ds_lock: self._drop_session("ds", "muted") raise parsed = self._parse_and_check("ds", raw_body, raw_req, prompt, deep) if parsed: sid = get_ds_session_id(parsed) if sid: try: sr = req_lib.post( "https://chat.deepseek.com/api/v0/share/create", headers=client.headers, json={"chat_session_id": sid, "message_ids": [1, 2]}, timeout=30, verify=False, proxies=NO_PROXY, ) biz = ((sr.json() or {}).get("data") or {}).get("biz_data") or {} sh_id = biz.get("share_id", "") if isinstance(biz, dict) else "" if sh_id: parsed["result"]["share_link"] = f"https://chat.deepseek.com/share/{sh_id}" except Exception: pass return parsed async def _collect_kimi(self, prompt, task, deep: bool = False, max_retry=2) -> Optional[dict]: """ Kimi 采集: 使用 KimiClient (最新 SCENARIO_K2D5 请求体) tools: SEARCH + CRON_JOB options: thinking / enable_plugin / reasoning_effort deep: 任务 deepThinking → options.thinking """ overload_retries = 0 max_overload = 5 for attempt in range(max_retry + 1 + max_overload): client = await self._ensure_kimi_client() if not client: print(" ✗ Kimi 客户端不可用") return None try: raw_body, raw_req = await asyncio.to_thread( client.chat, prompt, bool(deep), ) except Exception as e: err = str(e) if "unauthenticated" in err or "请登录" in err: async with self._kimi_lock: self._drop_session("kimi", "invalid") if attempt < max_retry: print(f" ↻ Kimi 异常换号重试 {attempt + 1}/{max_retry}: {e}") await asyncio.sleep(random.uniform(3, 8)) continue raise diag = self._kimi_response_diag(raw_body) raw_len = len(raw_body) if raw_body is not None else 0 if diag in ("unauthenticated", "account_abnormal"): probe = self._kimi_probe_account(client) detail = f"chat={diag}, user_probe={probe}, raw={raw_len}B" if probe == "account_abnormal" or diag == "account_abnormal": print(f" ✗ Kimi 账号状态异常 (风控/封禁), {detail}") print(" Moonshot: 账号状态异常, 联系 support@moonshot.cn; 池内此类号应淘汰") else: print(f" ✗ Kimi 未认证, {detail}") async with self._kimi_lock: self._drop_session("kimi", "invalid") if attempt < max_retry: print(f" ↻ Kimi 换号重试 {attempt + 1}/{max_retry}") await asyncio.sleep(random.uniform(2, 5)) continue print(" ✗ Kimi 认证失败, 重试耗尽 — cookie 池 platform_id=4 账号均不可用, 请补新号") return None if diag == "overloaded": overload_retries += 1 if overload_retries <= max_overload: wait = random.uniform(3, 5) print(f" ⏳ Kimi 算力过载, 同号等待 {wait:.0f}s 后重试 ({overload_retries}/{max_overload})") await asyncio.sleep(wait) continue print(" ✗ Kimi 持续过载, 重试 {max_overload} 次仍失败") return None if diag == "quota": print(f" ✗ Kimi 配额用完 (响应 {raw_len}B)") async with self._kimi_lock: self._drop_session("kimi", "quota") if attempt < max_retry: print(f" ↻ Kimi 配额用完换号重试 {attempt + 1}/{max_retry}") await asyncio.sleep(random.uniform(5, 15)) continue print(" ✗ Kimi 配额用完, 重试耗尽") return None if diag.startswith("error"): print(f" ✗ Kimi API 错误 {diag} (响应 {raw_len}B)") if attempt < max_retry: async with self._kimi_lock: self._drop_session("kimi", "invalid") await asyncio.sleep(random.uniform(2, 5)) continue return None parsed = self._parse_and_check("kimi", raw_body, raw_req, prompt, bool(deep)) if parsed: src_count = len(parsed.get("sources") or []) if src_count == 0 and attempt < max_retry: print(f" ⚠ Kimi 信源为 0, 换号重试 ({attempt + 1}/{max_retry})") async with self._kimi_lock: self._drop_session("kimi", "invalid") await asyncio.sleep(random.uniform(3, 8)) continue chat_id = parsed.get("chat_id", "") msg_ids = parsed.get("message_ids", []) if chat_id and msg_ids and client: try: # 分享接口: JSON, 去掉 connect 协议头 (与 run.py share 一致) headers = client._build_headers() headers["content-type"] = "application/json" headers.pop("connect-protocol-version", None) resp = client.requests.post( "https://www.kimi.com/apiv2/kimi.gateway.chat.v1.ChatService/CreateChatShare", headers=headers, json={"chat_id": chat_id, "message_ids": msg_ids}, timeout=30, impersonate="chrome146", ) if resp.status_code == 200 and resp.content: raw = resp.content js = raw.find(b"{") if js >= 0: resp_data = json.loads(raw[js: raw.rfind(b"}") + 1]) share_id = (resp_data.get("share") or {}).get("id", "") or resp_data.get("id", "") if share_id: parsed["result"]["share_link"] = f"https://www.kimi.com/share/{share_id}" except Exception: pass return parsed # 解析为空: 可能仍是异常短响应 print(f" ✗ Kimi 解析为空 diag={diag} raw_len={raw_len}") if attempt < max_retry: async with self._kimi_lock: self._drop_session("kimi", "invalid") await asyncio.sleep(random.uniform(3, 6)) continue return None async def _collect_qianwen(self, prompt, deep, task) -> Optional[dict]: client = await self._ensure_qianwen_client() if not client: client = QianwenClient() self.qw_client = client try: raw_body, raw_req = await client.chat(prompt, thinking=deep) except Exception as e: err = str(e).lower() if "401" in err or "login" in err or "auth" in err: async with self._qw_lock: self._drop_session("qianwen", "invalid") raise parsed = self._parse_and_check("qianwen", raw_body, raw_req, prompt, deep) if parsed and client._cookie_str: session_id = get_qw_session_id(parsed) if session_id: try: share_id = await client.create_share(session_id) if share_id: parsed["result"]["share_link"] = f"https://www.qianwen.com/share/chat/{share_id}" except Exception: pass return parsed async def _collect_wenxin(self, prompt, deep, task) -> Optional[dict]: client = await self._ensure_wenxin_client() if not client: client = WenxinClient() self.wx_client = client raw_body, raw_req = await client.chat(prompt, thinking=deep) parsed = self._parse_and_check("wenxin", raw_body, raw_req, prompt, deep) if parsed: lid = parsed.get("conversation_lid", "") if lid: page_link = f"https://chat.baidu.com/search/{lid}" if self.wenxin_browser_share and self.wx_share_page: try: share_url = await get_wenxin_share_url(lid, wx_page=self.wx_share_page) parsed["result"]["share_link"] = share_url or page_link except Exception: parsed["result"]["share_link"] = page_link else: parsed["result"]["share_link"] = page_link return parsed async def _doubao_create_share(self, conv_id: str) -> str: """页内 samantha 两步 API 生成分享链接""" if not conv_id or not self.doubao_adapter: return "" try: share_data = await self.doubao_adapter.page.evaluate("""async (convId) => { const aid = "497858"; try { const r1 = await fetch("/samantha/thread/share/info?aid=" + aid, { method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({conversation_id: convId}), credentials: "include" }); const d1 = await r1.json(); const shareToken = d1?.data?.share_token || ""; const preShareId = d1?.data?.pre_share_id || ""; const r2 = await fetch("/samantha/thread/share/save?aid=" + aid, { method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({ conversation_id: convId, share_token: shareToken, share_id: preShareId, message_index_end: 2, share_type: 1, is_allow_seo: false }), credentials: "include" }); const d2 = await r2.json(); const sid = d2?.data?.share_id || ""; return sid ? "https://www.doubao.com/thread/" + sid : ""; } catch { return ""; } }""", conv_id) return share_data or "" except Exception as e: print(f" [doubao] 分享链接失败: {e}") return "" async def _doubao_rotate_cookie(self) -> bool: """换一个 cookie 池账号并重新进入聊天页。成功返回 True""" if not self.doubao_use_cookie_pool or not self._ctx: return False self._drop_session("doubao", "invalid") try: await self._ctx.clear_cookies() except Exception: pass sess = self.pool.get_for_internal("doubao") if not sess or not sess.get("cookie"): print(" [doubao] 换号失败: 池中无更多 cookie") return False try: n = await self._inject_doubao_cookies(sess["cookie"]) self._doubao_cookie_str = sess["cookie"] self._session_ids["doubao"] = sess.get("id") page = self._doubao_page or self.doubao_adapter.page await page.goto("https://www.doubao.com/chat", wait_until="domcontentloaded", timeout=60000) await asyncio.sleep(2) if not await self._check_doubao_login(page): print(f" [doubao] 换号后仍未登录 id={sess.get('id')}") self._drop_session("doubao", "invalid") return False await self._bind_doubao_page(page) print(f" [doubao] 已换号 id={sess.get('id')} cookies={n}") return True except Exception as e: print(f" [doubao] 换号异常: {e}") return False async def _collect_doubao(self, prompt, deep, task) -> Optional[dict]: """ 豆包自动化采集 (与 run.py DoubaoAdapter 一致): new_chat → CDP 拦截 completion → 切专家/快速 → 发送 → 等流结束 → 解析 → 分享 """ if deep and self._doubao_dt_exhausted: print(" ⚠ 豆包专家次数已用完, 跳过 DT 任务") return None if not self.doubao_adapter: print(" ✗ 豆包 adapter 未初始化 (platform_switches.doubao 是否为 true?)") return None async with self._db_lock: for attempt in range(max(1, self.doubao_max_retry + 1)): if is_stopping(): return None try: print(f" [doubao] 开始采集 DT={int(deep)} attempt={attempt + 1} " f"pool_sid={self._session_ids.get('doubao')}") await self.doubao_adapter.new_chat() # 登录墙检测 if not await self._check_doubao_login(self.doubao_adapter.page): print(" ✗ 豆包未登录") if self.doubao_cookie_rotate and attempt < self.doubao_max_retry: if await self._doubao_rotate_cookie(): continue return None await self.doubao_adapter.setup_intercept() await self.doubao_adapter.set_deep_thinking(deep) await self.doubao_adapter.send_question(prompt) # 自定义超时 try: await asyncio.wait_for( self.doubao_adapter._finished_event.wait(), timeout=self.doubao_wait_timeout, ) except asyncio.TimeoutError: print(f" [WARN] 豆包响应超时 ({self.doubao_wait_timeout}s), 仍尝试取已捕获数据") # 流式收尾 (与 run.py wait_done 尾部一致, 略缩短) await asyncio.sleep(2) for _ in range(30): if is_stopping(): break try: if "/chat/" in (self.doubao_adapter.page.url or ""): break except Exception: pass await asyncio.sleep(1) await asyncio.sleep(1) raw_body, raw_req = self.doubao_adapter.get_last_response() if not raw_body: print(" ✗ 豆包未捕获到 chat/completion 响应") # 可能触发验证码或未登录 page_txt = "" try: page_txt = await self.doubao_adapter.page.inner_text("body", timeout=3000) except Exception: pass if any(k in (page_txt or "") for k in ("验证", "滑块", "登录")): print(" [doubao] 页面疑似验证码/登录拦截") if self.doubao_cookie_rotate and attempt < self.doubao_max_retry: if await self._doubao_rotate_cookie(): continue if attempt < self.doubao_max_retry: await asyncio.sleep(2) continue return None # 原始可能很大, 先落盘再解析 if isinstance(raw_body, bytes): raw_str = raw_body.decode("utf-8", errors="replace") else: raw_str = raw_body # 简单错误关键字 if any(k in raw_str[:500] for k in ("login", "未登录", "401", "403")): print(" ✗ 豆包响应含鉴权错误") if self.doubao_cookie_rotate and attempt < self.doubao_max_retry: if await self._doubao_rotate_cookie(): continue parsed = self._parse_and_check("doubao", raw_body, raw_req, prompt, deep) if parsed is None: if deep: # 可能是专家配额用完 (配额话术已在 _parse_and_check 处理) ans_head = (raw_str or "")[:300] if any(k in ans_head for k in ("次数", "专家", "升级", "额度")): self._doubao_dt_exhausted = True self._drop_session("doubao", "quota") print(" ⚠ 豆包专家模式可能已耗尽") if attempt < self.doubao_max_retry: continue return None if self.doubao_share: conv_id = parsed.get("conversation_id", "") if conv_id: share = await self._doubao_create_share(conv_id) if share: parsed["result"]["share_link"] = share print(f" [doubao] 分享链接 OK") else: print(f" [doubao] 分享链接未生成 (conv={conv_id[:16]}...)") return parsed except Exception as e: print(f" ✗ 豆包采集异常: {e}") traceback.print_exc() if self.doubao_cookie_rotate and attempt < self.doubao_max_retry: if await self._doubao_rotate_cookie(): continue return None return None # ------ 清理 ------ async def close(self): try: if self._cdp: await self._cdp.detach() except Exception: pass try: if self._pw: await self._pw.stop() except Exception: pass