chore: auto release commit (2026-06-15 17:32:47)
All checks were successful
技能自动化发布 / release (push) Successful in 7s

This commit is contained in:
2026-06-15 17:32:49 +08:00
parent 87ef00835b
commit b267ca3266
43 changed files with 3504 additions and 206 deletions

View File

@@ -0,0 +1,199 @@
"""account-manager CLI 集成(仅 subprocess不 import 兄弟 skill 内部模块)。"""
from __future__ import annotations
import json
import logging
import os
import subprocess
import sys
from typing import Any, Dict, List, Optional
from util.constants import LEASE_HOLDER, LEASE_TTL_SEC, TARGET_PLATFORM
from util.logging import mask_text
logger = logging.getLogger(__name__)
PLACEHOLDER_PLATFORM = TARGET_PLATFORM
ACCOUNT_SETUP_MESSAGE = (
f"未找到可用的 {PLACEHOLDER_PLATFORM} 账号,所以还没有打开浏览器。"
f"请先在 account-manager 中添加 platform={PLACEHOLDER_PLATFORM}"
"status=active、带 profile_dir 的账号后重新运行。"
)
LEASE_BUSY_MESSAGE = "目标平台账号当前被其他任务占用,请等待释放租约后重试。"
class AccountManagerError(Exception):
def __init__(self, code: str, message: str) -> None:
super().__init__(message)
self.code = code
self.message = message
def mask_login_id(login_id: str) -> str:
return mask_text(login_id)
def _resolve_account_manager_main() -> str:
"""解析 account-manager CLI 入口路径。
优先级ACCOUNT_MANAGER_ROOT → CLAW_SKILLS_ROOT / JIANGCHANG_SKILLS_ROOT → 开发机兜底。
末尾 dev 路径仅用于模板/本机调试,复制到真实 skill 后不要依赖个人机器绝对路径。
"""
env_root = (os.getenv("ACCOUNT_MANAGER_ROOT") or "").strip()
if env_root and os.path.isfile(os.path.join(env_root, "scripts", "main.py")):
return os.path.join(os.path.abspath(env_root), "scripts", "main.py")
skills_root = (os.getenv("CLAW_SKILLS_ROOT") or os.getenv("JIANGCHANG_SKILLS_ROOT") or "").strip()
if skills_root:
candidate = os.path.join(os.path.abspath(skills_root), "account-manager", "scripts", "main.py")
if os.path.isfile(candidate):
return candidate
# 开发环境兜底:仅模板/本机调试;生产依赖宿主注入的路径变量,勿硬编码个人目录
dev = r"D:\OpenClaw\client-commons\account-manager\scripts\main.py"
if os.path.isfile(dev):
return dev
raise AccountManagerError(
"ACCOUNT_NOT_FOUND",
"未找到 account-manager请配置 ACCOUNT_MANAGER_ROOT 或安装 account-manager 技能。",
)
def _parse_last_json(stdout: str) -> Any:
lines = [ln.strip() for ln in (stdout or "").splitlines() if ln.strip()]
for raw in reversed(lines):
if raw.startswith("ERROR:"):
continue
try:
return json.loads(raw)
except json.JSONDecodeError:
continue
raise AccountManagerError("UNKNOWN_ERROR", "stdout 中没有可解析的 JSON。")
def _run_argv(argv_suffix: List[str]) -> subprocess.CompletedProcess[str]:
main_py = _resolve_account_manager_main()
env = os.environ.copy()
return subprocess.run(
[sys.executable, main_py, *argv_suffix],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
env=env,
)
def _normalize_error_code(raw: str) -> str:
code = (raw or "").strip()
if code.startswith("ERROR:"):
code = code[6:]
return code
def _validate_pick_payload(payload: dict[str, Any]) -> dict[str, Any]:
if payload.get("success") is False:
err = payload.get("error") if isinstance(payload.get("error"), dict) else {}
code = _normalize_error_code(str(err.get("code") or ""))
message = str(err.get("message") or "")
if code == "LEASE_CONFLICT" or "LEASE_CONFLICT" in code:
raise AccountManagerError("LEASE_CONFLICT", LEASE_BUSY_MESSAGE)
if code in ("NO_ACCOUNT", "ACCOUNT_NOT_FOUND") or "NO_ACCOUNT" in code:
raise AccountManagerError("NO_ACCOUNT", message or "没有可用账号。")
raise AccountManagerError(code or "PICK_WEB_FAILED", message or "pick-web 失败。")
if not isinstance(payload, dict):
raise AccountManagerError("ACCOUNT_NOT_FOUND", "pick-web 返回格式异常。")
if not payload.get("profile_dir"):
raise AccountManagerError("ACCOUNT_NOT_FOUND", "pick-web 返回的账号缺少 profile_dir。")
return payload
def _pick_web_with_lease(platform: str) -> Dict[str, Any]:
proc = _run_argv(
[
"account",
"pick-web",
"--platform",
platform,
"--lease",
"--holder",
LEASE_HOLDER,
"--ttl-sec",
LEASE_TTL_SEC,
]
)
out = proc.stdout or ""
if proc.returncode != 0 and not out.strip():
raise AccountManagerError(
"PICK_WEB_FAILED",
(proc.stderr or "").strip() or "pick-web 子进程失败",
)
payload = _parse_last_json(out)
if isinstance(payload, dict) and payload.get("success") is False:
return _validate_pick_payload(payload)
if not isinstance(payload, dict):
raise AccountManagerError("ACCOUNT_NOT_FOUND", "pick-web 返回格式异常。")
return payload
def _pick_by_id(platform: str, account_id: int) -> Dict[str, Any]:
proc = _run_argv(["account", "get", str(account_id)])
out = proc.stdout or ""
if proc.returncode != 0 or out.startswith("ERROR:"):
raise AccountManagerError("ACCOUNT_NOT_FOUND", f"指定账号 {account_id} 不存在或获取失败。")
raw = _parse_last_json(out)
if isinstance(raw, dict) and "data" in raw and isinstance(raw["data"], dict):
data = raw["data"]
elif isinstance(raw, dict):
data = raw
else:
raise AccountManagerError("ACCOUNT_NOT_FOUND", "account get 返回非对象 JSON。")
platform_key = str(data.get("platform_key") or "").lower()
if platform_key != platform.lower():
raise AccountManagerError(
"ACCOUNT_NOT_FOUND",
f"账号 {account_id} 不是 {platform} 平台账号。",
)
profile_dir = str(data.get("profile_dir") or "").strip()
if not profile_dir:
raise AccountManagerError("ACCOUNT_NOT_FOUND", f"账号 {account_id} 缺少 profile_dir。")
data["profile_dir"] = profile_dir
data["lease_token"] = ""
return data
def pick_web_account(platform: str, account_id: Optional[str] = None) -> dict:
"""获取网页账号profile_dir + lease_token"""
platform_key = (platform or PLACEHOLDER_PLATFORM).strip() or PLACEHOLDER_PLATFORM
if account_id:
try:
aid = int(account_id)
except (TypeError, ValueError):
raise AccountManagerError("ACCOUNT_NOT_FOUND", f"账号 ID 无效:{account_id}")
return _pick_by_id(platform_key, aid)
try:
return _pick_web_with_lease(platform_key)
except AccountManagerError as exc:
if exc.code in ("NO_ACCOUNT", "ACCOUNT_NOT_FOUND") or "NO_ACCOUNT" in exc.code:
raise AccountManagerError("ACCOUNT_SETUP_REQUIRED", ACCOUNT_SETUP_MESSAGE) from exc
raise
def release_lease(lease_token: Optional[str]) -> None:
token = (lease_token or "").strip()
if not token:
return
try:
proc = _run_argv(["lease", "release", token])
if proc.returncode != 0:
logger.warning("lease_release_failed stdout=%s", (proc.stdout or "").strip())
except Exception:
logger.warning("lease_release_exception", exc_info=True)

View File

@@ -0,0 +1,135 @@
"""浏览器会话启动(系统 Chrome/Edge + Playwright persistent context"""
from __future__ import annotations
import logging
import os
from typing import Any, Optional, Tuple
from util.constants import DEFAULT_START_URL
logger = logging.getLogger(__name__)
DEFAULT_PAGE_TIMEOUT_MS = 15_000
GOTO_TIMEOUT_MS = 60_000
STEALTH_INIT_SCRIPT = """
(() => {
try { Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); } catch (e) {}
})();
"""
def get_start_url() -> str:
return (os.getenv("TARGET_START_URL") or DEFAULT_START_URL).strip() or DEFAULT_START_URL
def _headless_from_env() -> bool:
v = (os.getenv("OPENCLAW_BROWSER_HEADLESS") or "0").strip().lower()
return v in ("1", "true", "yes", "on")
def _find_chrome_executable() -> str | None:
try:
from jiangchang_skill_core.runtime_env import find_chrome_executable
chrome = find_chrome_executable()
if chrome and os.path.isfile(chrome):
return chrome
except ImportError:
pass
candidates = [
os.path.expandvars(r"%ProgramFiles%\Google\Chrome\Application\chrome.exe"),
os.path.expandvars(r"%ProgramFiles(x86)%\Google\Chrome\Application\chrome.exe"),
os.path.expandvars(r"%ProgramFiles%\Microsoft\Edge\Application\msedge.exe"),
os.path.expandvars(r"%ProgramFiles(x86)%\Microsoft\Edge\Application\msedge.exe"),
]
for path in candidates:
if path and os.path.isfile(path):
return path
return None
async def start_browser_session(
profile_dir: str,
*,
headless: Optional[bool] = None,
) -> Tuple[Any, Any, Any]:
"""
启动持久化 Chrome profilenew_page + goto 入口页,返回 (playwright, context, page)。
真实 skill 需要安装 playwright 与 jiangchang_skill_core。
"""
from playwright.async_api import async_playwright
hl = headless if headless is not None else _headless_from_env()
start_url = get_start_url()
chrome = _find_chrome_executable()
if not chrome:
raise RuntimeError(
"ERROR:MISSING_BROWSER 未检测到 Chrome 或 Edge 浏览器,请先安装系统浏览器后重试。"
)
launch_args = ["--start-maximized", "--disable-blink-features=AutomationControlled"]
logger.info(
"browser_launch chrome=%s profile_dir=%s start_url=%s args=%s headless=%s",
chrome,
profile_dir,
start_url,
launch_args,
hl,
)
print(f"[浏览器] 系统 Chrome: {chrome}")
print(f"[浏览器] profile_dir: [{profile_dir}]")
print(f"[浏览器] startup_url={start_url}")
print(f"[浏览器] args={launch_args}")
pw = await async_playwright().start()
context = None
try:
context = await pw.chromium.launch_persistent_context(
user_data_dir=profile_dir,
headless=hl,
executable_path=chrome,
locale="zh-CN",
no_viewport=True,
args=launch_args,
ignore_default_args=["--enable-automation"],
)
await context.add_init_script(STEALTH_INIT_SCRIPT)
page = await context.new_page()
page.set_default_timeout(DEFAULT_PAGE_TIMEOUT_MS)
try:
await page.goto(start_url, wait_until="domcontentloaded", timeout=GOTO_TIMEOUT_MS)
except Exception as exc:
raise RuntimeError(
f"浏览器已启动,但未能打开目标首页 {start_url}"
"请检查网络、Chrome profile、代理或页面访问状态。"
f"{exc}"
) from exc
return pw, context, page
except Exception:
if context is not None:
try:
await context.close()
except Exception:
pass
await pw.stop()
raise
async def close_browser_context(pw, context) -> None:
try:
if context is not None:
await context.close()
except Exception:
pass
try:
if pw is not None:
await pw.stop()
except Exception:
pass

View File

@@ -0,0 +1,450 @@
"""人工验证等待:保存登录倒计时、滑块、短信验证码等(不自动破解)。"""
from __future__ import annotations
import asyncio
import logging
import random
import re
import time
from dataclasses import dataclass
from typing import Optional
from util.constants import LOG_LOGGER_NAME
logger = logging.getLogger(LOG_LOGGER_NAME)
TEXT_EXCERPT_MAX = 120
SAVE_LOGIN_STRONG_MARKERS = (
"是否保存登录信息",
"下次登录更便捷",
"保存登录信息",
)
SAVE_LOGIN_CONTAINER_SELECTORS = (
"[role=\"dialog\"]",
".semi-modal",
".semi-modal-content",
)
SLIDER_SELECTORS = [
".vc-captcha-verify.slide",
".captcha_verify_bar--title",
".captcha-slider-tips",
"#captcha_verify_image",
".captcha-slider-btn",
]
SMS_SELECTORS = [
".second-verify-panel",
".uc-ui-verify_sms-input",
".uc-ui-verify-new_header-title",
"input[placeholder='手机号']",
"input[placeholder='请输入验证码']",
]
DIALOG_TEXT_CONTAINERS = [
"[role='dialog']",
".semi-modal",
".semi-modal-content",
".vc-captcha-verify",
".second-verify-panel",
]
RISK_MARKERS = (
"操作频繁",
"访问过于频繁",
"风险验证",
"安全验证",
)
@dataclass
class HumanVerificationState:
present: bool
kind: str
message: str
selector: str = ""
text_excerpt: str = ""
@dataclass
class HumanVerificationWaitResult:
timed_out: bool
kind: str = ""
code: str = ""
message: str = ""
stage: str = ""
def _text_excerpt(text: str) -> str:
cleaned = re.sub(r"\s+", " ", (text or "").strip())
if len(cleaned) <= TEXT_EXCERPT_MAX:
return cleaned
return cleaned[:TEXT_EXCERPT_MAX] + "..."
def _contains_any(text: str, markers: tuple[str, ...]) -> bool:
return any(m in text for m in markers)
async def _selector_visible(page, selector: str) -> bool:
try:
loc = page.locator(selector).first
return await loc.is_visible(timeout=500)
except Exception:
return False
async def _selector_inner_text(page, selector: str) -> str:
try:
return await page.locator(selector).first.inner_text(timeout=1000)
except Exception:
return ""
async def _visible_text_from_first_visible(page, selectors: tuple[str, ...] | list[str]) -> tuple[str, str]:
for sel in selectors:
try:
loc = page.locator(sel).first
if await loc.is_visible(timeout=500):
text = await loc.inner_text(timeout=1000)
return sel, text or ""
except Exception:
continue
return "", ""
def _save_login_popup_from_text(text: str) -> bool:
if not text:
return False
if _contains_any(text, SAVE_LOGIN_STRONG_MARKERS):
return True
if "保存" in text and "取消" in text and ("登录" in text or "登录信息" in text):
return True
return False
async def detect_save_login_popup(page) -> bool:
sel, text = await _visible_text_from_first_visible(page, SAVE_LOGIN_CONTAINER_SELECTORS)
if sel and _save_login_popup_from_text(text):
return True
return False
async def wait_save_login_countdown_if_present(page, timeout_sec: int = 8) -> bool:
"""若出现“保存登录信息”倒计时弹窗则等待其消失;未出现则立即返回 False。"""
if not await detect_save_login_popup(page):
return False
logger.info("save_login_popup_detected")
print("[登录] 检测到“保存登录信息”倒计时弹窗,等待其自动关闭...")
start = time.monotonic()
deadline = start + timeout_sec
while time.monotonic() < deadline:
if not await detect_save_login_popup(page):
elapsed = round(time.monotonic() - start, 2)
logger.info("save_login_popup_gone elapsed_sec=%s", elapsed)
return True
await asyncio.sleep(0.5)
elapsed = round(time.monotonic() - start, 2)
logger.warning(
"save_login_popup_still_visible elapsed_sec=%s timeout_sec=%s",
elapsed,
timeout_sec,
)
print(
f"[登录] 警告:保存登录信息弹窗超过 {timeout_sec} 秒仍未消失,"
"将继续后续流程(您可能已手工点击保存/取消)。"
)
return True
async def detect_slider_verification(page) -> Optional[HumanVerificationState]:
sel_slide = ".vc-captcha-verify.slide"
if await _selector_visible(page, sel_slide):
text = await _selector_inner_text(page, sel_slide)
return HumanVerificationState(
present=True,
kind="slider",
message="滑块验证码",
selector=sel_slide,
text_excerpt=_text_excerpt(text),
)
sel_title = ".captcha_verify_bar--title"
if await _selector_visible(page, sel_title):
title_text = await _selector_inner_text(page, sel_title)
if "请完成下列验证后继续" in title_text:
return HumanVerificationState(
present=True,
kind="slider",
message="滑块验证码",
selector=sel_title,
text_excerpt=_text_excerpt(title_text),
)
sel_tips = ".captcha-slider-tips"
if await _selector_visible(page, sel_tips):
tips_text = await _selector_inner_text(page, sel_tips)
if "拖动完成上方拼图" in tips_text:
return HumanVerificationState(
present=True,
kind="slider",
message="滑块验证码",
selector=sel_tips,
text_excerpt=_text_excerpt(tips_text),
)
if await _selector_visible(page, "#captcha_verify_image"):
for btn_sel in (".captcha-slider-btn", ".captcha_verify_slide--button", ".captcha-slider"):
if await _selector_visible(page, btn_sel):
return HumanVerificationState(
present=True,
kind="slider",
message="滑块验证码",
selector=f"#captcha_verify_image+{btn_sel}",
text_excerpt="",
)
return None
async def detect_sms_verification(page) -> Optional[HumanVerificationState]:
sel_panel = ".second-verify-panel"
if await _selector_visible(page, sel_panel):
text = await _selector_inner_text(page, sel_panel)
return HumanVerificationState(
present=True,
kind="sms",
message="短信验证码",
selector=sel_panel,
text_excerpt=_text_excerpt(text),
)
sel_sms = ".uc-ui-verify_sms-input"
if await _selector_visible(page, sel_sms):
text = await _selector_inner_text(page, sel_sms)
return HumanVerificationState(
present=True,
kind="sms",
message="短信验证码",
selector=sel_sms,
text_excerpt=_text_excerpt(text),
)
sel_hdr = ".uc-ui-verify-new_header-title"
if await _selector_visible(page, sel_hdr):
hdr_text = await _selector_inner_text(page, sel_hdr)
if "短信验证" in hdr_text:
return HumanVerificationState(
present=True,
kind="sms",
message="短信验证码",
selector=sel_hdr,
text_excerpt=_text_excerpt(hdr_text),
)
phone_visible = await _selector_visible(
page,
".second-verify-panel input[placeholder='手机号'], .uc-ui-verify_sms-input input[placeholder='手机号']",
)
code_visible = await _selector_visible(
page,
".second-verify-panel input[placeholder='请输入验证码'], .uc-ui-verify_sms-input input[placeholder='请输入验证码']",
)
if phone_visible and code_visible:
return HumanVerificationState(
present=True,
kind="sms",
message="短信验证码",
selector="sms-phone-and-code-inputs",
text_excerpt="手机号+请输入验证码",
)
return None
async def detect_human_verification_from_containers(page) -> Optional[HumanVerificationState]:
sel, text = await _visible_text_from_first_visible(page, DIALOG_TEXT_CONTAINERS)
if not sel or not text:
return None
if _contains_any(text, RISK_MARKERS):
return HumanVerificationState(
present=True,
kind="risk",
message="安全/频控验证",
selector=sel,
text_excerpt=_text_excerpt(text),
)
if sel in (".vc-captcha-verify", ".vc-captcha-verify-visibility"):
if "请完成下列验证后继续" in text or "拖动完成上方拼图" in text:
return HumanVerificationState(
present=True,
kind="slider",
message="滑块验证码",
selector=sel,
text_excerpt=_text_excerpt(text),
)
if sel in (".second-verify-panel", ".uc-ui-verify_sms-input"):
if "短信验证" in text or "获取验证码" in text:
return HumanVerificationState(
present=True,
kind="sms",
message="短信验证码",
selector=sel,
text_excerpt=_text_excerpt(text),
)
return None
async def detect_human_verification(page) -> HumanVerificationState:
"""主检测入口:仅在验证码/弹窗容器级检测,不扫整页 body。"""
state = await detect_slider_verification(page)
if state is not None:
return state
state = await detect_sms_verification(page)
if state is not None:
return state
state = await detect_human_verification_from_containers(page)
if state is not None:
return state
return HumanVerificationState(present=False, kind="", message="", selector="", text_excerpt="")
def detect_human_verification_from_text(text: str) -> HumanVerificationState:
"""
根据 HTML/容器文本判断验证弹窗(单元测试辅助,不能作为真实页面主检测入口)。
只检查验证容器 DOM 标记,不根据正文关键词误判。
"""
html_lower = (text or "").lower()
has_slider_dom = (
"vc-captcha-verify" in html_lower
and "slide" in html_lower
and (
"captcha-slider-tips" in html_lower
or "captcha_verify_bar--title" in html_lower
or "captcha_verify_image" in html_lower
)
)
if has_slider_dom:
selector = ".vc-captcha-verify.slide"
if "captcha-slider-tips" in html_lower:
selector = ".captcha-slider-tips"
return HumanVerificationState(
present=True,
kind="slider",
message="滑块验证码",
selector=selector,
text_excerpt="",
)
has_sms_dom = "second-verify-panel" in html_lower or "uc-ui-verify_sms-input" in html_lower
if has_sms_dom:
selector = ".second-verify-panel" if "second-verify-panel" in html_lower else ".uc-ui-verify_sms-input"
return HumanVerificationState(
present=True,
kind="sms",
message="短信验证码",
selector=selector,
text_excerpt="",
)
return HumanVerificationState(present=False, kind="", message="", selector="", text_excerpt="")
def _wait_log_message(state: HumanVerificationState, wait_sec: int) -> str:
if state.kind == "slider":
return f"[人工验证] 检测到滑块验证码,请在浏览器中手工完成,最多等待 {wait_sec} 秒..."
if state.kind == "sms":
return f"[人工验证] 检测到短信验证码,请在浏览器中手工完成,最多等待 {wait_sec} 秒..."
return f"[人工验证] 检测到安全验证,请在浏览器中手工处理,最多等待 {wait_sec} 秒..."
def _timeout_error_code(kind: str) -> str:
if kind == "slider":
return "SLIDER_VERIFICATION_TIMEOUT"
if kind == "sms":
return "SMS_VERIFICATION_TIMEOUT"
return "HUMAN_VERIFICATION_TIMEOUT"
def _timeout_error_message(kind: str, wait_sec: int) -> str:
if kind == "slider":
return (
f"滑块验证码未在 {wait_sec} 秒内完成,任务已停止。"
"请重新运行任务并及时在浏览器中完成验证。"
)
if kind == "sms":
return (
f"短信验证码未在 {wait_sec} 秒内完成,任务已停止。"
"请重新运行任务并及时完成短信验证。"
)
return (
f"安全验证未在 {wait_sec} 秒内完成,任务已停止。"
"请重新运行任务并及时在浏览器中完成验证。"
)
async def wait_human_verification_if_present(
page,
*,
wait_sec: int = 180,
stage: str = "unknown",
) -> Optional[HumanVerificationWaitResult]:
"""
若存在人工验证则等待用户完成;无验证或已完成返回 None超时返回 HumanVerificationWaitResult。
不自动拖动滑块、不自动填写短信验证码。
"""
state = await detect_human_verification(page)
if not state.present:
return None
logger.warning(
"human_verification_detected stage=%s kind=%s selector=%s text=%s",
stage,
state.kind,
state.selector,
state.text_excerpt,
)
print(_wait_log_message(state, wait_sec))
last_state = state
start = time.monotonic()
deadline = start + wait_sec
while time.monotonic() < deadline:
state = await detect_human_verification(page)
if not state.present:
elapsed = round(time.monotonic() - start, 2)
logger.info(
"human_verification_passed stage=%s kind=%s elapsed_sec=%s",
stage,
last_state.kind,
elapsed,
)
await asyncio.sleep(random.uniform(1.0, 2.0))
return None
last_state = state
await asyncio.sleep(2.0)
logger.warning(
"human_verification_timeout stage=%s kind=%s wait_sec=%s selector=%s",
stage,
last_state.kind,
wait_sec,
last_state.selector,
)
return HumanVerificationWaitResult(
timed_out=True,
kind=last_state.kind,
code=_timeout_error_code(last_state.kind),
message=_timeout_error_message(last_state.kind, wait_sec),
stage=stage,
)

View File

@@ -0,0 +1,641 @@
"""真实网站关键词搜索 RPA 示例async Playwright"""
from __future__ import annotations
import asyncio
import logging
import os
import random
import re
import time
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Optional, Tuple
from service.browser_session import close_browser_context, get_start_url, start_browser_session
from service.human_verification import (
HumanVerificationWaitResult,
wait_human_verification_if_present,
wait_save_login_countdown_if_present,
)
from util.constants import (
DEFAULT_MAX_ITEMS,
DEFAULT_MAX_SCROLLS,
DEFAULT_NO_NEW_ROUNDS_LIMIT,
HUMAN_WAIT_TIMEOUT,
LOG_LOGGER_NAME,
RESULT_CONTAINER_SELECTOR,
RESULT_END_TEXT,
RESULT_ITEM_SELECTOR,
SCROLL_CONTAINER_SELECTORS,
SEARCH_BUTTON_SELECTOR,
SEARCH_INPUT_SELECTORS,
)
from util.logging import mask_text, safe_log_value
logger = logging.getLogger(LOG_LOGGER_NAME)
@dataclass
class ScrapeError:
code: str
message: str
browser_started: bool = False
stage: Optional[str] = None
@dataclass
class ScrapeRunResult:
success: bool
items: List[Dict[str, Any]] = field(default_factory=list)
stop_reason: Optional[str] = None
error: Optional[ScrapeError] = None
StepCallback = Callable[[str], None]
# 示例登录态 selector — 复制到新 skill 后必须用目标站点 F12/DOM 实测替换,禁止凭经验猜测或跨平台照抄
LOGGED_IN_SELECTORS = (
"a[href*='/user/self']",
"[data-user-avatar]",
".user-avatar",
)
LOGGED_OUT_SELECTOR = "button:has-text('登录')"
def format_stop_reason_for_user(stop_reason: str | None, collected_count: int) -> str:
"""将内部 stop_reason 枚举翻译为用户可理解的中文说明。"""
if stop_reason == "max_items":
return (
f"已达到本次默认采集规模({collected_count} 条)。"
"如需更多,可以继续说「继续采集更多」或「尽量多采集」。"
)
if stop_reason == "end_marker":
return "页面提示没有更多结果,本次采集已完成。"
if stop_reason == "no_new_rounds":
return "连续多次加载没有发现新结果,本次采集已完成。"
if stop_reason == "max_scrolls":
return (
"已达到本次页面加载保护范围,本次采集已完成。"
"如需更深度采集,可以说明「尽量多采集」。"
)
if stop_reason == "human_verification_timeout":
return "人工验证未在规定时间内完成,任务已停止。"
return "本次采集已完成。"
def _mask_account_ref(account: Dict[str, Any]) -> str:
for key in ("login_id", "phone", "mobile"):
raw = str(account.get(key) or "").strip()
if raw:
return mask_text(raw)
aid = account.get("id") or account.get("account_id")
if aid is not None:
return f"id:{aid}"
return "unknown"
def _step_cb(cb: Optional[StepCallback], text: str) -> None:
if cb:
cb(text)
async def _random_delay() -> None:
lo = int(os.getenv("RPA_STEP_DELAY_MIN_MS") or "900")
hi = int(os.getenv("RPA_STEP_DELAY_MAX_MS") or "2600")
await asyncio.sleep(random.uniform(lo / 1000.0, hi / 1000.0))
async def _scroll_wait() -> None:
await asyncio.sleep(random.uniform(1.2, 3.5))
def _headless() -> bool:
v = (os.getenv("OPENCLAW_BROWSER_HEADLESS") or "0").strip().lower()
return v in ("1", "true", "yes", "on")
def _no_new_limit() -> int:
try:
return int(os.getenv("RPA_NO_NEW_ROUNDS_LIMIT") or str(DEFAULT_NO_NEW_ROUNDS_LIMIT))
except ValueError:
return DEFAULT_NO_NEW_ROUNDS_LIMIT
async def _visible(locator) -> bool:
try:
return await locator.count() > 0 and await locator.is_visible(timeout=500)
except Exception:
return False
async def _is_logged_out(page) -> bool:
try:
btn = page.locator(LOGGED_OUT_SELECTOR).first
return await _visible(btn)
except Exception:
return False
async def _is_logged_in(page) -> bool:
if await _is_logged_out(page):
return False
for sel in LOGGED_IN_SELECTORS:
try:
loc = page.locator(sel).first
if await _visible(loc):
return True
except Exception:
continue
return False
async def _open_login_panel_if_needed(page) -> None:
if not await _is_logged_out(page):
return
try:
btn = page.locator(LOGGED_OUT_SELECTOR).first
if await _visible(btn):
await btn.click()
await asyncio.sleep(random.uniform(1.0, 3.0))
except Exception:
pass
async def _ensure_logged_in(page, *, wait_sec: int) -> None:
if await _is_logged_in(page):
return
await _open_login_panel_if_needed(page)
if await _is_logged_in(page):
return
print(f"[登录] 请在浏览器中完成登录,最多等待 {wait_sec} 秒...")
deadline = time.monotonic() + wait_sec
while time.monotonic() < deadline:
if await _is_logged_in(page):
print("[登录] 检测到登录成功")
return
await asyncio.sleep(2.0)
raise RuntimeError(
f"ERROR:LOGIN_TIMEOUT 浏览器已打开,但未完成登录。"
f"请重新运行任务并在浏览器打开后及时完成登录。"
)
async def _find_search_input(page) -> Tuple[Optional[Any], str]:
for sel in SEARCH_INPUT_SELECTORS:
try:
loc = page.locator(sel).first
if await _visible(loc):
return loc, sel
except Exception:
continue
candidates: List[Tuple[int, Any, str]] = []
try:
inputs = await page.locator("input").all()
except Exception:
inputs = []
for inp in inputs:
try:
if not await inp.is_visible():
continue
box = await inp.bounding_box()
if not box or box.get("width", 0) < 120:
continue
placeholder = await inp.get_attribute("placeholder") or ""
score = box.get("width", 0)
if "搜索" in placeholder:
score += 500
if box.get("y", 9999) < 200:
score += 200
candidates.append((score, inp, f"input placeholder={placeholder!r}"))
except Exception:
continue
if candidates:
candidates.sort(key=lambda x: x[0], reverse=True)
_, inp, sel = candidates[0]
return inp, sel
try:
ph = page.get_by_placeholder(re.compile("搜索"))
if await ph.count() > 0 and await ph.first.is_visible():
return ph.first, "placeholder=搜索"
except Exception:
pass
return None, ""
async def _find_search_button(page):
try:
btn = page.get_by_role("button", name="搜索")
if await btn.count() > 0 and await btn.first.is_visible():
return btn.first
except Exception:
pass
try:
btn = page.locator(SEARCH_BUTTON_SELECTOR).first
if await btn.count() > 0 and await btn.is_visible():
return btn
except Exception:
pass
return None
async def _wait_search_results(page, timeout_sec: float = 45.0) -> bool:
deadline = time.monotonic() + timeout_sec
while time.monotonic() < deadline:
try:
url = page.url or ""
if "/search" in url or "search" in url.lower():
if await page.locator(RESULT_CONTAINER_SELECTOR).count() > 0:
return True
except Exception:
pass
await asyncio.sleep(0.8)
return False
async def _has_end_marker(page) -> bool:
try:
container = page.locator(RESULT_CONTAINER_SELECTOR).first
if await container.count() > 0:
text = await container.inner_text(timeout=2000)
if RESULT_END_TEXT in text or "暂时没有更多了" in text:
return True
except Exception:
pass
try:
if await page.locator(f"text={RESULT_END_TEXT}").count() > 0:
return True
except Exception:
pass
return False
async def _mouse_wheel_scroll(page) -> Tuple[str, int]:
wheel_delta = random.randint(650, 1200)
for sel in SCROLL_CONTAINER_SELECTORS:
loc = page.locator(sel)
try:
if await loc.count() > 0 and await loc.first.is_visible():
await loc.first.hover()
await _random_delay()
await page.mouse.wheel(0, wheel_delta)
return sel, wheel_delta
except Exception:
continue
await page.mouse.wheel(0, wheel_delta)
return "page", wheel_delta
async def _parse_item_locator(item) -> Dict[str, Any]:
item_id = await item.get_attribute("data-item-id") or await item.get_attribute("id") or ""
title = ""
try:
title = (await item.inner_text(timeout=1000)).strip()[:200]
except Exception:
pass
return {
"item_id": item_id or title[:32] or str(id(item)),
"title": title,
}
async def _collect_visible_items(page, seen_ids: set[str]) -> List[Dict[str, Any]]:
new_items: List[Dict[str, Any]] = []
try:
items = await page.locator(RESULT_ITEM_SELECTOR).all()
except Exception:
items = []
for item in items:
try:
parsed = await _parse_item_locator(item)
item_id = parsed.get("item_id")
if not item_id or item_id in seen_ids:
continue
seen_ids.add(str(item_id))
new_items.append(parsed)
except Exception:
continue
return new_items
async def _count_visible_items(page) -> int:
try:
return await page.locator(RESULT_ITEM_SELECTOR).count()
except Exception:
return 0
def _fail_result(error: ScrapeError, *, collected: int = 0) -> ScrapeRunResult:
logger.warning(
"rpa_failed code=%s stage=%s message=%s collected=%s",
error.code,
error.stage,
error.message,
collected,
)
return ScrapeRunResult(success=False, error=error)
def _fail_on_verification_timeout(
verification_timeout: HumanVerificationWaitResult,
*,
collected: int = 0,
) -> ScrapeRunResult:
return _fail_result(
ScrapeError(
code=verification_timeout.code,
message=verification_timeout.message,
browser_started=True,
stage=verification_timeout.stage or "unknown",
),
collected=collected,
)
async def run_keyword_search_async(
account: Dict[str, Any],
keyword: str,
*,
max_items: int = DEFAULT_MAX_ITEMS,
max_scrolls: int = DEFAULT_MAX_SCROLLS,
data_dir: str = "",
batch_id: str = "",
step_callback: Optional[StepCallback] = None,
) -> ScrapeRunResult:
profile_dir = account.get("profile_dir") or ""
account_ref = _mask_account_ref(account)
wait_sec = int(os.getenv("HUMAN_WAIT_TIMEOUT") or str(HUMAN_WAIT_TIMEOUT))
no_new_limit = _no_new_limit()
logger.info(
"rpa_start keyword=%s max_items=%s max_scrolls=%s wait_sec=%s batch_id=%s account_ref=%s",
safe_log_value(keyword),
max_items,
max_scrolls,
wait_sec,
batch_id,
account_ref,
)
if not profile_dir:
err = ScrapeError(
"ACCOUNT_NOT_FOUND",
"账号缺少 profile_dir。",
browser_started=False,
stage="account_prepare",
)
return _fail_result(err)
pw = None
context = None
page = None
browser_started = False
collected_count = 0
try:
_step_cb(step_callback, "启动关键词搜索采集任务")
try:
pw, context, page = await start_browser_session(profile_dir, headless=_headless())
except RuntimeError as exc:
msg = str(exc)
err = ScrapeError(
"MISSING_BROWSER" if "MISSING_BROWSER" in msg else "UNKNOWN_ERROR",
msg.replace("ERROR:MISSING_BROWSER ", "").replace("ERROR:", ""),
browser_started="未能打开目标首页" in msg,
stage="browser_start" if "未能打开目标首页" in msg else "account_prepare",
)
return _fail_result(err)
browser_started = True
logger.info(
"browser_ready start_url=%s current_url=%s profile_dir_exists=%s",
get_start_url(),
page.url,
os.path.isdir(profile_dir),
)
_step_cb(step_callback, "打开目标站点首页")
await _random_delay()
verification_timeout = await wait_human_verification_if_present(
page, wait_sec=wait_sec, stage="browser_start"
)
if verification_timeout:
return _fail_on_verification_timeout(verification_timeout)
_step_cb(step_callback, "检查账号登录状态")
logger.info("login_check_start current_url=%s", page.url)
try:
await _ensure_logged_in(page, wait_sec=wait_sec)
except RuntimeError as exc:
msg = str(exc)
code = "LOGIN_TIMEOUT" if "LOGIN_TIMEOUT" in msg else "REQUIRE_LOGIN"
err = ScrapeError(
code,
msg.replace("ERROR:LOGIN_TIMEOUT ", "").replace("ERROR:", ""),
browser_started=True,
stage="login_check",
)
return _fail_result(err)
logger.info("login_check_done current_url=%s", page.url)
await wait_save_login_countdown_if_present(page)
verification_timeout = await wait_human_verification_if_present(
page, wait_sec=wait_sec, stage="login_check"
)
if verification_timeout:
return _fail_on_verification_timeout(verification_timeout)
_step_cb(step_callback, "定位搜索框")
search_input, input_selector = await _find_search_input(page)
if search_input is None:
err = ScrapeError(
"SEARCH_INPUT_NOT_FOUND",
"未找到搜索输入框。",
browser_started=True,
stage="search",
)
return _fail_result(err)
logger.info("search_input_found selector=%s", input_selector)
_step_cb(step_callback, f"输入关键词:{keyword}")
await search_input.click()
await _random_delay()
await page.keyboard.press("Control+A")
await page.keyboard.type(keyword, delay=random.randint(90, 240))
await _random_delay()
_step_cb(step_callback, "点击搜索")
search_btn = await _find_search_button(page)
search_method = "button"
if search_btn is not None:
await search_btn.click()
else:
search_method = "enter"
await page.keyboard.press("Enter")
logger.info(
"search_submitted keyword=%s method=%s current_url=%s",
safe_log_value(keyword),
search_method,
page.url,
)
await _random_delay()
verification_timeout = await wait_human_verification_if_present(
page, wait_sec=wait_sec, stage="search_verification"
)
if verification_timeout:
return _fail_on_verification_timeout(verification_timeout)
_step_cb(step_callback, "等待搜索结果")
if not await _wait_search_results(page):
verification_timeout = await wait_human_verification_if_present(
page, wait_sec=wait_sec, stage="search_results"
)
if verification_timeout:
return _fail_on_verification_timeout(verification_timeout)
if not await _wait_search_results(page, timeout_sec=10.0):
err = ScrapeError(
"RESULT_CONTAINER_NOT_FOUND",
"搜索结果未出现,且未检测到需要人工处理的验证码,请检查关键词或页面状态。",
browser_started=True,
stage="search",
)
return _fail_result(err)
container_count = await page.locator(RESULT_CONTAINER_SELECTOR).count()
logger.info(
"search_results_ready current_url=%s container_count=%s",
page.url,
container_count,
)
try:
await page.locator(RESULT_CONTAINER_SELECTOR).first.wait_for(state="visible", timeout=15_000)
except Exception:
err = ScrapeError(
"RESULT_CONTAINER_NOT_FOUND",
"搜索结果容器未出现。",
browser_started=True,
stage="search",
)
return _fail_result(err)
seen_ids: set[str] = set()
collected: List[Dict[str, Any]] = []
no_new_rounds = 0
stop_reason = "max_scrolls"
rounds_executed = 0
for scroll_round in range(max_scrolls + 1):
rounds_executed = scroll_round
_step_cb(step_callback, "读取当前结果列表")
visible_count = await _count_visible_items(page)
batch = await _collect_visible_items(page, seen_ids)
new_count = len(batch)
if batch:
collected.extend(batch)
no_new_rounds = 0
else:
no_new_rounds += 1
collected_count = len(collected)
end_marker = await _has_end_marker(page)
logger.info(
"collect_round round=%s max_scrolls=%s visible_count=%s new_count=%s "
"collected=%s no_new_rounds=%s no_new_limit=%s end_marker=%s current_url=%s",
scroll_round,
max_scrolls,
visible_count,
new_count,
collected_count,
no_new_rounds,
no_new_limit,
end_marker,
page.url,
)
if len(collected) >= max_items:
collected = collected[:max_items]
stop_reason = "max_items"
break
if end_marker:
stop_reason = "end_marker"
_step_cb(step_callback, "发现结果到底或达到采集数量")
break
if no_new_rounds >= no_new_limit:
stop_reason = "no_new_rounds"
logger.info(
"collect_no_new_stop round=%s no_new_rounds=%s no_new_limit=%s collected=%s",
scroll_round,
no_new_rounds,
no_new_limit,
collected_count,
)
break
if scroll_round >= max_scrolls:
break
_step_cb(step_callback, "滚动加载更多结果")
scroll_target, wheel_delta = await _mouse_wheel_scroll(page)
logger.info(
"scroll round=%s target=%s wheel_delta=%s",
scroll_round,
scroll_target,
wheel_delta,
)
await _scroll_wait()
verification_timeout = await wait_human_verification_if_present(
page, wait_sec=wait_sec, stage="collect"
)
if verification_timeout:
return _fail_on_verification_timeout(
verification_timeout,
collected=collected_count,
)
logger.info(
"collect_stop reason=%s collected=%s rounds=%s no_new_rounds=%s",
stop_reason,
collected_count,
rounds_executed,
no_new_rounds,
)
if not collected:
err = ScrapeError(
"NO_RESULTS",
"未采集到任何结果。",
browser_started=True,
stage="collect",
)
return _fail_result(err, collected=0)
_step_cb(step_callback, "采集完成")
logger.info(
"rpa_success collected=%s stop_reason=%s",
len(collected[:max_items]),
stop_reason,
)
return ScrapeRunResult(success=True, items=collected[:max_items], stop_reason=stop_reason)
except Exception as exc:
err = ScrapeError(
"UNKNOWN_ERROR",
str(exc),
browser_started=browser_started,
stage="rpa",
)
return _fail_result(err, collected=collected_count)
finally:
await close_browser_context(pw, context)

View File

@@ -0,0 +1,137 @@
"""轻量任务编排示例(无真实数据库依赖)。"""
from __future__ import annotations
import asyncio
import uuid
from typing import Any, Optional
from service.account_client import AccountManagerError, pick_web_account, release_lease
from service.task_rpa import (
ScrapeRunResult,
format_stop_reason_for_user,
run_keyword_search_async,
)
from util.constants import (
DEFAULT_MAX_ITEMS,
DEFAULT_MAX_SCROLLS,
SKILL_SLUG,
TARGET_PLATFORM,
)
def _error_payload(
code: str,
message: str,
*,
browser_started: Optional[bool] = None,
stage: Optional[str] = None,
) -> dict[str, Any]:
payload: dict[str, Any] = {
"success": False,
"error": {"code": code, "message": message},
}
if browser_started is not None:
payload["browser_started"] = browser_started
if stage:
payload["stage"] = stage
return payload
def _success_payload(
*,
keyword: str,
collected_count: int,
items: list[dict[str, Any]],
stop_reason: Optional[str],
) -> dict[str, Any]:
return {
"success": True,
"skill": SKILL_SLUG,
"keyword": keyword,
"collected_count": collected_count,
"items": items,
"stop_reason": stop_reason,
"stop_reason_message": format_stop_reason_for_user(stop_reason, collected_count),
}
async def run_keyword_search_task(
keyword: str,
*,
max_items: int = DEFAULT_MAX_ITEMS,
max_scrolls: int = DEFAULT_MAX_SCROLLS,
account_id: Optional[str] = None,
platform: str = TARGET_PLATFORM,
) -> dict[str, Any]:
"""
编排一次关键词搜索采集。
真实 skill 中还应:
- 写 SQLite 入库items 表)
- 写 scrape_tasks / task_logs 任务日志
- 使用 RpaVideoSession 录制步骤留痕
- 失败时 capture_failure 截图
"""
batch_id = uuid.uuid4().hex[:12]
data_dir = "" # 示例占位;真实 skill 使用 get_skill_data_dir()
lease_token: Optional[str] = None
account: Optional[dict[str, Any]] = None
try:
account = pick_web_account(platform, account_id)
lease_token = account.get("lease_token")
scrape_result: ScrapeRunResult = await run_keyword_search_async(
account,
keyword,
max_items=max_items,
max_scrolls=max_scrolls,
data_dir=data_dir,
batch_id=batch_id,
)
if not scrape_result.success:
err = scrape_result.error
return _error_payload(
err.code if err else "UNKNOWN_ERROR",
err.message if err else "未知错误",
browser_started=err.browser_started if err else True,
stage=err.stage if err else "rpa",
)
collected_count = len(scrape_result.items)
return _success_payload(
keyword=keyword,
collected_count=collected_count,
items=scrape_result.items[:10],
stop_reason=scrape_result.stop_reason,
)
except AccountManagerError as exc:
return _error_payload(
exc.code,
exc.message,
browser_started=False,
stage="account_prepare",
)
finally:
release_lease(lease_token)
def run_keyword_search_task_sync(
keyword: str,
*,
max_items: int = DEFAULT_MAX_ITEMS,
max_scrolls: int = DEFAULT_MAX_SCROLLS,
account_id: Optional[str] = None,
platform: str = TARGET_PLATFORM,
) -> dict[str, Any]:
return asyncio.run(
run_keyword_search_task(
keyword,
max_items=max_items,
max_scrolls=max_scrolls,
account_id=account_id,
platform=platform,
)
)

View File

@@ -0,0 +1,31 @@
"""示例常量(复制到新 skill 时请替换为目标站点与业务字段)。"""
SKILL_SLUG = "example-real-browser-rpa"
LOG_LOGGER_NAME = "openclaw.skill.example_real_browser_rpa"
TARGET_PLATFORM = "target_platform"
LEASE_HOLDER = "example-real-browser-rpa"
LEASE_TTL_SEC = "1800"
DEFAULT_START_URL = "https://example.com"
DEFAULT_MAX_ITEMS = 100
DEFAULT_MAX_SCROLLS = 40
DEFAULT_NO_NEW_ROUNDS_LIMIT = 5
HUMAN_WAIT_TIMEOUT = 180
RESULT_END_TEXT = "没有更多结果了"
# 示例 selector — 复制到新 skill 时按目标站点 DOM 替换
SEARCH_INPUT_SELECTORS = (
'input[type="search"]',
'input[name="q"]',
'input[placeholder*="搜索"]',
)
SEARCH_BUTTON_SELECTOR = 'button:has-text("搜索")'
RESULT_CONTAINER_SELECTOR = "#search-result-container"
RESULT_ITEM_SELECTOR = "[data-result-item]"
SCROLL_CONTAINER_SELECTORS = (
"#search-result-container",
".search-result-scroll",
".route-scroll-container",
)

View File

@@ -0,0 +1,34 @@
"""示例结构化日志工具。"""
from __future__ import annotations
import logging
import re
from typing import Any
from util.constants import LOG_LOGGER_NAME
def get_logger(name: str | None = None) -> logging.Logger:
return logging.getLogger(name or LOG_LOGGER_NAME)
def mask_text(value: str, *, keep_start: int = 2, keep_end: int = 2) -> str:
s = (value or "").strip()
if len(s) >= 11 and s.isdigit():
return s[:3] + "****" + s[-4:]
if len(s) > keep_start + keep_end:
return s[:keep_start] + "****" + s[-keep_end:]
return "****"
def safe_log_value(value: Any) -> str:
if value is None:
return ""
text = str(value)
if re.fullmatch(r"\d{7,}", text):
return mask_text(text)
if "@" in text and len(text) > 6:
local, _, domain = text.partition("@")
return f"{mask_text(local, keep_start=1, keep_end=1)}@{domain}"
return text