Initial commit

.env.example

+# Dify API
+# Example: http://localhost or https://your-dify.example.com
+DIFY_BASE_URL=
+DIFY_API_KEY=
+
+# Either set dataset id directly, or let the agent auto-create one by name.
+DIFY_DATASET_ID=
+DIFY_DATASET_NAME=湖北人社政策知识库
+DIFY_INDEXING_TECHNIQUE=high_quality
+DIFY_BATCH_SIZE=50
+
+# Local crawler
+POLICY_REQUEST_TIMEOUT=20
+POLICY_USER_AGENT=
+
+# Optional local overrides
+# POLICY_DB_PATH=
+# POLICY_SITES_PATH=

.gitignore

+.env
+.venv/
+data/
+logs/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.pytest_cache/
+.mypy_cache/
+.coverage
+htmlcov/
+.vscode/
+.claude/
+*.yml

README.md

+# 湖北人社政策采集代理
+
+这个仓库默认采集湖北省内各级人力资源和社会保障部门官网的政务公开政策栏目，并同步到 Dify 知识库。
+
+当前默认站点配置文件是 `policy_agent/sources/hubei_hrss_sites.json`，已内置 18 个湖北省内人社局/人社厅政策入口；站点清单见 `hubei_hrss_zwgk_links.md` 和 `hubei_hrss_zwgk_links.csv`。
+
+## 目录
+
+- `policy_agent/sources/hubei_hrss_sites.json`
+  湖北省人社系统站点配置
+- `policy_agent/crawler.py`
+  通用抓取与正文抽取，包含宜昌统一公开平台 API 适配
+- `policy_agent/storage.py`
+  SQLite 去重、版本管理、同步状态记录
+- `policy_agent/dify_client.py`
+  Dify Dataset API 封装
+- `policy_agent/cli.py`
+  命令行入口
+- `policy_agent/scripts/setup_local.ps1`
+  一键初始化脚本
+- `policy_agent/scripts/run_daily_update.ps1`
+  单次执行脚本
+- `policy_agent/scripts/register_midnight_task.ps1`
+  每天 0 点计划任务注册脚本
+
+## 运行要求
+
+- Windows PowerShell
+- Python 3.10 及以上
+- 可访问 Dify API
+- 如需 Playwright 渲染，允许下载 Chromium
+
+## 快速开始
+
+### 1. 克隆仓库
+
+```powershell
+git clone <your-repo-url>
+cd "Policy Collection Agent"
+```
+
+### 2. 一键初始化
+
+```powershell
+powershell -ExecutionPolicy Bypass -File .\policy_agent\scripts\setup_local.ps1
+```
+
+脚本会自动：
+
+- 创建仓库内 `.venv`
+- 安装 `requirements.txt`
+- 安装 Playwright Chromium
+- 若不存在 `.env`，从 `.env.example` 复制一份
+
+### 3. 配置 `.env`
+
+至少填写：
+
+```env
+DIFY_BASE_URL=
+DIFY_API_KEY=
+```
+
+可选配置：
+
+```env
+DIFY_DATASET_ID=
+DIFY_DATASET_NAME=湖北人社政策知识库
+DIFY_INDEXING_TECHNIQUE=high_quality
+DIFY_BATCH_SIZE=50
+POLICY_REQUEST_TIMEOUT=20
+POLICY_USER_AGENT=
+```
+
+说明：
+
+- 优先填写 `DIFY_DATASET_ID`
+- 如果未填 `DIFY_DATASET_ID`，程序会按 `DIFY_DATASET_NAME` 查找或自动创建知识库
+- 如果要临时切回别的站点文件，可用 `POLICY_SITES_PATH` 覆盖默认来源
+
+### 4. 检查配置
+
+```powershell
+.\.venv\Scripts\python.exe -m policy_agent.cli --project-root . check-config
+```
+
+### 5. 预览单站点
+
+```powershell
+.\.venv\Scripts\python.exe -m policy_agent.cli --project-root . preview --site "武汉市人力资源和社会保障局" --limit 5 --max-pages 20
+```
+
+### 6. 执行一次全量同步
+
+```powershell
+.\.venv\Scripts\python.exe -m policy_agent.cli --project-root . run --max-pages 20
+```
+
+## 常用命令
+
+```powershell
+.\.venv\Scripts\python.exe -m policy_agent.cli --project-root . check-config
+.\.venv\Scripts\python.exe -m policy_agent.cli --project-root . list-sites
+.\.venv\Scripts\python.exe -m policy_agent.cli --project-root . preview --site "湖北省人力资源和社会保障厅" --limit 10 --max-pages 20
+.\.venv\Scripts\python.exe -m policy_agent.cli --project-root . run --site "宜昌市人力资源和社会保障局" --max-pages 20
+.\.venv\Scripts\python.exe -m policy_agent.cli --project-root . run --max-pages 300 --max-depth 3
+```
+
+## 每天 0 点自动运行
+
+使用管理员 PowerShell：
+
+```powershell
+Set-Location "D:\Policy Collection Agent"
+.\policy_agent\scripts\register_midnight_task.ps1
+```
+
+默认任务名是 `HubeiPolicyDailyUpdate`，触发时间为每天 `00:00`。
+
+## 去重与版本规则
+
+- 同一 URL 且正文哈希不变：跳过
+- 同一 URL 但正文发生变化：更新现有记录
+- 不同 URL 但正文哈希相同：视为重复
+- 同一 `policy_key` 出现新版本时：旧版本标记为 `revised`，新版本标记为 `effective`
+- Dify 中旧 `document_id` 失效时：自动重建文档并回写新 ID
+
+## 本地文件位置
+
+- SQLite 数据文件默认在 `data/policy_agent.db`
+- 日志文件默认在 `logs/policy_agent.log`
+
+这些文件已经加入 `.gitignore`，不会污染仓库。
+
+## 当前说明
+
+- 默认配置聚焦“湖北人社系统政策公开栏目”，不再默认抓取综合政府门户站
+- 宜昌市人社局走统一公开平台 API，`deptid` 已改为可配置，便于后续迁移到其他宜昌部门
+- 如果 Dify 返回 `Could not connect to Weaviate`，说明是知识库向量库异常，不是采集代码本身的问题

collect_hubei_gov_sites.py

+from __future__ import annotations
+
+import csv
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable
+from urllib.parse import urljoin, urlparse
+
+import requests
+from bs4 import BeautifulSoup
+
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+    ),
+    "Accept-Language": "zh-CN,zh;q=0.9",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+}
+
+OUTPUT_MD = Path("hubei_government_zwgk_links.md")
+OUTPUT_CSV = Path("hubei_government_zwgk_links.csv")
+
+KEYWORDS = ("政务公开", "政府信息公开", "信息公开")
+FALLBACK_PATHS = (
+    "/zwgk/",
+    "/zwgk/index.shtml",
+    "/xxgk/",
+    "/zfxxgk/",
+    "/gk/",
+)
+
+
+@dataclass(frozen=True)
+class Region:
+    name: str
+    level: str
+    candidates: tuple[str, ...]
+
+
+REGIONS: tuple[Region, ...] = (
+    Region("湖北省人民政府", "省级", ("https://www.hubei.gov.cn/",)),
+    Region("武汉市人民政府", "副省级市", ("https://www.wuhan.gov.cn/",)),
+    Region("黄石市人民政府", "地级市", ("https://www.huangshi.gov.cn/",)),
+    Region("十堰市人民政府", "地级市", ("https://www.shiyan.gov.cn/",)),
+    Region("宜昌市人民政府", "地级市", ("https://www.yichang.gov.cn/",)),
+    Region("襄阳市人民政府", "地级市", ("https://www.xiangyang.gov.cn/",)),
+    Region("鄂州市人民政府", "地级市", ("https://www.ezhou.gov.cn/",)),
+    Region("荆门市人民政府", "地级市", ("https://www.jingmen.gov.cn/",)),
+    Region("孝感市人民政府", "地级市", ("https://www.xiaogan.gov.cn/",)),
+    Region("荆州市人民政府", "地级市", ("https://www.jingzhou.gov.cn/",)),
+    Region("黄冈市人民政府", "地级市", ("https://www.huanggang.gov.cn/", "https://www.hg.gov.cn/")),
+    Region("咸宁市人民政府", "地级市", ("https://www.xianning.gov.cn/",)),
+    Region("随州市人民政府", "地级市", ("https://www.suizhou.gov.cn/",)),
+    Region("恩施土家族苗族自治州人民政府", "自治州", ("https://www.enshi.gov.cn/",)),
+    Region("仙桃市人民政府", "省直辖县级市", ("https://www.xiantao.gov.cn/",)),
+    Region("潜江市人民政府", "省直辖县级市", ("https://www.hbqj.gov.cn/",)),
+    Region("天门市人民政府", "省直辖县级市", ("https://www.tianmen.gov.cn/",)),
+    Region("神农架林区人民政府", "林区", ("https://www.snj.gov.cn/",)),
+)
+
+
+def fetch(url: str) -> requests.Response | None:
+    try:
+        response = requests.get(url, headers=HEADERS, timeout=20, verify=False)
+        response.raise_for_status()
+        if not response.encoding or response.encoding.lower() == "iso-8859-1":
+            response.encoding = response.apparent_encoding or "utf-8"
+        return response
+    except requests.RequestException:
+        return None
+
+
+def simplify_text(text: str) -> str:
+    return "".join(text.split())
+
+
+def iter_candidate_links(soup: BeautifulSoup, base_url: str) -> Iterable[tuple[str, str]]:
+    for anchor in soup.find_all("a", href=True):
+        text = simplify_text(anchor.get_text(" ", strip=True))
+        href = anchor["href"].strip()
+        if not href or href.lower().startswith("javascript:"):
+            continue
+        yield text, urljoin(base_url, href)
+
+
+def find_zwgk_link(home_url: str, html: str) -> tuple[str | None, str]:
+    soup = BeautifulSoup(html, "html.parser")
+    candidates: list[tuple[int, str, str]] = []
+    for text, absolute_url in iter_candidate_links(soup, home_url):
+        if not any(keyword in text for keyword in KEYWORDS):
+            continue
+        score = 0
+        if "政务公开" in text:
+            score += 5
+        if "政府信息公开" in text:
+            score += 4
+        if "信息公开目录" in text:
+            score += 1
+        parsed = urlparse(absolute_url)
+        if parsed.netloc == urlparse(home_url).netloc:
+            score += 2
+        if any(token in absolute_url.lower() for token in ("/zwgk", "/xxgk", "/zfxxgk", "channelid")):
+            score += 2
+        candidates.append((score, text, absolute_url))
+
+    if candidates:
+        score, text, link = max(candidates, key=lambda item: item[0])
+        return link, text
+    return None, ""
+
+
+def verify_public_link(homepage: str, public_link: str) -> tuple[str | None, str]:
+    response = fetch(public_link)
+    if not response:
+        return None, "政务公开入口无法访问"
+
+    text = simplify_text(response.text[:3000])
+    title = simplify_text(BeautifulSoup(response.text, "html.parser").title.get_text()) if BeautifulSoup(response.text, "html.parser").title else ""
+    if any(keyword in text or keyword in title for keyword in KEYWORDS):
+        return response.url, "已验证"
+    return response.url, "入口可访问，页面关键词较弱"
+
+
+def probe_region(region: Region) -> dict[str, str]:
+    result = {
+        "地区": region.name,
+        "级别": region.level,
+        "官网首页": "",
+        "政务公开链接": "",
+        "状态": "官网不可访问",
+        "说明": "",
+    }
+
+    homepage_response = None
+    for candidate in region.candidates:
+        homepage_response = fetch(candidate)
+        if homepage_response:
+            result["官网首页"] = homepage_response.url
+            break
+
+    if not homepage_response:
+        return result
+
+    public_link, label = find_zwgk_link(homepage_response.url, homepage_response.text)
+    if public_link:
+        verified_link, note = verify_public_link(homepage_response.url, public_link)
+        if verified_link:
+            result["政务公开链接"] = verified_link
+            result["状态"] = "可访问"
+            result["说明"] = f"首页栏目文案：{label}；{note}"
+            return result
+
+    for path in FALLBACK_PATHS:
+        fallback_url = urljoin(homepage_response.url, path)
+        verified_link, note = verify_public_link(homepage_response.url, fallback_url)
+        if verified_link:
+            result["政务公开链接"] = verified_link
+            result["状态"] = "可访问"
+            if label:
+                result["说明"] = f"首页栏目文案：{label}；回退路径命中；{note}"
+            else:
+                result["说明"] = f"首页未直接识别栏目，回退路径命中；{note}"
+            return result
+
+    result["状态"] = "官网可访问，但未识别政务公开入口"
+    if label:
+        result["说明"] = f"首页发现相关栏目文案：{label}，但链接验证失败"
+    else:
+        result["说明"] = "首页与常见公开路径均未命中"
+    return result
+
+
+def write_markdown(rows: list[dict[str, str]]) -> None:
+    lines = [
+        "# 湖北省政府官网政务公开入口清单",
+        "",
+        "说明：本清单按“湖北省人民政府 + 各市州及省直辖县级行政区政府门户”整理，重点记录官网首页及其可访问的“政务公开/政府信息公开”入口。",
+        "",
+        "| 地区 | 级别 | 官网首页 | 政务公开链接 | 状态 | 说明 |",
+        "| --- | --- | --- | --- | --- | --- |",
+    ]
+    for row in rows:
+        home = row["官网首页"] or "-"
+        public = row["政务公开链接"] or "-"
+        note = row["说明"].replace("|", "\\|")
+        lines.append(
+            f"| {row['地区']} | {row['级别']} | {home} | {public} | {row['状态']} | {note or '-'} |"
+        )
+    OUTPUT_MD.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def write_csv(rows: list[dict[str, str]]) -> None:
+    fieldnames = ["地区", "级别", "官网首页", "政务公开链接", "状态", "说明"]
+    with OUTPUT_CSV.open("w", newline="", encoding="utf-8-sig") as file:
+        writer = csv.DictWriter(file, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def main() -> None:
+    requests.packages.urllib3.disable_warnings()  # type: ignore[attr-defined]
+    rows = [probe_region(region) for region in REGIONS]
+    write_markdown(rows)
+    write_csv(rows)
+    for row in rows:
+        print(
+            f"{row['地区']}\t{row['状态']}\t{row['官网首页']}\t{row['政务公开链接']}\t{row['说明']}"
+        )
+
+
+if __name__ == "__main__":
+    main()

debug_html.py

+# -*- coding: utf-8 -*-
+"""调试：打印页面关键 HTML 结构"""
+import sys, io
+sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
+
+import requests
+from bs4 import BeautifulSoup
+import re
+
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Accept-Language": "zh-CN,zh;q=0.9",
+}
+URL = "https://www.wuhan.gov.cn/zwgk/xxgk/zfgz_new/202602/t20260211_2728640.shtml"
+
+resp = requests.get(URL, headers=HEADERS, timeout=15)
+resp.encoding = "utf-8"
+soup = BeautifulSoup(resp.text, "html.parser")
+
+print("=== 所有 h1/h2 标签 ===")
+for tag in soup.find_all(["h1","h2"]):
+    print(f"  <{tag.name} class='{tag.get('class','')}' id='{tag.get('id','')}'>")
+    print(f"    文字: {tag.get_text(strip=True)[:80]}")
+
+print("\n=== 包含'政府规章'/'决定' 文字的元素 ===")
+for el in soup.find_all(string=re.compile(r'关于修改|政府规章|政府令')):
+    p = el.parent
+    print(f"  <{p.name} class='{p.get('class','')}'>: {str(el).strip()[:80]}")
+
+print("\n=== 文号模式（包含年号的文本）===")
+for el in soup.find_all(string=re.compile(r'第\d+号|〔\d{4}〕|\(\d{4}\)')):
+    p = el.parent
+    print(f"  <{p.name} class='{p.get('class','')}'>: {str(el).strip()[:100]}")
+
+print("\n=== class 含 'title'/'tit'/'head' 的 div/p/span ===")
+for el in soup.find_all(class_=re.compile(r'title|tit|head|subject', re.I)):
+    print(f"  <{el.name} class='{el.get('class','')}'>: {el.get_text(strip=True)[:80]}")

find_igs_api.py

+# -*- coding: utf-8 -*-
+"""
+find_igs_api.py
+暴力探测 IGS CMS（泰豪软件）常见 API 端点
+武汉市政府网站使用 IGS CMS 系统
+"""
+import sys, io
+sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
+
+import requests
+import json
+import re
+
+session = requests.Session()
+session.headers.update({
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Accept-Language": "zh-CN,zh;q=0.9",
+    "Referer": "https://www.wuhan.gov.cn/zwgk/?channelid=26164",
+})
+
+BASE = "https://www.wuhan.gov.cn"
+CHANNEL_ID = "26164"
+
+# 先访问一次列表页，获取 Cookie
+session.get(f"{BASE}/zwgk/?channelid={CHANNEL_ID}", timeout=10)
+
+# IGS CMS 所有已知 API 变体（根据泰豪 IGS 产品文档及社区逆向整理）
+CANDIDATES = [
+    # 标准 IGS CMS 文章列表接口
+    ("/igs/front/siteContent/getSiteContentList.jhtml",           {"channelId": CHANNEL_ID, "pageIndex": 1, "pageSize": 15}),
+    ("/igs/front/article/queryArticleList.jhtml",                 {"channelId": CHANNEL_ID, "pageIndex": 1, "pageSize": 15}),
+    ("/igs/front/article/getArticleList.jhtml",                   {"channelId": CHANNEL_ID, "pageIndex": 1, "pageSize": 15}),
+    ("/igs/front/content/getContentList.jhtml",                   {"channelId": CHANNEL_ID, "pageIndex": 1, "pageSize": 15}),
+    ("/igs/front/channel/getChildList.jhtml",                     {"channelId": CHANNEL_ID, "pageIndex": 1, "pageSize": 15}),
+    # IGS 新版 REST 格式
+    ("/igs/front/article/list",                                    {"channelId": CHANNEL_ID, "page": 1, "size": 15}),
+    ("/igs/front/v1/article/list",                                 {"channelId": CHANNEL_ID, "page": 1, "size": 15}),
+    # 武汉政府网特有路径（基于 URL 模式推断）
+    ("/zwgk/api/list",                                             {"channelId": CHANNEL_ID, "pageIndex": 1}),
+    ("/zwgk/ssi/queryList.jhtml",                                  {"channelId": CHANNEL_ID, "pageIndex": 1}),
+    # 站群系统常见格式
+    ("/ssi/front/queryArticleList.jhtml",                          {"channelId": CHANNEL_ID, "pageIndex": 1, "pageSize": 15}),
+    ("/ssi/front/article/queryList.jhtml",                         {"channelId": CHANNEL_ID, "pageIndex": 1, "pageSize": 15}),
+    # 尝试 channelid（小写）
+    ("/igs/front/article/queryArticleList.jhtml",                  {"channelid": CHANNEL_ID, "pageIndex": 1, "pageSize": 15}),
+    ("/igs/front/siteContent/getSiteContentList.jhtml",            {"channelid": CHANNEL_ID, "pageIndex": 1, "pageSize": 15}),
+]
+
+print("=" * 70)
+print(f"探测 IGS CMS API 端点（channelId={CHANNEL_ID}）")
+print("=" * 70)
+
+found = []
+for path, params in CANDIDATES:
+    url = BASE + path
+    # 尝试 POST
+    try:
+        r = session.post(url, data=params, timeout=8)
+        ct = r.headers.get("Content-Type", "")
+        body = r.text[:400]
+        is_json = "json" in ct or (body.strip().startswith("{") or body.strip().startswith("["))
+        status_str = f"POST [{r.status_code}]"
+        if r.status_code in (200, 201) and len(r.content) > 100 and r.status_code != 404:
+            note = "★ 可能有效" if is_json else "HTML响应"
+            print(f"{status_str} {path} {note}")
+            print(f"  CT={ct}  size={len(r.content)}")
+            print(f"  {body[:200]}")
+            if is_json:
+                found.append((url, "POST", params, body))
+        elif r.status_code not in (404, 301, 302, 403):
+            print(f"{status_str} {path}  CT={ct}")
+    except Exception as e:
+        pass
+
+    # 尝试 GET（带参数）
+    try:
+        r = session.get(url, params=params, timeout=8)
+        ct = r.headers.get("Content-Type", "")
+        body = r.text[:400]
+        is_json = "json" in ct or (body.strip().startswith("{") or body.strip().startswith("["))
+        status_str = f"GET  [{r.status_code}]"
+        if r.status_code in (200, 201) and len(r.content) > 100 and r.status_code != 404:
+            note = "★ 可能有效" if is_json else "HTML响应"
+            print(f"{status_str} {path} {note}")
+            print(f"  CT={ct}  size={len(r.content)}")
+            print(f"  {body[:200]}")
+            if is_json:
+                found.append((url, "GET", params, body))
+        elif r.status_code not in (404, 301, 302, 403):
+            print(f"{status_str} {path}  CT={ct}")
+    except Exception as e:
+        pass
+
+print("\n" + "=" * 70)
+print(f"发现 {len(found)} 个有效 JSON 接口")
+print("=" * 70)
+for url, method, params, body in found:
+    print(f"  [{method}] {url}  params={params}")
+    print(f"  响应: {body[:300]}")

find_list_api.py

+# -*- coding: utf-8 -*-
+"""
+find_list_api.py
+专门寻找武汉政府网政策列表的真实 API 端点
+通过 Playwright 拦截浏览器的所有网络请求
+"""
+import sys, io
+sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
+
+from playwright.sync_api import sync_playwright
+import json, re
+
+LIST_URL = "https://www.wuhan.gov.cn/zwgk/?channelid=26164"
+
+def find_list_api():
+    api_requests = []
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        context = browser.new_context(
+            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+            locale="zh-CN",
+        )
+        page = context.new_page()
+
+        # 拦截所有请求
+        def on_request(request):
+            url = request.url
+            method = request.method
+            # 过滤掉图片、CSS、字体等静态资源
+            if not any(url.endswith(ext) for ext in ['.png', '.jpg', '.gif', '.css', '.woff', '.woff2', '.ico']):
+                api_requests.append({
+                    "method": method,
+                    "url": url,
+                    "headers": dict(request.headers),
+                    "post_data": request.post_data,
+                })
+
+        # 拦截所有响应，特别关注 JSON 格式的
+        json_responses = []
+        def on_response(response):
+            url = response.url
+            ct = response.headers.get("content-type", "")
+            if "json" in ct or "jhtml" in url:
+                try:
+                    body = response.text()
+                    if body and len(body) > 50:
+                        json_responses.append({
+                            "url": url,
+                            "status": response.status,
+                            "content_type": ct,
+                            "body_preview": body[:500],
+                        })
+                except:
+                    pass
+
+        page.on("request", on_request)
+        page.on("response", on_response)
+
+        print(f"正在访问: {LIST_URL}")
+        page.goto(LIST_URL, wait_until="networkidle", timeout=30000)
+        page.wait_for_timeout(3000)
+
+        print(f"\n共拦截到 {len(api_requests)} 个非静态请求:")
+        print("-" * 60)
+        for req in api_requests:
+            print(f"[{req['method']}] {req['url']}")
+            if req['post_data']:
+                print(f"  POST DATA: {req['post_data'][:200]}")
+
+        print(f"\n共拦截到 {len(json_responses)} 个 JSON 响应:")
+        print("-" * 60)
+        for resp in json_responses:
+            print(f"[{resp['status']}] {resp['url']}")
+            print(f"  CT: {resp['content_type']}")
+            print(f"  BODY: {resp['body_preview']}")
+            print()
+
+        # 获取渲染后的文章列表
+        print("\n渲染后页面的文章链接:")
+        print("-" * 60)
+        links = page.query_selector_all("a[href]")
+        articles = []
+        for link in links:
+            href = link.get_attribute("href") or ""
+            text = link.inner_text().strip()
+            if len(text) > 8 and ("/zwgk/" in href or re.search(r'\d{6}/t\d', href)):
+                articles.append((text, href))
+
+        for title, href in articles[:20]:
+            print(f"  [{title[:50]}] -> {href}")
+
+        browser.close()
+        return api_requests, json_responses
+
+if __name__ == "__main__":
+    find_list_api()

hubei_government_zwgk_links.csv

+地区,级别,官网首页,政务公开链接,状态,备注
+湖北省人民政府,省级,https://www.hubei.gov.cn/,https://www.hubei.gov.cn/xxgk/,当前环境受限,官网首页当前环境返回 412，xxgk 目录可从站内公开内容路径确认，建议浏览器直接访问核对
+武汉市人民政府,副省级市,https://www.wuhan.gov.cn/,https://www.wuhan.gov.cn/zwgk/?channelid=26164,可访问,参考你提供的武汉示例链接
+黄石市人民政府,地级市,https://www.huangshi.gov.cn/,https://www.huangshi.gov.cn/xxxgk/,可访问,首页政务公开入口
+十堰市人民政府,地级市,https://www.shiyan.gov.cn/,http://www.shiyan.gov.cn/xxgk/,可访问,首页政府信息公开入口
+宜昌市人民政府,地级市,http://www.yichang.gov.cn/,http://www.yichang.gov.cn/zfxxgk/,可访问,首页政府信息公开入口
+襄阳市人民政府,地级市,http://www.xiangyang.gov.cn/wzsy/,http://xxgk.xiangyang.gov.cn/,可访问,首页政府信息公开跳转到独立公开子站
+鄂州市人民政府,地级市,https://www.ezhou.gov.cn/sy/,https://www.ezhou.gov.cn/gk/,可访问,首页政府信息公开入口
+荆门市人民政府,地级市,https://www.jingmen.gov.cn/,https://www.jingmen.gov.cn/col/col16474/index.html,可访问,首页政府信息公开目录入口
+孝感市人民政府,地级市,https://www.xiaogan.gov.cn/,https://www.xiaogan.gov.cn/c/www/zc.jhtml,可访问,首页政府信息公开入口
+荆州市人民政府,地级市,https://www.jingzhou.gov.cn/,http://zwgk.jingzhou.gov.cn/list_children.shtml?column_id=54272,可访问,首页政府信息公开目录入口
+黄冈市人民政府,地级市,https://www.hg.gov.cn/,https://www.hg.gov.cn/zwgk/public/column/6636765?type=4&action=list&nav=0&id=7025468,可访问,首页政务公开入口
+咸宁市人民政府,地级市,http://www.xianning.gov.cn/,http://www.xianning.gov.cn/xxgk/,可访问,首页政府信息公开入口
+随州市人民政府,地级市,http://www.suizhou.gov.cn/,http://www.suizhou.gov.cn/zwgk/,可访问,首页政府信息公开入口
+恩施土家族苗族自治州人民政府,自治州,http://www.enshi.gov.cn/,http://www.enshi.gov.cn/zc/,可访问,首页政府信息公开入口
+仙桃市人民政府,省直辖县级市,https://www.xiantao.gov.cn/,https://www.xiantao.gov.cn/zfxxgk/,可访问,首页政府信息公开入口
+潜江市人民政府,省直辖县级市,https://www.hbqj.gov.cn/,https://www.hbqj.gov.cn/xxgk/,可访问,首页含政府信息公开模块，公开内容集中在 xxgk 目录
+天门市人民政府,省直辖县级市,http://www.tianmen.gov.cn/,http://www.tianmen.gov.cn/zwgk/,可访问,首页政府信息公开入口
+神农架林区人民政府,林区,http://www.snj.gov.cn/,http://www.snj.gov.cn/zwgk/,可访问,首页政府信息公开入口
\ No newline at end of file

hubei_government_zwgk_links.md

+# 湖北省政府官网政务公开入口清单
+
+说明：本清单按“省级门户 + 湖北省各市州、省直辖县级行政区政府门户”整理，重点记录官网首页及其“政务公开 / 政府信息公开”入口。  
+校验时间：2026-04-09  
+备注：湖北省人民政府门户在当前环境下直接请求返回 `412 Precondition Failed`，已单独标注；其余市州和省直辖县级行政区入口已逐个核到可访问页面。
+
+| 地区 | 级别 | 官网首页 | 政务公开链接 | 状态 | 备注 |
+| --- | --- | --- | --- | --- | --- |
+| 湖北省人民政府 | 省级 | https://www.hubei.gov.cn/ | https://www.hubei.gov.cn/xxgk/ | 当前环境受限 | 官网首页当前环境返回 412，`xxgk` 目录可从站内公开内容路径确认，建议浏览器直接访问核对 |
+| 武汉市人民政府 | 副省级市 | https://www.wuhan.gov.cn/ | https://www.wuhan.gov.cn/zwgk/?channelid=26164 | 可访问 | 参考你提供的武汉示例链接 |
+| 黄石市人民政府 | 地级市 | https://www.huangshi.gov.cn/ | https://www.huangshi.gov.cn/xxxgk/ | 可访问 | 首页“政务公开”入口 |
+| 十堰市人民政府 | 地级市 | https://www.shiyan.gov.cn/ | http://www.shiyan.gov.cn/xxgk/ | 可访问 | 首页“政府信息公开”入口 |
+| 宜昌市人民政府 | 地级市 | http://www.yichang.gov.cn/ | http://www.yichang.gov.cn/zfxxgk/ | 可访问 | 首页“政府信息公开”入口 |
+| 襄阳市人民政府 | 地级市 | http://www.xiangyang.gov.cn/wzsy/ | http://xxgk.xiangyang.gov.cn/ | 可访问 | 首页“政府信息公开”跳转到独立公开子站 |
+| 鄂州市人民政府 | 地级市 | https://www.ezhou.gov.cn/sy/ | https://www.ezhou.gov.cn/gk/ | 可访问 | 首页“政府信息公开”入口 |
+| 荆门市人民政府 | 地级市 | https://www.jingmen.gov.cn/ | https://www.jingmen.gov.cn/col/col16474/index.html | 可访问 | 首页“政府信息公开目录”入口 |
+| 孝感市人民政府 | 地级市 | https://www.xiaogan.gov.cn/ | https://www.xiaogan.gov.cn/c/www/zc.jhtml | 可访问 | 首页“政府信息公开”入口 |
+| 荆州市人民政府 | 地级市 | https://www.jingzhou.gov.cn/ | http://zwgk.jingzhou.gov.cn/list_children.shtml?column_id=54272 | 可访问 | 首页“政府信息公开目录”入口 |
+| 黄冈市人民政府 | 地级市 | https://www.hg.gov.cn/ | https://www.hg.gov.cn/zwgk/public/column/6636765?type=4&action=list&nav=0&id=7025468 | 可访问 | 首页“政务公开”入口 |
+| 咸宁市人民政府 | 地级市 | http://www.xianning.gov.cn/ | http://www.xianning.gov.cn/xxgk/ | 可访问 | 首页“政府信息公开”入口 |
+| 随州市人民政府 | 地级市 | http://www.suizhou.gov.cn/ | http://www.suizhou.gov.cn/zwgk/ | 可访问 | 首页“政府信息公开”入口 |
+| 恩施土家族苗族自治州人民政府 | 自治州 | http://www.enshi.gov.cn/ | http://www.enshi.gov.cn/zc/ | 可访问 | 首页“政府信息公开”入口 |
+| 仙桃市人民政府 | 省直辖县级市 | https://www.xiantao.gov.cn/ | https://www.xiantao.gov.cn/zfxxgk/ | 可访问 | 首页“政府信息公开”入口 |
+| 潜江市人民政府 | 省直辖县级市 | https://www.hbqj.gov.cn/ | https://www.hbqj.gov.cn/xxgk/ | 可访问 | 首页含“政府信息公开”模块，公开内容集中在 `xxgk` 目录 |
+| 天门市人民政府 | 省直辖县级市 | http://www.tianmen.gov.cn/ | http://www.tianmen.gov.cn/zwgk/ | 可访问 | 首页“政府信息公开”入口 |
+| 神农架林区人民政府 | 林区 | http://www.snj.gov.cn/ | http://www.snj.gov.cn/zwgk/ | 可访问 | 首页“政府信息公开”入口 |

hubei_hrss_zwgk_links.csv

+name,homepage,public_entry,notes
+湖北省人力资源和社会保障厅,https://rst.hubei.gov.cn/,https://rst.hubei.gov.cn/zfxxgk/zc/gfxwj/,省厅规范性文件
+武汉市人力资源和社会保障局,https://rsj.wuhan.gov.cn/,https://rsj.wuhan.gov.cn/zwgk_17/zc/gfxwj/zwgk_list.html,示例站点
+黄石市人力资源和社会保障局,https://rsj.huangshi.gov.cn/,https://rsj.huangshi.gov.cn/xxgk/zc/gfxwj/,含政策解读栏目
+十堰市人力资源和社会保障局,http://rsj.shiyan.gov.cn/,http://rsj.shiyan.gov.cn/srlzyhshbzj/zc/gfxwj_new/,原zc页面会跳到gfxwj_new
+宜昌市人力资源和社会保障局,http://rsj.yichang.gov.cn/,http://www.yichang.gov.cn/zfxxgk/list.html?depid=858&catid=522&t=4,统一公开平台并使用官方API
+襄阳市人力资源和社会保障局,http://rsj.xiangyang.gov.cn/,http://rsj.xiangyang.gov.cn/zwgk/zc/zcfg/,含政策解读栏目
+鄂州市人力资源和社会保障局,https://rsj.ezhou.gov.cn/,https://rsj.ezhou.gov.cn/xxgk/zc/bmgfxwj/,部门规范性文件
+荆门市人力资源和社会保障局,http://rsj.jingmen.gov.cn/,http://rsj.jingmen.gov.cn/col/col10845/index.html,另补政策解读与参保政策入口
+孝感市人力资源和社会保障局,https://rsj.xiaogan.gov.cn/,https://rsj.xiaogan.gov.cn/c/xgsrlzyhshbzj/zc.jhtml,http会自动跳转到https
+荆州市人力资源和社会保障局,http://rsj.jingzhou.gov.cn/,http://jzrsj.zwgk.jingzhou.gov.cn/list.shtml?column_id=35910,政策文件在独立公开域名
+黄冈市人力资源和社会保障局,https://rsj.hg.gov.cn/,https://rsj.hg.gov.cn/zwgk/public/column/6636189?type=4&catId=7026842&action=list&nav=0,公开平台列表示例
+咸宁市人力资源和社会保障局,http://rsj.xianning.gov.cn/,http://rsj.xianning.gov.cn/xxgk/zc/wjzl/,规范性文件资料库
+随州市人力资源和社会保障局,http://rsj.suizhou.gov.cn/,http://rsj.suizhou.gov.cn/fbjd_16/zwgk/xxgkml/zcfg/,另补通知公告入口
+恩施州人力资源和社会保障局,http://rsj.enshi.gov.cn/,http://rsj.enshi.gov.cn/xxgk/zc/gfxwj/,同时补充政策解读和其他主动公开
+仙桃市人力资源和社会保障局,https://www.xiantao.gov.cn/bmxxgk/srsj/,https://www.xiantao.gov.cn/bmxxgk/srsj/zfxxgk/zc/gfxwj/,由部门公开页逐级跳转进入
+潜江市人力资源和社会保障局,https://www.hbqj.gov.cn/srlzyhshbzj/,https://www.hbqj.gov.cn/srlzyhshbzj/zfxxgk/zc/gfxwj/,当前仅确认规范性文件入口稳定
+天门市人力资源和社会保障局,https://www.tianmen.gov.cn/zwgk/bmhxzxxgkml/bm/srlzyhshbzj/,https://www.tianmen.gov.cn/zwgk/bmhxzxxgkml/bm/srlzyhshbzj/zfxxgk/zc/bmwj/,同时补充政策解读入口
+神农架林区人力资源和社会保障局,http://rsj.snj.gov.cn/,http://rsj.snj.gov.cn/zc_38462/,林区入口

hubei_hrss_zwgk_links.md

+# 湖北省人社局政务公开政策入口
+
+当前项目默认采集的是湖北省内人力资源和社会保障部门官网的政策公开栏目，不再使用综合政府门户站。
+
+| 名称 | 官网 | 政策入口 | 备注 |
+| --- | --- | --- | --- |
+| 湖北省人力资源和社会保障厅 | https://rst.hubei.gov.cn/ | https://rst.hubei.gov.cn/zfxxgk/zc/gfxwj/ | 省厅规范性文件 |
+| 武汉市人力资源和社会保障局 | https://rsj.wuhan.gov.cn/ | https://rsj.wuhan.gov.cn/zwgk_17/zc/gfxwj/zwgk_list.html | 示例站点 |
+| 黄石市人力资源和社会保障局 | https://rsj.huangshi.gov.cn/ | https://rsj.huangshi.gov.cn/xxgk/zc/gfxwj/ | 含政策解读栏目 |
+| 十堰市人力资源和社会保障局 | http://rsj.shiyan.gov.cn/ | http://rsj.shiyan.gov.cn/srlzyhshbzj/zc/gfxwj_new/ | 原 `zc/` 页面会跳到 `gfxwj_new/` |
+| 宜昌市人力资源和社会保障局 | http://rsj.yichang.gov.cn/ | http://www.yichang.gov.cn/zfxxgk/list.html?depid=858&catid=522&t=4 | 统一公开平台，列表由官方 API 提供 |
+| 襄阳市人力资源和社会保障局 | http://rsj.xiangyang.gov.cn/ | http://rsj.xiangyang.gov.cn/zwgk/zc/zcfg/ | 含政策解读栏目 |
+| 鄂州市人力资源和社会保障局 | https://rsj.ezhou.gov.cn/ | https://rsj.ezhou.gov.cn/xxgk/zc/bmgfxwj/ | 部门规范性文件 |
+| 荆门市人力资源和社会保障局 | http://rsj.jingmen.gov.cn/ | http://rsj.jingmen.gov.cn/col/col10845/index.html | 另补了政策解读与参保政策入口 |
+| 孝感市人力资源和社会保障局 | https://rsj.xiaogan.gov.cn/ | https://rsj.xiaogan.gov.cn/c/xgsrlzyhshbzj/zc.jhtml | `http` 会自动跳转到 `https` |
+| 荆州市人力资源和社会保障局 | http://rsj.jingzhou.gov.cn/ | http://jzrsj.zwgk.jingzhou.gov.cn/list.shtml?column_id=35910 | 政策文件在独立公开域名 |
+| 黄冈市人力资源和社会保障局 | https://rsj.hg.gov.cn/ | https://rsj.hg.gov.cn/zwgk/public/column/6636189?type=4&catId=7026842&action=list&nav=0 | 公开平台列表示例 |
+| 咸宁市人力资源和社会保障局 | http://rsj.xianning.gov.cn/ | http://rsj.xianning.gov.cn/xxgk/zc/wjzl/ | 规范性文件资料库 |
+| 随州市人力资源和社会保障局 | http://rsj.suizhou.gov.cn/ | http://rsj.suizhou.gov.cn/fbjd_16/zwgk/xxgkml/zcfg/ | 另补通知公告入口 |
+| 恩施州人力资源和社会保障局 | http://rsj.enshi.gov.cn/ | http://rsj.enshi.gov.cn/xxgk/zc/gfxwj/ | 同时补充政策解读、其他主动公开 |
+| 仙桃市人力资源和社会保障局 | https://www.xiantao.gov.cn/bmxxgk/srsj/ | https://www.xiantao.gov.cn/bmxxgk/srsj/zfxxgk/zc/gfxwj/ | 由部门公开页逐级跳转进入 |
+| 潜江市人力资源和社会保障局 | https://www.hbqj.gov.cn/srlzyhshbzj/ | https://www.hbqj.gov.cn/srlzyhshbzj/zfxxgk/zc/gfxwj/ | 当前仅确认规范性文件入口稳定 |
+| 天门市人力资源和社会保障局 | https://www.tianmen.gov.cn/zwgk/bmhxzxxgkml/bm/srlzyhshbzj/ | https://www.tianmen.gov.cn/zwgk/bmhxzxxgkml/bm/srlzyhshbzj/zfxxgk/zc/bmwj/ | 同时补充政策解读入口 |
+| 神农架林区人力资源和社会保障局 | http://rsj.snj.gov.cn/ | http://rsj.snj.gov.cn/zc_38462/ | 林区入口 |

policy_agent/__init__.py

+"""Hubei policy collection agent."""
+
+from .pipeline import PipelineResult, PolicyPipeline
+
+__all__ = ["PipelineResult", "PolicyPipeline"]

policy_agent/cli.py

+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from .config import load_config, load_sites
+from .pipeline import PolicyPipeline
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Hubei policy collection agent")
+    parser.add_argument(
+        "--project-root",
+        type=Path,
+        default=Path(__file__).resolve().parent.parent,
+        help="Project root path",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    subparsers.add_parser("check-config", help="Validate local config")
+    subparsers.add_parser("list-sites", help="List configured sites")
+    preview_parser = subparsers.add_parser("preview", help="Preview crawl results without Dify sync")
+    preview_parser.add_argument("--site", required=True, help="Site name")
+    preview_parser.add_argument("--limit", type=int, default=10, help="Preview article count")
+    preview_parser.add_argument("--max-pages", type=int, help="Override max pages for this run")
+    preview_parser.add_argument("--max-depth", type=int, help="Override max depth for this run")
+
+    run_parser = subparsers.add_parser("run", help="Run crawler and Dify sync once")
+    run_parser.add_argument("--site", action="append", help="Optional site name filter")
+    run_parser.add_argument("--max-pages", type=int, help="Override max pages for this run")
+    run_parser.add_argument("--max-depth", type=int, help="Override max depth for this run")
+    return parser
+
+
+def main() -> None:
+    if hasattr(sys.stdout, "reconfigure"):
+        sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    if hasattr(sys.stderr, "reconfigure"):
+        sys.stderr.reconfigure(encoding="utf-8", errors="replace")
+    parser = build_parser()
+    args = parser.parse_args()
+    config = load_config(args.project_root)
+
+    if args.command == "check-config":
+        errors = config.validate()
+        if errors:
+            raise SystemExit("\n".join(errors))
+        print("配置检查通过")
+        return
+
+    if args.command == "list-sites":
+        for site in load_sites(config):
+            print(f"{site.name}\t{site.public_entry}")
+        return
+
+    if args.command == "preview":
+        from .crawler import PolicyCrawler
+
+        sites = [site for site in load_sites(config) if site.name == args.site]
+        if not sites:
+            raise SystemExit(f"未找到站点: {args.site}")
+        if args.max_pages is not None:
+            from dataclasses import replace
+
+            sites = [replace(sites[0], max_pages=args.max_pages)]
+        if args.max_depth is not None:
+            from dataclasses import replace
+
+            sites = [replace(sites[0], max_depth=args.max_depth)]
+        crawler = PolicyCrawler(config)
+        outcome = crawler.crawl_site(sites[0])
+        print(f"pages={outcome.fetched_pages}\tfailed={outcome.failed_pages}\tarticles={len(outcome.articles)}")
+        for item in outcome.articles[: args.limit]:
+            print(f"{item.publish_date}\t{item.category}\t{item.title}\t{item.source_url}")
+        return
+
+    if args.command == "run":
+        pipeline = PolicyPipeline(config)
+        try:
+            results = pipeline.run(site_names=args.site, max_pages=args.max_pages, max_depth=args.max_depth)
+        finally:
+            pipeline.close()
+        for result in results:
+            print(
+                f"{result.site_name}\tpages={result.fetched_pages}\tarticles={result.discovered_articles}"
+                f"\tinserted={result.inserted}\tupdated={result.updated}\tdeleted={result.deleted}"
+                f"\tduplicates={result.duplicates}"
+                f"\tfailed={result.failed_pages}"
+            )
+
+
+if __name__ == "__main__":
+    main()

policy_agent/config.py

+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+from .env import load_dotenv
+from .models import SiteConfig
+
+
+@dataclass(slots=True)
+class AppConfig:
+    project_root: Path
+    data_dir: Path
+    logs_dir: Path
+    db_path: Path
+    sites_path: Path
+    request_timeout: int
+    user_agent: str
+    dify_base_url: str
+    dify_api_key: str
+    dify_dataset_id: str
+    dify_dataset_name: str
+    dify_indexing_technique: str
+    dify_batch_size: int
+
+    def validate(self) -> list[str]:
+        errors: list[str] = []
+        if not self.sites_path.exists():
+            errors.append(f"站点配置不存在: {self.sites_path}")
+        if not self.dify_base_url:
+            errors.append("缺少 DIFY_BASE_URL")
+        if not self.dify_api_key:
+            errors.append("缺少 DIFY_API_KEY")
+        if not self.dify_dataset_id and not self.dify_dataset_name:
+            errors.append("至少配置 DIFY_DATASET_ID 或 DIFY_DATASET_NAME")
+        return errors
+
+
+def load_config(project_root: Path | None = None) -> AppConfig:
+    root = project_root or Path(__file__).resolve().parent.parent
+    load_dotenv(root / ".env")
+
+    data_dir = root / "data"
+    logs_dir = root / "logs"
+    data_dir.mkdir(exist_ok=True)
+    logs_dir.mkdir(exist_ok=True)
+
+    return AppConfig(
+        project_root=root,
+        data_dir=data_dir,
+        logs_dir=logs_dir,
+        db_path=Path(os.getenv("POLICY_DB_PATH", data_dir / "policy_agent.db")),
+        sites_path=Path(os.getenv("POLICY_SITES_PATH", root / "policy_agent" / "sources" / "hubei_hrss_sites.json")),
+        request_timeout=int(os.getenv("POLICY_REQUEST_TIMEOUT", "20")),
+        user_agent=os.getenv(
+            "POLICY_USER_AGENT",
+            (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+            ),
+        ),
+        dify_base_url=os.getenv("DIFY_BASE_URL", "").rstrip("/"),
+        dify_api_key=os.getenv("DIFY_API_KEY", "").strip(),
+        dify_dataset_id=os.getenv("DIFY_DATASET_ID", "").strip(),
+        dify_dataset_name=os.getenv("DIFY_DATASET_NAME", "湖北人社政策知识库").strip(),
+        dify_indexing_technique=os.getenv("DIFY_INDEXING_TECHNIQUE", "high_quality").strip(),
+        dify_batch_size=int(os.getenv("DIFY_BATCH_SIZE", "50")),
+    )
+
+
+def load_sites(config: AppConfig) -> list[SiteConfig]:
+    rows = json.loads(config.sites_path.read_text(encoding="utf-8"))
+    return [SiteConfig(**row) for row in rows]

policy_agent/dify_client.py

+from __future__ import annotations
+
+from typing import Any
+
+import requests
+
+from .config import AppConfig
+
+
+class DifyClient:
+    def __init__(self, config: AppConfig) -> None:
+        self.config = config
+        self.session = requests.Session()
+        self.session.headers.update(
+            {
+                "Authorization": f"Bearer {config.dify_api_key}",
+                "Content-Type": "application/json",
+            }
+        )
+
+    def _request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]:
+        response = self.session.request(
+            method,
+            f"{self.config.dify_base_url}{path}",
+            timeout=self.config.request_timeout,
+            **kwargs,
+        )
+        response.raise_for_status()
+        return response.json() if response.text.strip() else {}
+
+    def list_datasets(self) -> list[dict[str, Any]]:
+        page = 1
+        datasets: list[dict[str, Any]] = []
+        while True:
+            payload = self._request("GET", f"/v1/datasets?page={page}&limit=100")
+            items = payload.get("data", [])
+            datasets.extend(items)
+            if len(items) < 100:
+                return datasets
+            page += 1
+
+    def create_dataset(self, name: str) -> str:
+        payload = self._request(
+            "POST",
+            "/v1/datasets",
+            json={
+                "name": name,
+                "indexing_technique": self.config.dify_indexing_technique,
+                "permission": "only_me",
+            },
+        )
+        return str(payload["id"])
+
+    def ensure_dataset(self) -> str:
+        if self.config.dify_dataset_id:
+            return self.config.dify_dataset_id
+
+        for dataset in self.list_datasets():
+            if dataset.get("name") == self.config.dify_dataset_name:
+                return str(dataset["id"])
+
+        dataset_id = self.create_dataset(self.config.dify_dataset_name)
+        self.config.dify_dataset_id = dataset_id
+        return dataset_id
+
+    def create_document_by_text(self, dataset_id: str, name: str, text: str) -> str:
+        payload = self._request(
+            "POST",
+            f"/v1/datasets/{dataset_id}/document/create-by-text",
+            json={
+                "name": name,
+                "text": text,
+                "indexing_technique": self.config.dify_indexing_technique,
+                "process_rule": {"mode": "automatic"},
+            },
+        )
+        document = payload.get("document") or payload
+        return str(document["id"])
+
+    def update_document_by_text(self, dataset_id: str, document_id: str, name: str, text: str) -> None:
+        self._request(
+            "POST",
+            f"/v1/datasets/{dataset_id}/documents/{document_id}/update_by_text",
+            json={
+                "name": name,
+                "text": text,
+                "process_rule": {"mode": "automatic"},
+            },
+        )
+
+    def delete_document(self, dataset_id: str, document_id: str) -> None:
+        self._request(
+            "DELETE",
+            f"/v1/datasets/{dataset_id}/documents/{document_id}",
+        )

policy_agent/env.py

+from __future__ import annotations
+
+from pathlib import Path
+
+
+def load_dotenv(dotenv_path: Path) -> None:
+    """Load a very small subset of .env files into process env."""
+    import os
+
+    if not dotenv_path.exists():
+        return
+
+    for raw_line in dotenv_path.read_text(encoding="utf-8").splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        key = key.strip()
+        value = value.strip().strip('"').strip("'")
+        os.environ.setdefault(key, value)

policy_agent/models.py

+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(slots=True)
+class SiteConfig:
+    name: str
+    region: str
+    level: str
+    homepage: str
+    public_entry: str
+    allowed_hosts: list[str]
+    seed_urls: list[str] = field(default_factory=list)
+    include_url_keywords: list[str] = field(default_factory=list)
+    exclude_url_keywords: list[str] = field(default_factory=list)
+    max_depth: int = 2
+    max_pages: int = 80
+    use_playwright: bool = False
+    crawl_mode: str = "default"
+    api_deptid: str = ""
+    api_list_url: str = ""
+
+
+@dataclass(slots=True)
+class PolicyRecord:
+    site_name: str
+    region: str
+    level: str
+    source_site: str
+    source_url: str
+    title: str
+    content: str
+    publish_date: str
+    crawl_date: str
+    category: str
+    doc_no: str
+    effective_date: str
+    status: str
+    policy_key: str
+    version_no: int
+    content_hash: str
+    summary_text: str
+    attachments_json: str
+
+
+@dataclass(slots=True)
+class PipelineResult:
+    site_name: str
+    fetched_pages: int = 0
+    discovered_articles: int = 0
+    inserted: int = 0
+    updated: int = 0
+    deleted: int = 0
+    duplicates: int = 0
+    failed_pages: int = 0

policy_agent/pipeline.py

+from __future__ import annotations
+
+import logging
+import sys
+from dataclasses import replace
+from pathlib import Path
+
+import requests
+
+from .config import AppConfig, load_config, load_sites
+from .crawler import PolicyCrawler
+from .dify_client import DifyClient
+from .models import PipelineResult, SiteConfig
+from .storage import PolicyStore
+
+
+def _build_logger(log_path: Path) -> logging.Logger:
+    logger = logging.getLogger("policy_agent")
+    logger.setLevel(logging.INFO)
+    logger.handlers.clear()
+    formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
+
+    file_handler = logging.FileHandler(log_path, encoding="utf-8")
+    file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setFormatter(formatter)
+    logger.addHandler(stream_handler)
+    return logger
+
+
+class PolicyPipeline:
+    def __init__(self, config: AppConfig | None = None) -> None:
+        self.config = config or load_config()
+        self.store = PolicyStore(self.config.db_path)
+        self.crawler = PolicyCrawler(self.config)
+        self.dify = DifyClient(self.config)
+        self.logger = _build_logger(self.config.logs_dir / "policy_agent.log")
+
+    def close(self) -> None:
+        self.store.close()
+
+    def run(
+        self,
+        site_names: list[str] | None = None,
+        max_pages: int | None = None,
+        max_depth: int | None = None,
+    ) -> list[PipelineResult]:
+        errors = self.config.validate()
+        if errors:
+            raise ValueError("配置不完整: " + "；".join(errors))
+
+        dataset_id = self.dify.ensure_dataset()
+        self.logger.info("Using Dify dataset: %s", dataset_id)
+
+        sites = load_sites(self.config)
+        if site_names:
+            wanted = {name.strip() for name in site_names}
+            sites = [site for site in sites if site.name in wanted]
+        if max_pages is not None:
+            sites = [replace(site, max_pages=max_pages) for site in sites]
+        if max_depth is not None:
+            sites = [replace(site, max_depth=max_depth) for site in sites]
+
+        results: list[PipelineResult] = []
+        for site in sites:
+            results.append(self._run_site(site, dataset_id))
+        return results
+
+    def _run_site(self, site: SiteConfig, dataset_id: str) -> PipelineResult:
+        run_id = self.store.begin_run(site.name)
+        self.logger.info("Start crawling: %s", site.name)
+        result = PipelineResult(site_name=site.name)
+        notes = ""
+        try:
+            outcome = self.crawler.crawl_site(site)
+            result.fetched_pages = outcome.fetched_pages
+            result.failed_pages = outcome.failed_pages
+            result.discovered_articles = len(outcome.articles)
+
+            for article in outcome.articles:
+                action, row = self.store.upsert_policy(article)
+                if action == "duplicate" and row["dify_document_id"]:
+                    result.duplicates += 1
+                elif row["dify_document_id"]:
+                    try:
+                        self.dify.update_document_by_text(
+                            dataset_id=dataset_id,
+                            document_id=row["dify_document_id"],
+                            name=article.title[:180],
+                            text=article.summary_text,
+                        )
+                        result.updated += 1
+                    except requests.HTTPError as exc:
+                        response_text = exc.response.text if exc.response is not None else ""
+                        status_code = exc.response.status_code if exc.response is not None else 0
+                        if status_code == 400 and "Document is not available" in response_text:
+                            document_id = self.dify.create_document_by_text(
+                                dataset_id=dataset_id,
+                                name=article.title[:180],
+                                text=article.summary_text,
+                            )
+                            self.store.update_dify_document_id(str(row["source_url"]), document_id)
+                            result.updated += 1
+                        else:
+                            raise
+                else:
+                    document_id = self.dify.create_document_by_text(
+                        dataset_id=dataset_id,
+                        name=article.title[:180],
+                        text=article.summary_text,
+                    )
+                    self.store.update_dify_document_id(str(row["source_url"]), document_id)
+                    if action == "updated":
+                        result.updated += 1
+                    elif action == "duplicate":
+                        result.duplicates += 1
+                    else:
+                        result.inserted += 1
+
+                result.deleted += self._delete_revised_documents(
+                    dataset_id=dataset_id,
+                    policy_key=str(row["policy_key"]),
+                    current_source_url=str(row["source_url"]),
+                )
+
+            notes = f"articles={result.discovered_articles};deleted={result.deleted}"
+            self.logger.info(
+                "Finished %s: pages=%s articles=%s inserted=%s updated=%s deleted=%s duplicates=%s failed=%s",
+                site.name,
+                result.fetched_pages,
+                result.discovered_articles,
+                result.inserted,
+                result.updated,
+                result.deleted,
+                result.duplicates,
+                result.failed_pages,
+            )
+            return result
+        except Exception as exc:
+            notes = f"error={exc}"
+            self.logger.exception("Pipeline failed for %s", site.name)
+            raise
+        finally:
+            self.store.finish_run(run_id, result, notes=notes)
+
+    def _delete_revised_documents(self, dataset_id: str, policy_key: str, current_source_url: str) -> int:
+        deleted = 0
+        revised_rows = self.store.get_outdated_policies(policy_key, current_source_url)
+        for revised_row in revised_rows:
+            document_id = revised_row["dify_document_id"]
+            if not document_id:
+                continue
+
+            try:
+                self.dify.delete_document(dataset_id=dataset_id, document_id=str(document_id))
+            except requests.HTTPError as exc:
+                response_text = exc.response.text if exc.response is not None else ""
+                status_code = exc.response.status_code if exc.response is not None else 0
+                if status_code == 404:
+                    pass
+                elif status_code == 400 and "Document is not available" in response_text:
+                    pass
+                else:
+                    raise
+
+            self.store.clear_dify_document_id(int(revised_row["id"]))
+            deleted += 1
+        return deleted

policy_agent/scripts/register_midnight_task.ps1

+param(
+    [string]$TaskName = "HubeiPolicyDailyUpdate",
+    [string]$ProjectRoot = ""
+)
+
+$ErrorActionPreference = "Stop"
+
+if ([string]::IsNullOrWhiteSpace($ProjectRoot)) {
+    $ProjectRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path
+} else {
+    $ProjectRoot = (Resolve-Path $ProjectRoot).Path
+}
+
+$RunScript = Join-Path $ProjectRoot "policy_agent\scripts\run_daily_update.ps1"
+$PowerShellExe = "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe"
+
+if (-not (Test-Path $RunScript)) {
+    throw "Run script not found: $RunScript"
+}
+
+$Action = New-ScheduledTaskAction `
+    -Execute $PowerShellExe `
+    -Argument "-NoProfile -ExecutionPolicy Bypass -File `"$RunScript`" -ProjectRoot `"$ProjectRoot`""
+
+$Trigger = New-ScheduledTaskTrigger -Daily -At 00:00
+$Settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew
+
+Register-ScheduledTask `
+    -TaskName $TaskName `
+    -Action $Action `
+    -Trigger $Trigger `
+    -Settings $Settings `
+    -Description "Daily Hubei policy sync to Dify knowledge base" `
+    -Force | Out-Null
+
+Write-Host "Scheduled task registered: $TaskName"
+Write-Host "Project root: $ProjectRoot"

policy_agent/scripts/run_daily_update.ps1

+param(
+    [string]$ProjectRoot = "",
+    [string]$PythonExe = "",
+    [int]$MaxPages = 0,
+    [int]$MaxDepth = 0,
+    [string[]]$Site = @()
+)
+
+$ErrorActionPreference = "Stop"
+
+if ([string]::IsNullOrWhiteSpace($ProjectRoot)) {
+    $ProjectRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path
+} else {
+    $ProjectRoot = (Resolve-Path $ProjectRoot).Path
+}
+
+function Resolve-PythonExe {
+    param(
+        [string]$PreferredPythonExe,
+        [string]$ResolvedProjectRoot
+    )
+
+    $candidates = @()
+    if (-not [string]::IsNullOrWhiteSpace($PreferredPythonExe)) {
+        $candidates += $PreferredPythonExe
+    }
+    if (-not [string]::IsNullOrWhiteSpace($env:POLICY_PYTHON)) {
+        $candidates += $env:POLICY_PYTHON
+    }
+
+    $venvPython = Join-Path $ResolvedProjectRoot ".venv\Scripts\python.exe"
+    if (Test-Path $venvPython) {
+        $candidates += $venvPython
+    }
+
+    foreach ($candidate in $candidates) {
+        if (Test-Path $candidate) {
+            return (Resolve-Path $candidate).Path
+        }
+    }
+
+    $pythonCommand = Get-Command python -ErrorAction SilentlyContinue
+    if ($pythonCommand) {
+        return $pythonCommand.Source
+    }
+
+    throw "Python executable not found. Use -PythonExe, set POLICY_PYTHON, or run setup_local.ps1 first."
+}
+
+$PythonExe = Resolve-PythonExe -PreferredPythonExe $PythonExe -ResolvedProjectRoot $ProjectRoot
+$EnvFile = Join-Path $ProjectRoot ".env"
+
+if (-not (Test-Path $EnvFile)) {
+    throw ".env not found: $EnvFile. Copy .env.example to .env and fill in Dify configuration first."
+}
+
+$arguments = @(
+    "-m", "policy_agent.cli",
+    "--project-root", $ProjectRoot,
+    "run"
+)
+
+if ($MaxPages -gt 0) {
+    $arguments += @("--max-pages", $MaxPages)
+}
+if ($MaxDepth -gt 0) {
+    $arguments += @("--max-depth", $MaxDepth)
+}
+foreach ($siteName in $Site) {
+    if (-not [string]::IsNullOrWhiteSpace($siteName)) {
+        $arguments += @("--site", $siteName)
+    }
+}
+
+Set-Location $ProjectRoot
+& $PythonExe @arguments
+exit $LASTEXITCODE

policy_agent/scripts/setup_local.ps1

+param(
+    [string]$ProjectRoot = "",
+    [string]$PythonExe = "",
+    [switch]$SkipPlaywrightInstall
+)
+
+$ErrorActionPreference = "Stop"
+
+if ([string]::IsNullOrWhiteSpace($ProjectRoot)) {
+    $ProjectRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path
+} else {
+    $ProjectRoot = (Resolve-Path $ProjectRoot).Path
+}
+
+function Resolve-BootstrapPython {
+    param([string]$PreferredPythonExe)
+
+    if (-not [string]::IsNullOrWhiteSpace($PreferredPythonExe)) {
+        if (-not (Test-Path $PreferredPythonExe)) {
+            throw "Python executable not found: $PreferredPythonExe"
+        }
+        return (Resolve-Path $PreferredPythonExe).Path
+    }
+
+    $pythonCommand = Get-Command python -ErrorAction SilentlyContinue
+    if ($pythonCommand) {
+        return $pythonCommand.Source
+    }
+
+    $pyLauncher = Get-Command py -ErrorAction SilentlyContinue
+    if ($pyLauncher) {
+        return $pyLauncher.Source
+    }
+
+    throw "Python executable not found. Install Python 3.10+ first or pass -PythonExe."
+}
+
+$BootstrapPython = Resolve-BootstrapPython -PreferredPythonExe $PythonExe
+$VenvDir = Join-Path $ProjectRoot ".venv"
+$VenvPython = Join-Path $VenvDir "Scripts\python.exe"
+$RequirementsPath = Join-Path $ProjectRoot "requirements.txt"
+$EnvExamplePath = Join-Path $ProjectRoot ".env.example"
+$EnvPath = Join-Path $ProjectRoot ".env"
+
+if (-not (Test-Path $VenvPython)) {
+    Write-Host "Creating local virtual environment: $VenvDir"
+    if ($BootstrapPython -like "*\\py.exe") {
+        & $BootstrapPython -3 -m venv $VenvDir
+    } else {
+        & $BootstrapPython -m venv $VenvDir
+    }
+}
+
+Write-Host "Installing Python dependencies..."
+& $VenvPython -m pip install --upgrade pip
+& $VenvPython -m pip install -r $RequirementsPath
+
+if (-not $SkipPlaywrightInstall) {
+    Write-Host "Installing Playwright Chromium browser..."
+    & $VenvPython -m playwright install chromium
+}
+
+if ((Test-Path $EnvExamplePath) -and -not (Test-Path $EnvPath)) {
+    Copy-Item $EnvExamplePath $EnvPath
+    Write-Host "Created .env from .env.example. Fill in DIFY_BASE_URL and DIFY_API_KEY before running."
+}
+
+Write-Host ""
+Write-Host "Setup complete."
+Write-Host "Virtualenv Python: $VenvPython"
+Write-Host "Next step:"
+Write-Host "  1. Edit $EnvPath"
+Write-Host "  2. Run: $VenvPython -m policy_agent.cli --project-root `"$ProjectRoot`" check-config"

policy_agent/sources/hubei_hrss_sites.json

+[
+  {
+    "name": "湖北省人力资源和社会保障厅",
+    "region": "湖北省",
+    "level": "省级",
+    "homepage": "https://rst.hubei.gov.cn/",
+    "public_entry": "https://rst.hubei.gov.cn/zfxxgk/zc/gfxwj/",
+    "allowed_hosts": ["rst.hubei.gov.cn"],
+    "seed_urls": [
+      "https://rst.hubei.gov.cn/zfxxgk/zc/gfxwj/",
+      "https://rst.hubei.gov.cn/zfxxgk/zc/zcjd/"
+    ],
+    "include_url_keywords": [
+      "/zfxxgk/zc/gfxwj/",
+      "/zfxxgk/zc/zcjd/"
+    ],
+    "exclude_url_keywords": [
+      "/bmdt/",
+      "/dtyw/",
+      "/tzgg/",
+      "index.shtml?name="
+    ],
+    "max_depth": 2,
+    "max_pages": 80,
+    "crawl_mode": "render_entry"
+  },
+  {
+    "name": "武汉市人力资源和社会保障局",
+    "region": "武汉市",
+    "level": "副省级市",
+    "homepage": "https://rsj.wuhan.gov.cn/",
+    "public_entry": "https://rsj.wuhan.gov.cn/zwgk_17/zc/gfxwj/zwgk_list.html",
+    "allowed_hosts": ["rsj.wuhan.gov.cn"],
+    "seed_urls": [
+      "https://rsj.wuhan.gov.cn/zwgk_17/zc/gfxwj/zwgk_list.html",
+      "https://rsj.wuhan.gov.cn/zwgk_17/zc/zcjd_1/zwgk_list.html"
+    ],
+    "include_url_keywords": [
+      "/zwgk_17/zc/gfxwj/",
+      "/zwgk_17/zc/zcjd_1/"
+    ],
+    "exclude_url_keywords": [
+      "/qtzdgkwj/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80,
+    "crawl_mode": "wuhan_jsonp",
+    "api_list_url": "https://rsj.wuhan.gov.cn/cslm/index.jsonp"
+  },
+  {
+    "name": "黄石市人力资源和社会保障局",
+    "region": "黄石市",
+    "level": "地级市",
+    "homepage": "https://rsj.huangshi.gov.cn/",
+    "public_entry": "https://rsj.huangshi.gov.cn/xxgk/zc/gfxwj/",
+    "allowed_hosts": ["rsj.huangshi.gov.cn"],
+    "seed_urls": [
+      "https://rsj.huangshi.gov.cn/xxgk/zc/gfxwj/",
+      "https://rsj.huangshi.gov.cn/xxgk/zc/zcjd/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "十堰市人力资源和社会保障局",
+    "region": "十堰市",
+    "level": "地级市",
+    "homepage": "http://rsj.shiyan.gov.cn/",
+    "public_entry": "http://rsj.shiyan.gov.cn/srlzyhshbzj/zc/gfxwj_new/",
+    "allowed_hosts": ["rsj.shiyan.gov.cn"],
+    "seed_urls": [
+      "http://rsj.shiyan.gov.cn/srlzyhshbzj/zc/gfxwj_new/",
+      "http://rsj.shiyan.gov.cn/srlzyhshbzj/zc/zcjd/"
+    ],
+    "include_url_keywords": [
+      "/srlzyhshbzj/zc/gfxwj_new/",
+      "/srlzyhshbzj/zc/zcjd/"
+    ],
+    "exclude_url_keywords": [
+      "/qtzdgkwj/",
+      "/gzdt/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "宜昌市人力资源和社会保障局",
+    "region": "宜昌市",
+    "level": "地级市",
+    "homepage": "http://rsj.yichang.gov.cn/",
+    "public_entry": "http://www.yichang.gov.cn/zfxxgk/list.html?depid=858&catid=522&t=4",
+    "allowed_hosts": ["www.yichang.gov.cn", "rsj.yichang.gov.cn"],
+    "seed_urls": [
+      "http://www.yichang.gov.cn/zfxxgk/list.html?depid=858&catid=522&t=4"
+    ],
+    "max_depth": 2,
+    "max_pages": 80,
+    "crawl_mode": "yichang_api",
+    "api_deptid": "858"
+  },
+  {
+    "name": "襄阳市人力资源和社会保障局",
+    "region": "襄阳市",
+    "level": "地级市",
+    "homepage": "http://rsj.xiangyang.gov.cn/",
+    "public_entry": "http://rsj.xiangyang.gov.cn/zwgk/zc/zcfg/",
+    "allowed_hosts": ["rsj.xiangyang.gov.cn"],
+    "seed_urls": [
+      "http://rsj.xiangyang.gov.cn/zwgk/zc/zcfg/",
+      "http://rsj.xiangyang.gov.cn/zwgk/zc/zcjd/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "鄂州市人力资源和社会保障局",
+    "region": "鄂州市",
+    "level": "地级市",
+    "homepage": "https://rsj.ezhou.gov.cn/",
+    "public_entry": "https://rsj.ezhou.gov.cn/xxgk/zc/bmgfxwj/",
+    "allowed_hosts": ["rsj.ezhou.gov.cn"],
+    "seed_urls": [
+      "https://rsj.ezhou.gov.cn/xxgk/zc/bmgfxwj/",
+      "https://rsj.ezhou.gov.cn/xxgk/zc/zcjd/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "荆门市人力资源和社会保障局",
+    "region": "荆门市",
+    "level": "地级市",
+    "homepage": "http://rsj.jingmen.gov.cn/",
+    "public_entry": "http://rsj.jingmen.gov.cn/col/col10845/index.html",
+    "allowed_hosts": ["rsj.jingmen.gov.cn"],
+    "seed_urls": [
+      "http://rsj.jingmen.gov.cn/col/col10845/index.html",
+      "http://rsj.jingmen.gov.cn/col/col10846/index.html",
+      "http://rsj.jingmen.gov.cn/col/col10881/index.html"
+    ],
+    "include_url_keywords": [
+      "/col/col10845/",
+      "/col/col10846/",
+      "/col/col10881/",
+      "/art_10845_",
+      "/art_10846_",
+      "/art_10881_"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "孝感市人力资源和社会保障局",
+    "region": "孝感市",
+    "level": "地级市",
+    "homepage": "https://rsj.xiaogan.gov.cn/",
+    "public_entry": "https://rsj.xiaogan.gov.cn/c/xgsrlzyhshbzj/zc.jhtml",
+    "allowed_hosts": ["rsj.xiaogan.gov.cn"],
+    "seed_urls": [
+      "https://rsj.xiaogan.gov.cn/c/xgsrlzyhshbzj/zc.jhtml"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "荆州市人力资源和社会保障局",
+    "region": "荆州市",
+    "level": "地级市",
+    "homepage": "http://rsj.jingzhou.gov.cn/",
+    "public_entry": "http://jzrsj.zwgk.jingzhou.gov.cn/list.shtml?column_id=35910",
+    "allowed_hosts": ["rsj.jingzhou.gov.cn", "jzrsj.zwgk.jingzhou.gov.cn"],
+    "seed_urls": [
+      "http://jzrsj.zwgk.jingzhou.gov.cn/list.shtml?column_id=35910",
+      "http://jzrsj.zwgk.jingzhou.gov.cn/list.shtml?column_id=35912"
+    ],
+    "include_url_keywords": [
+      "column_id=35910",
+      "column_id=35912",
+      "/35910/",
+      "/35912/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80,
+    "crawl_mode": "render_entry"
+  },
+  {
+    "name": "黄冈市人力资源和社会保障局",
+    "region": "黄冈市",
+    "level": "地级市",
+    "homepage": "https://rsj.hg.gov.cn/",
+    "public_entry": "https://rsj.hg.gov.cn/zwgk/public/column/6636189?type=4&catId=7026842&action=list&nav=0",
+    "allowed_hosts": ["rsj.hg.gov.cn"],
+    "seed_urls": [
+      "https://rsj.hg.gov.cn/zwgk/public/column/6636189?type=4&catId=7026842&action=list&nav=0",
+      "https://rsj.hg.gov.cn/zwgk/public/column/6636189?type=4&catId=7025468&action=list&nav=0"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "咸宁市人力资源和社会保障局",
+    "region": "咸宁市",
+    "level": "地级市",
+    "homepage": "http://rsj.xianning.gov.cn/",
+    "public_entry": "http://rsj.xianning.gov.cn/xxgk/zc/wjzl/",
+    "allowed_hosts": ["rsj.xianning.gov.cn"],
+    "seed_urls": [
+      "http://rsj.xianning.gov.cn/xxgk/zc/wjzl/",
+      "http://rsj.xianning.gov.cn/xxgk/zc/zcjd/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "随州市人力资源和社会保障局",
+    "region": "随州市",
+    "level": "地级市",
+    "homepage": "http://rsj.suizhou.gov.cn/",
+    "public_entry": "http://rsj.suizhou.gov.cn/fbjd_16/zwgk/xxgkml/zcfg/",
+    "allowed_hosts": ["rsj.suizhou.gov.cn"],
+    "seed_urls": [
+      "http://rsj.suizhou.gov.cn/fbjd_16/zwgk/xxgkml/zcfg/",
+      "http://rsj.suizhou.gov.cn/fbjd_16/zwgk/zc/qtzdgkwj/tzgg/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "恩施州人力资源和社会保障局",
+    "region": "恩施土家族苗族自治州",
+    "level": "自治州",
+    "homepage": "http://rsj.enshi.gov.cn/",
+    "public_entry": "http://rsj.enshi.gov.cn/xxgk/zc/gfxwj/",
+    "allowed_hosts": ["rsj.enshi.gov.cn"],
+    "seed_urls": [
+      "http://rsj.enshi.gov.cn/xxgk/zc/gfxwj/",
+      "http://rsj.enshi.gov.cn/xxgk/zc/zcjd/",
+      "http://rsj.enshi.gov.cn/xxgk/zc/qtzdgk/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "仙桃市人力资源和社会保障局",
+    "region": "仙桃市",
+    "level": "省直辖县级市",
+    "homepage": "https://www.xiantao.gov.cn/bmxxgk/srsj/",
+    "public_entry": "https://www.xiantao.gov.cn/bmxxgk/srsj/zfxxgk/zc/gfxwj/",
+    "allowed_hosts": ["www.xiantao.gov.cn"],
+    "seed_urls": [
+      "https://www.xiantao.gov.cn/bmxxgk/srsj/zfxxgk/zc/gfxwj/",
+      "https://www.xiantao.gov.cn/bmxxgk/srsj/zfxxgk/zc/zcjd/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "潜江市人力资源和社会保障局",
+    "region": "潜江市",
+    "level": "省直辖县级市",
+    "homepage": "https://www.hbqj.gov.cn/srlzyhshbzj/",
+    "public_entry": "https://www.hbqj.gov.cn/srlzyhshbzj/zfxxgk/zc/gfxwj/",
+    "allowed_hosts": ["www.hbqj.gov.cn"],
+    "seed_urls": [
+      "https://www.hbqj.gov.cn/srlzyhshbzj/zfxxgk/zc/gfxwj/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "天门市人力资源和社会保障局",
+    "region": "天门市",
+    "level": "省直辖县级市",
+    "homepage": "https://www.tianmen.gov.cn/zwgk/bmhxzxxgkml/bm/srlzyhshbzj/",
+    "public_entry": "https://www.tianmen.gov.cn/zwgk/bmhxzxxgkml/bm/srlzyhshbzj/zfxxgk/zc/bmwj/",
+    "allowed_hosts": ["www.tianmen.gov.cn"],
+    "seed_urls": [
+      "https://www.tianmen.gov.cn/zwgk/bmhxzxxgkml/bm/srlzyhshbzj/zfxxgk/zc/bmwj/",
+      "https://www.tianmen.gov.cn/zwgk/bmhxzxxgkml/bm/srlzyhshbzj/zfxxgk/zc/zcjd/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "神农架林区人力资源和社会保障局",
+    "region": "神农架林区",
+    "level": "林区",
+    "homepage": "http://rsj.snj.gov.cn/",
+    "public_entry": "http://rsj.snj.gov.cn/zc_38462/",
+    "allowed_hosts": ["rsj.snj.gov.cn", "www.snj.gov.cn"],
+    "seed_urls": [
+      "http://rsj.snj.gov.cn/zc_38462/"
+    ],
+    "include_url_keywords": [
+      "/zc_38462/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80,
+    "crawl_mode": "render_entry"
+  }
+]

policy_agent/sources/hubei_sites.json

+[
+  {
+    "name": "湖北省人民政府",
+    "region": "湖北省",
+    "level": "省级",
+    "homepage": "https://www.hubei.gov.cn/",
+    "public_entry": "https://www.hubei.gov.cn/xxgk/",
+    "allowed_hosts": ["www.hubei.gov.cn"],
+    "seed_urls": ["https://www.hubei.gov.cn/xxgk/zc/", "https://www.hubei.gov.cn/xxgk/zcjd/"],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "武汉市人民政府",
+    "region": "武汉市",
+    "level": "副省级市",
+    "homepage": "https://www.wuhan.gov.cn/",
+    "public_entry": "https://www.wuhan.gov.cn/zwgk/?channelid=26164",
+    "allowed_hosts": ["www.wuhan.gov.cn"],
+    "seed_urls": [
+      "https://www.wuhan.gov.cn/zwgk/tzgg/",
+      "https://www.wuhan.gov.cn/hdjl/dczj/",
+      "https://www.wuhan.gov.cn/gfxwj/index.shtml"
+    ],
+    "max_depth": 2,
+    "max_pages": 80,
+    "use_playwright": false
+  },
+  {
+    "name": "黄石市人民政府",
+    "region": "黄石市",
+    "level": "地级市",
+    "homepage": "https://www.huangshi.gov.cn/",
+    "public_entry": "https://www.huangshi.gov.cn/xxxgk/",
+    "allowed_hosts": ["www.huangshi.gov.cn"],
+    "seed_urls": [
+      "https://www.huangshi.gov.cn/xxxgk/2020_gfxwj/",
+      "https://www.huangshi.gov.cn/xxxgk/qtzdgkwj/",
+      "https://www.huangshi.gov.cn/xxxgk/zcjd/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80,
+    "use_playwright": false
+  },
+  {
+    "name": "十堰市人民政府",
+    "region": "十堰市",
+    "level": "地级市",
+    "homepage": "https://www.shiyan.gov.cn/",
+    "public_entry": "http://www.shiyan.gov.cn/xxgk/",
+    "allowed_hosts": ["www.shiyan.gov.cn"],
+    "seed_urls": [
+      "http://www.shiyan.gov.cn/xxgk/zc_67263/xxgk_tzgg/",
+      "http://www.shiyan.gov.cn/zcjd/",
+      "http://www.shiyan.gov.cn/xxgk/xxgk_fdgk/qtzdgknr/data/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80,
+    "use_playwright": true
+  },
+  {
+    "name": "宜昌市人民政府",
+    "region": "宜昌市",
+    "level": "地级市",
+    "homepage": "http://www.yichang.gov.cn/",
+    "public_entry": "http://www.yichang.gov.cn/zfxxgk/",
+    "allowed_hosts": ["www.yichang.gov.cn"],
+    "seed_urls": [],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "襄阳市人民政府",
+    "region": "襄阳市",
+    "level": "地级市",
+    "homepage": "http://www.xiangyang.gov.cn/wzsy/",
+    "public_entry": "http://xxgk.xiangyang.gov.cn/",
+    "allowed_hosts": ["www.xiangyang.gov.cn", "xxgk.xiangyang.gov.cn"],
+    "seed_urls": [],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "鄂州市人民政府",
+    "region": "鄂州市",
+    "level": "地级市",
+    "homepage": "https://www.ezhou.gov.cn/sy/",
+    "public_entry": "https://www.ezhou.gov.cn/gk/",
+    "allowed_hosts": ["www.ezhou.gov.cn"],
+    "seed_urls": [
+      "https://www.ezhou.gov.cn/gk/gz1/",
+      "https://www.ezhou.gov.cn/gk/gfxwj/",
+      "https://www.ezhou.gov.cn/gk/zcjd/",
+      "https://www.ezhou.gov.cn/gk/gsgg_1/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80,
+    "use_playwright": true
+  },
+  {
+    "name": "荆门市人民政府",
+    "region": "荆门市",
+    "level": "地级市",
+    "homepage": "https://www.jingmen.gov.cn/",
+    "public_entry": "https://www.jingmen.gov.cn/col/col16474/index.html",
+    "allowed_hosts": ["www.jingmen.gov.cn"],
+    "seed_urls": [],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "孝感市人民政府",
+    "region": "孝感市",
+    "level": "地级市",
+    "homepage": "https://www.xiaogan.gov.cn/",
+    "public_entry": "https://www.xiaogan.gov.cn/c/www/zc.jhtml",
+    "allowed_hosts": ["www.xiaogan.gov.cn"],
+    "seed_urls": [
+      "https://www.xiaogan.gov.cn/c/www/gfxwj.jhtml",
+      "https://www.xiaogan.gov.cn/c/www/zcjd.jhtml",
+      "https://www.xiaogan.gov.cn/c/www/gsgg.jhtml"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "荆州市人民政府",
+    "region": "荆州市",
+    "level": "地级市",
+    "homepage": "https://www.jingzhou.gov.cn/",
+    "public_entry": "http://zwgk.jingzhou.gov.cn/list_children.shtml?column_id=54272",
+    "allowed_hosts": ["www.jingzhou.gov.cn", "zwgk.jingzhou.gov.cn"],
+    "seed_urls": [],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "黄冈市人民政府",
+    "region": "黄冈市",
+    "level": "地级市",
+    "homepage": "https://www.hg.gov.cn/",
+    "public_entry": "https://www.hg.gov.cn/zwgk/public/column/6636765?type=4&action=list&nav=0&id=7025468",
+    "allowed_hosts": ["www.hg.gov.cn"],
+    "seed_urls": [],
+    "max_depth": 2,
+    "max_pages": 80,
+    "use_playwright": false
+  },
+  {
+    "name": "咸宁市人民政府",
+    "region": "咸宁市",
+    "level": "地级市",
+    "homepage": "http://www.xianning.gov.cn/",
+    "public_entry": "http://www.xianning.gov.cn/xxgk/",
+    "allowed_hosts": ["www.xianning.gov.cn", "xtzyk.xianning.gov.cn"],
+    "seed_urls": [
+      "http://www.xianning.gov.cn/xxgk/zc/zfwj/xzbf/",
+      "http://www.xianning.gov.cn/xxgk/zc/zfwj/xzbh/",
+      "http://www.xianning.gov.cn/xxgk/zc/zcjd/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "随州市人民政府",
+    "region": "随州市",
+    "level": "地级市",
+    "homepage": "http://www.suizhou.gov.cn/",
+    "public_entry": "http://www.suizhou.gov.cn/zwgk/",
+    "allowed_hosts": ["www.suizhou.gov.cn"],
+    "seed_urls": [
+      "http://www.suizhou.gov.cn/zwgk/zfwj/szbf/szf/",
+      "http://www.suizhou.gov.cn/zwgk/zfwj/szbf/szbf/",
+      "http://www.suizhou.gov.cn/zwgk/zfwj/qt_5744/tzgg/",
+      "http://www.suizhou.gov.cn/zwgk/zfwj/zcjd/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "恩施土家族苗族自治州人民政府",
+    "region": "恩施州",
+    "level": "自治州",
+    "homepage": "http://www.enshi.gov.cn/",
+    "public_entry": "http://www.enshi.gov.cn/zc/",
+    "allowed_hosts": ["www.enshi.gov.cn"],
+    "seed_urls": [
+      "http://www.enshi.gov.cn/zc/zc/gfxwj_1/",
+      "http://www.enshi.gov.cn/zc/zc/zcwj/",
+      "http://www.enshi.gov.cn/zc/zc/zcjd/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80,
+    "use_playwright": true
+  },
+  {
+    "name": "仙桃市人民政府",
+    "region": "仙桃市",
+    "level": "省直辖县级市",
+    "homepage": "https://www.xiantao.gov.cn/",
+    "public_entry": "https://www.xiantao.gov.cn/zfxxgk/",
+    "allowed_hosts": ["www.xiantao.gov.cn"],
+    "seed_urls": [
+      "https://www.xiantao.gov.cn/zfxxgk/zfwjk/gfwj/",
+      "https://www.xiantao.gov.cn/zfxxgk/zfwjk/qtwj/",
+      "https://www.xiantao.gov.cn/zfxxgk/zfwjk/zcjd/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80,
+    "use_playwright": true
+  },
+  {
+    "name": "潜江市人民政府",
+    "region": "潜江市",
+    "level": "省直辖县级市",
+    "homepage": "https://www.hbqj.gov.cn/",
+    "public_entry": "https://www.hbqj.gov.cn/xxgk/",
+    "allowed_hosts": ["www.hbqj.gov.cn"],
+    "seed_urls": [
+      "https://www.hbqj.gov.cn/xxgk/zc/gfxwj/",
+      "https://www.hbqj.gov.cn/xxgk/zc/qtwj/",
+      "https://www.hbqj.gov.cn/xxgk/zc/zcjzcjd/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80,
+    "use_playwright": true
+  },
+  {
+    "name": "天门市人民政府",
+    "region": "天门市",
+    "level": "省直辖县级市",
+    "homepage": "http://www.tianmen.gov.cn/",
+    "public_entry": "http://www.tianmen.gov.cn/zwgk/",
+    "allowed_hosts": ["www.tianmen.gov.cn"],
+    "seed_urls": [
+      "http://www.tianmen.gov.cn/zwgk/zc/zcfg/",
+      "http://www.tianmen.gov.cn/zwgk/zc/zcjd/",
+      "http://www.tianmen.gov.cn/zwgk/zc/tzgg/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80
+  },
+  {
+    "name": "神农架林区人民政府",
+    "region": "神农架林区",
+    "level": "林区",
+    "homepage": "http://www.snj.gov.cn/",
+    "public_entry": "http://www.snj.gov.cn/zwgk/",
+    "allowed_hosts": ["www.snj.gov.cn"],
+    "seed_urls": [
+      "http://www.snj.gov.cn/zwgk/zc/gfxwj/",
+      "http://www.snj.gov.cn/zwgk/zc/zfbwj/",
+      "http://www.snj.gov.cn/zwgk/zc/zcjd/"
+    ],
+    "max_depth": 2,
+    "max_pages": 80,
+    "use_playwright": true
+  }
+]

requirements.txt

+requests>=2.31,<3
+beautifulsoup4>=4.12,<5
+lxml>=5,<6
+playwright>=1.48,<2

test_wuhan_scraper.py

+# -*- coding: utf-8 -*-
+"""
+test_wuhan_scraper.py
+武汉政府网爬取可行性测试 - 完整版
+运行方式: conda activate policy && python test_wuhan_scraper.py
+"""
+import sys, io
+sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
+
+import requests
+from bs4 import BeautifulSoup
+import re, json
+
+session = requests.Session()
+session.verify = False
+session.headers.update({
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Accept-Language": "zh-CN,zh;q=0.9",
+    "Referer": "https://www.wuhan.gov.cn/",
+})
+
+import urllib3
+urllib3.disable_warnings()
+
+BASE = "https://www.wuhan.gov.cn"
+CHANNEL_ID = "26164"
+LIST_URL = f"{BASE}/zwgk/?channelid={CHANNEL_ID}"
+
+# ──────────────────────────────────────────────────────────
+# PART 1: 探测列表页的真实 API 接口
+# ──────────────────────────────────────────────────────────
+print("=" * 65)
+print("PART 1: 探测列表 API 接口")
+print("=" * 65)
+
+# 先访问列表页获取 Cookie
+session.get(LIST_URL, timeout=10)
+
+IGS_APIS = [
+    # IGS CMS 标准接口
+    f"{BASE}/igs/front/siteContent/getSiteContentList.jhtml",
+    f"{BASE}/igs/front/article/queryArticleList.jhtml",
+    f"{BASE}/igs/front/article/getArticleList.jhtml",
+    f"{BASE}/igs/front/content/getContentList.jhtml",
+    # 武汉网站特有前缀
+    f"{BASE}/zwgk/api/articleList",
+    f"{BASE}/zwgk/list.json",
+    f"{BASE}/ssi/front/queryArticleList.jhtml",
+]
+
+PARAM_SETS = [
+    {"channelId": CHANNEL_ID, "pageIndex": "1", "pageSize": "15"},
+    {"channelid": CHANNEL_ID, "pageIndex": "1", "pageSize": "15"},
+    {"channelId": CHANNEL_ID, "page": "1", "limit": "15"},
+]
+
+found_api = None
+for url in IGS_APIS:
+    for params in PARAM_SETS:
+        try:
+            r = session.post(url, data=params, timeout=8)
+            ct = r.headers.get("Content-Type", "")
+            if r.status_code == 200 and len(r.content) > 200:
+                body = r.text[:400]
+                is_json = "json" in ct or body.strip().startswith(("{", "["))
+                if is_json:
+                    print(f"★ 发现接口！POST {url}")
+                    print(f"  参数: {params}")
+                    print(f"  响应: {body}")
+                    found_api = (url, "POST", params)
+                    break
+        except Exception as e:
+            pass
+        try:
+            r = session.get(url, params=params, timeout=8)
+            ct = r.headers.get("Content-Type", "")
+            if r.status_code == 200 and len(r.content) > 200:
+                body = r.text[:400]
+                is_json = "json" in ct or body.strip().startswith(("{", "["))
+                if is_json:
+                    print(f"★ 发现接口！GET {url}")
+                    print(f"  参数: {params}")
+                    print(f"  响应: {body}")
+                    found_api = (url, "GET", params)
+                    break
+        except Exception as e:
+            pass
+    if found_api:
+        break
+
+if not found_api:
+    print("未发现 JSON API 接口，需要用 Playwright 渲染")
+
+
+# ──────────────────────────────────────────────────────────
+# PART 2: 分析文章详情页 HTML 结构（无需 API）
+# ──────────────────────────────────────────────────────────
+print("\n" + "=" * 65)
+print("PART 2: 抓取文章详情页")
+print("=" * 65)
+
+ARTICLE_URL = f"{BASE}/zwgk/xxgk/zfgz_new/202602/t20260211_2728640.shtml"
+print(f"测试 URL: {ARTICLE_URL}")
+
+r = session.get(ARTICLE_URL, timeout=15)
+r.encoding = "utf-8"
+soup = BeautifulSoup(r.text, "html.parser")
+
+# 提取字段
+title_el = soup.select_one("h3.doctitle")
+title = title_el.get_text(strip=True) if title_el else "未找到"
+
+# 日期：在 URL 里提取或从页面
+date_match = re.search(r'/(\d{4})(\d{2})/t', ARTICLE_URL)
+date = f"{date_match.group(1)}-{date_match.group(2)}" if date_match else "未知"
+
+# 文号
+docrn_el = soup.select_one("div.docrn")
+doc_num = docrn_el.get_text(strip=True) if docrn_el else "无文号"
+
+# 正文
+article_head = soup.select_one("div.articleHead")
+content = article_head.get_text(separator="\n", strip=True) if article_head else ""
+
+print(f"标题: {title}")
+print(f"日期: {date}")
+print(f"文号: {doc_num}")
+print(f"正文前200字: {content[:200]}")
+print(f"正文总长: {len(content)} 字")
+
+# 附件
+attachments = []
+for a in soup.find_all("a", href=True):
+    href = a["href"]
+    if any(href.endswith(ext) for ext in [".pdf", ".doc", ".docx", ".xls", ".xlsx"]):
+        attachments.append((a.get_text(strip=True), href))
+print(f"附件: {attachments if attachments else '无'}")
+
+
+# ──────────────────────────────────────────────────────────
+# PART 3: 尝试 Playwright（需要已安装 Chromium）
+# ──────────────────────────────────────────────────────────
+print("\n" + "=" * 65)
+print("PART 3: Playwright 拦截列表页 API")
+print("=" * 65)
+
+try:
+    from playwright.sync_api import sync_playwright
+
+    api_calls = []
+    json_responses = []
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        ctx = browser.new_context(locale="zh-CN")
+        page = ctx.new_page()
+
+        def on_response(response):
+            url = response.url
+            ct = response.headers.get("content-type", "")
+            if "json" in ct or "jhtml" in url:
+                try:
+                    body = response.text()
+                    json_responses.append({"url": url, "ct": ct, "body": body[:600]})
+                except:
+                    pass
+            # 记录所有非静态请求
+            if not re.search(r'\.(png|jpg|gif|css|woff|woff2|ico|svg)(\?|$)', url):
+                api_calls.append(f"[{response.status}] {response.request.method} {url}")
+
+        page.on("response", on_response)
+        page.goto(LIST_URL, wait_until="networkidle", timeout=30000)
+        page.wait_for_timeout(3000)
+
+        print(f"拦截到 {len(api_calls)} 个请求")
+        for c in api_calls:
+            print(f"  {c}")
+
+        print(f"\n其中 JSON 响应 {len(json_responses)} 个:")
+        for jr in json_responses:
+            print(f"  {jr['url']}")
+            print(f"  {jr['body'][:200]}")
+
+        # 渲染后的文章链接
+        links = page.eval_on_selector_all("a[href]", "els => els.map(e => ({text: e.innerText.trim(), href: e.href}))")
+        articles = [l for l in links if len(l['text']) > 8 and ('/zwgk/' in l['href'] or re.search(r'\d{6}/t\d', l['href']))]
+        print(f"\n渲染后文章链接 {len(articles)} 篇:")
+        for a in articles[:15]:
+            print(f"  [{a['text'][:50]}] -> {a['href']}")
+
+        browser.close()
+
+except ImportError:
+    print("Playwright 未安装，跳过")
+except Exception as e:
+    print(f"Playwright 错误: {e}")
+    print("请先运行: playwright install chromium")