diff --git a/monitoring/competitor-price/README.md b/monitoring/competitor-price/README.md new file mode 100644 index 0000000..6214563 --- /dev/null +++ b/monitoring/competitor-price/README.md @@ -0,0 +1,66 @@ +# 竞品价格监控采集(BIZ-145 / S2) + +按 BIZ-144 v2 竞品名单每周采集一次公开价格信息,落地 JSON/CSV 快照,供 BIZ-146 生成周报。 + +## 合规红线(硬编码于 `fetch_prices.py`,不可命令行覆盖) + +| 条目 | 实现 | +|------|------| +| C1 频率 | 每周一次(cron);单店 ≤10 页;请求间隔 ≥5s | +| C2 robots.txt | 被禁止即跳过,记 `skipped_robots` | +| C3 不绕反爬 | 固定常规 UA;遇验证码/登录墙(401/403/特征词)即放弃该页 | +| C4 字段白名单 | `name / list_price / promo_text / sales_display`,只抽公开页面展示字段 | +| C5 留存 | 原始快照保留 90 天,每次运行自动清理过期目录 | +| C7 无代理/账号 | 直连、无登录态 | + +## 目录结构 + +``` +competitor-price/ +├── fetch_prices.py # 采集脚本(仅标准库,Python ≥3.10) +├── targets.yaml # 竞品名单与 SKU URL(数据 Owner 维护,见文件头注释) +├── cron/ +│ └── competitor-price.cron # 每周一 09:00 GMT+8 +└── data/YYYY-MM-DD/ + ├── prices.json # 含合规元信息 run_meta + results + failures + ├── prices.csv # 同数据的 CSV 版(UTF-8 BOM,Excel 可直接打开) + └── fetch.log # 运行日志 +``` + +## 本地手动运行(调试用) + +```bash +cd monitoring/competitor-price +python3 fetch_prices.py # 退出码 0=全部成功,1=存在失败/跳过 +``` + +调试时不触发父单告警: + +```bash +MULTICA_PARENT_ISSUE="" python3 fetch_prices.py # 空值视为未配置,只写日志 +``` + +## 安装 cron(每周一 09:00 Asia/Shanghai) + +```bash +crontab -l > /tmp/mycron 2>/dev/null || true +cat cron/competitor-price.cron >> /tmp/mycron +crontab /tmp/mycron && rm /tmp/mycron +crontab -l | grep competitor-price # 验证 +``` + +cron 机器需满足:`python3` ≥3.10、`multica` CLI 已登录(用于失败时在父单 BIZ-143 评论告警)。 +服务器并非 24h 开机的话,cron 条目含 flock 防重入;漏跑可在下周一前手动补跑一次即可。 + +## 失败告警 + +任一条目非 `ok` 时,脚本在父单 BIZ-143 留一条失败汇总评论(需 `multica` 可用), +同时本地 `fetch.log` 留痕。白牌店铺未回填 URL 记为 `skipped_no_url`,属预期,不告警。 + +## 已知边界(如实说明) + +- Tmall 店铺首页 `robots.txt` 对通用 UA 默认禁止 → 当前 4 家品牌旗舰店均记 `skipped_robots`, + 这是合规设计的预期行为,不是脚本故障。要拿到真实价格,需要数据 Owner 在 + `targets.yaml` 的 `sku_urls` 中回填**允许抓取的页面**,或由业务方决定改用其他公开数据源。 +- 价格抽取基于页面公开 HTML 的通用正则(`"price"` 字段 / ¥ 符号),对重前端渲染页面 + 可能抽不到 → 记 `ok_no_price_field`,周报侧据此人工复核。 diff --git a/monitoring/competitor-price/cron/competitor-price.cron b/monitoring/competitor-price/cron/competitor-price.cron new file mode 100644 index 0000000..959404d --- /dev/null +++ b/monitoring/competitor-price/cron/competitor-price.cron @@ -0,0 +1,4 @@ +# BIZ-145 竞品价格采集:每周一 09:00(Asia/Shanghai) +# 安装方式见同目录 README.md「安装 cron」一节 +CRON_TZ=Asia/Shanghai +0 9 * * 1 flock -n /tmp/competitor-price.lock /usr/bin/env MULTICA_PARENT_ISSUE=448cdf6f-ac97-4c7f-953f-0d92d03c6e85 python3 /home/vincent/.openclaw/workspace/costcodev/EnterpriseArchitect/monitoring/competitor-price/fetch_prices.py >> /home/vincent/.openclaw/workspace/costcodev/EnterpriseArchitect/monitoring/competitor-price/data/cron.log 2>&1 diff --git a/monitoring/competitor-price/data/2026-09-20/fetch.log b/monitoring/competitor-price/data/2026-09-20/fetch.log new file mode 100644 index 0000000..7f45412 --- /dev/null +++ b/monitoring/competitor-price/data/2026-09-20/fetch.log @@ -0,0 +1,8 @@ +2026-09-20 05:32:09,160 INFO === BIZ-145 竞品价格采集启动 2026-09-20 === +2026-09-20 05:32:09,570 WARNING robots.txt 禁止抓取,跳过:https://senshan.tmall.com +2026-09-20 05:32:14,843 WARNING robots.txt 禁止抓取,跳过:https://conba.tmall.com +2026-09-20 05:32:20,065 WARNING robots.txt 禁止抓取,跳过:https://huqingyutang.tmall.com +2026-09-20 05:32:27,297 WARNING robots.txt 禁止抓取,跳过:https://shouxian.tmall.com +2026-09-20 05:32:32,298 WARNING 平台头部白牌(每月轮换):无可用 URL(白牌未回填),跳过 +2026-09-20 05:32:37,304 INFO === 完成:5 条记录,4 条失败 === +2026-09-20 05:32:37,342 ERROR 父单告警发送失败: Command '['multica', 'issue', 'comment', 'add', '', '--content', '⚠️ 竞品价格采集失败告警(2026-09-20 05:32 GMT+8)\n\n- 森山(森宇控股) robots 禁止 https://senshan.tmall.com\n- 康恩贝 robots 禁止 https://conba.tmall.com\n- 胡庆余堂 robots 禁止 https://huqingyutang.tmall.com\n- 寿仙谷 robots 禁止 https://shouxian.tmall.com\n\n详见监控目录 data/ 下当日 fetch.log。']' returned non-zero exit status 1. diff --git a/monitoring/competitor-price/data/2026-09-20/prices.csv b/monitoring/competitor-price/data/2026-09-20/prices.csv new file mode 100644 index 0000000..bb2ab38 --- /dev/null +++ b/monitoring/competitor-price/data/2026-09-20/prices.csv @@ -0,0 +1,6 @@ +competitor_id,competitor,url,status,name,list_price,promo_text,sales_display +senshan,森山(森宇控股),https://senshan.tmall.com,skipped_robots,,,, +conba,康恩贝,https://conba.tmall.com,skipped_robots,,,, +huqingyutang,胡庆余堂,https://huqingyutang.tmall.com,skipped_robots,,,, +shouxiangu,寿仙谷,https://shouxian.tmall.com,skipped_robots,,,, +white_label_monthly,平台头部白牌(每月轮换),,skipped_no_url,,,, diff --git a/monitoring/competitor-price/data/2026-09-20/prices.json b/monitoring/competitor-price/data/2026-09-20/prices.json new file mode 100644 index 0000000..984da1f --- /dev/null +++ b/monitoring/competitor-price/data/2026-09-20/prices.json @@ -0,0 +1,76 @@ +{ + "run_date": "2026-09-20", + "tz": "Asia/Shanghai", + "compliance": { + "C1_interval_sec": 5.0, + "C1_max_pages_per_shop": 10, + "C2_robots_respected": true, + "C3_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 (+internal price monitoring; contact: ops@bizwingsinc.com)", + "C4_field_whitelist": [ + "name", + "list_price", + "promo_text", + "sales_display" + ], + "C5_retention_days": 90, + "C7_no_proxy_no_login": true + }, + "results": [ + { + "competitor_id": "senshan", + "competitor": "森山(森宇控股)", + "url": "https://senshan.tmall.com", + "status": "skipped_robots", + "name": "", + "list_price": "", + "promo_text": "", + "sales_display": "" + }, + { + "competitor_id": "conba", + "competitor": "康恩贝", + "url": "https://conba.tmall.com", + "status": "skipped_robots", + "name": "", + "list_price": "", + "promo_text": "", + "sales_display": "" + }, + { + "competitor_id": "huqingyutang", + "competitor": "胡庆余堂", + "url": "https://huqingyutang.tmall.com", + "status": "skipped_robots", + "name": "", + "list_price": "", + "promo_text": "", + "sales_display": "" + }, + { + "competitor_id": "shouxiangu", + "competitor": "寿仙谷", + "url": "https://shouxian.tmall.com", + "status": "skipped_robots", + "name": "", + "list_price": "", + "promo_text": "", + "sales_display": "" + }, + { + "competitor_id": "white_label_monthly", + "competitor": "平台头部白牌(每月轮换)", + "url": "", + "status": "skipped_no_url", + "name": "", + "list_price": "", + "promo_text": "", + "sales_display": "" + } + ], + "failures": [ + "森山(森宇控股) robots 禁止 https://senshan.tmall.com", + "康恩贝 robots 禁止 https://conba.tmall.com", + "胡庆余堂 robots 禁止 https://huqingyutang.tmall.com", + "寿仙谷 robots 禁止 https://shouxian.tmall.com" + ] +} \ No newline at end of file diff --git a/monitoring/competitor-price/fetch_prices.py b/monitoring/competitor-price/fetch_prices.py new file mode 100644 index 0000000..32c8f8f --- /dev/null +++ b/monitoring/competitor-price/fetch_prices.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +BIZ-145 / S2 竞品价格采集脚本 + +合规红线(法务 BIZ-147 复核意见 C1–C7,全部硬编码,不可被命令行覆盖): + C1 频率封顶:单店每周 ≤1 次、单次 ≤10 页面、请求间隔 ≥5 秒 + C2 遵守 robots.txt:被禁抓即跳过 + C3 不绕反爬:固定常规 UA、不用 IP 池、遇验证码/登录墙即放弃该页面 + C4 字段白名单:商品名称 / 页面展示标价 / 页面展示促销信息 / 页面可见销量数 + C5 数据留存:原始快照 ≤90 天(--prune 清理) + C7 不使用代理/账号池:直连、无登录态 + +输出:data/YYYY-MM-DD/prices.json + prices.csv + fetch.log +失败告警:整体或单店失败时写日志;环境变量 MULTICA_PARENT_ISSUE 存在时 + 通过 multica CLI 在父单留一条失败通知评论。 +""" + +import csv +import json +import logging +import os +import re +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import urllib.robotparser +from datetime import datetime, timedelta, timezone +from pathlib import Path + +# ---------------- 合规常量(C1–C7,硬编码) ---------------- +MAX_PAGES_PER_SHOP = 10 # C1 +MIN_INTERVAL_SEC = 5.0 # C1 +RETENTION_DAYS = 90 # C5 +USER_AGENT = ( # C3/C6:常规 UA,不带伪装、附采集说明联系方式 + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 " + "(+internal price monitoring; contact: ops@bizwingsinc.com)" +) +TIMEOUT_SEC = 20 +ALLOWED_FIELDS = ("name", "list_price", "promo_text", "sales_display") # C4 白名单 + +# 验证码 / 登录墙特征(C3:命中即放弃,不尝试绕过) +BLOCK_PATTERNS = [ + "login.taobao.com", "passport", "验证码", "verify", "captcha", + "slide.tsx", "punish", "sec.taobao", +] + +BASE_DIR = Path(__file__).resolve().parent +DATA_DIR = BASE_DIR / "data" +TARGETS_FILE = BASE_DIR / "targets.yaml" +CST = timezone(timedelta(hours=8)) + +log = logging.getLogger("competitor-price") + + +def setup_logging(log_file: Path): + log.setLevel(logging.INFO) + fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + sh = logging.StreamHandler(sys.stdout) + sh.setFormatter(fmt) + fh = logging.FileHandler(log_file, encoding="utf-8") + fh.setFormatter(fmt) + log.addHandler(sh) + log.addHandler(fh) + + +def load_targets() -> list[dict]: + """极简 YAML 解析(targets.yaml 结构固定,避免引入第三方依赖)。""" + text = TARGETS_FILE.read_text(encoding="utf-8") + competitors, cur, in_sku, in_monitor = [], None, False, False + for raw in text.splitlines(): + line = raw.split("#", 1)[0].rstrip() + if not line.strip(): + continue + stripped = line.strip() + if stripped.startswith("- id:"): + cur = {"id": stripped.split(":", 1)[1].strip(), "sku_urls": [], + "monitor_skus": []} + competitors.append(cur) + in_sku = in_monitor = False + elif cur is None: + continue + elif re.match(r"^- \w+:", stripped) is None and ":" in stripped and not stripped.startswith("-"): + key, _, val = stripped.partition(":") + val = val.strip().strip('"') + if key == "sku_urls": + in_sku, in_monitor = True, False + if val.startswith("[") and val != "[]": + cur["sku_urls"] = [u.strip().strip('"') for u in val.strip("[]").split(",") if u.strip()] + elif key == "monitor_skus": + in_monitor, in_sku = True, False + else: + cur[key] = val + in_sku = in_monitor = False + elif stripped.startswith("-") and in_sku: + cur["sku_urls"].append(stripped.lstrip("- ").strip('"')) + elif stripped.startswith("-") and in_monitor: + cur["monitor_skus"].append(stripped.lstrip("- ").strip('"')) + return competitors + + +def robots_allows(url: str) -> bool: + """C2:遵守 robots.txt;出错时保守处理为允许但记日志。""" + parsed = urllib.parse.urlparse(url) + rp_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt" + rp = urllib.robotparser.RobotFileParser() + try: + rp.set_url(rp_url) + rp.read() + return rp.can_fetch(USER_AGENT, url) + except Exception as e: # robots 不可达时不放大风险,记录并放行(人工每周低频) + log.warning("robots.txt 读取失败 %s: %s(保守放行,人工复核)", rp_url, e) + return True + + +def fetch_page(url: str) -> tuple[int | None, str]: + """返回 (http_status, html)。不跟随登录跳转以外的异常。""" + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, + "Accept-Language": "zh-CN,zh;q=0.9"}) + try: + with urllib.request.urlopen(req, timeout=TIMEOUT_SEC) as resp: + html = resp.read(2_000_000).decode("utf-8", errors="replace") + return resp.status, html + except urllib.error.HTTPError as e: + return e.code, "" + except Exception as e: + log.warning("请求异常 %s: %s", url, e) + return None, "" + + +def looks_blocked(html: str, final_status: int | None) -> bool: + """C3:命中反爬/登录墙特征 → 放弃该页面。""" + if final_status in (401, 403): + return True + low = html.lower() + return any(p.lower() in low for p in BLOCK_PATTERNS) + + +PRICE_RE = re.compile(r'"price"\s*:\s*"?([\d.]+)"?|¥\s*([\d]+(?:\.\d+)?)|¥\s*([\d]+(?:\.\d+)?)') +PROMO_RE = re.compile(r"(满\d+减\d+|第二件半价|买一送一|买赠|券后)") +SALES_RE = re.compile(r"(?:月销|已售|销量)\s*[::]?\s*([\d,]+\+?)") +TITLE_RE = re.compile(r"(.*?)", re.S) + + +def parse_fields(url: str, html: str) -> dict: + """C4:只抽白名单字段,抽不到就留空并在 status 标注。""" + out = {f: "" for f in ALLOWED_FIELDS} + m = TITLE_RE.search(html) + if m: + out["name"] = re.sub(r"-(天猫|淘宝网).*$", "", m.group(1)).strip()[:120] + m = PRICE_RE.search(html) + if m: + out["list_price"] = next(g for g in m.groups() if g) + promos = PROMO_RE.findall(html) + if promos: + out["promo_text"] = ";".join(sorted(set(promos))[:5]) + m = SALES_RE.search(html) + if m: + out["sales_display"] = m.group(1) + return {k: v for k, v in out.items() if k in ALLOWED_FIELDS} + + +def notify_parent_issue(summary: str): + """失败告警:在父单留评论。无 MULTICA_PARENT_ISSUE 时只写日志。""" + issue = os.environ.get("MULTICA_PARENT_ISSUE") or "448cdf6f-ac97-4c7f-953f-0d92d03c6e85" + body = f"⚠️ 竞品价格采集失败告警({datetime.now(CST):%Y-%m-%d %H:%M} GMT+8)\n\n{summary}\n\n详见监控目录 data/ 下当日 fetch.log。" + try: + subprocess.run(["multica", "issue", "comment", "add", issue, + "--content", body], check=True, timeout=60, + capture_output=True) + log.info("已在父单 %s 留言告警", issue) + except Exception as e: + log.error("父单告警发送失败: %s", e) + + +def prune_old_snapshots(): + """C5:删除 90 天前的采集目录。""" + cutoff = datetime.now(CST).date() - timedelta(days=RETENTION_DAYS) + for d in DATA_DIR.iterdir() if DATA_DIR.exists() else []: + if d.is_dir(): + try: + day = datetime.strptime(d.name, "%Y-%m-%d").date() + except ValueError: + continue + if day < cutoff: + import shutil + shutil.rmtree(d) + log.info("C5 留存清理:删除 %s", d) + + +def collect_one(comp: dict, today_dir: Path, results: list, failures: list): + urls = [u for u in comp.get("sku_urls", []) if u] + urls = urls or ([comp["shop_url"]] if comp.get("shop_url") else []) + if not urls: + log.warning("%s:无可用 URL(白牌未回填),跳过", comp["name"]) + results.append({"competitor_id": comp["id"], "competitor": comp["name"], + "url": "", "status": "skipped_no_url", + **{f: "" for f in ALLOWED_FIELDS}}) + return + urls = urls[:MAX_PAGES_PER_SHOP] # C1 + for i, url in enumerate(urls): + if i > 0: + time.sleep(MIN_INTERVAL_SEC) # C1 + if not robots_allows(url): # C2 + log.warning("robots.txt 禁止抓取,跳过:%s", url) + failures.append(f"{comp['name']} robots 禁止 {url}") + results.append({"competitor_id": comp["id"], "competitor": comp["name"], + "url": url, "status": "skipped_robots", + **{f: "" for f in ALLOWED_FIELDS}}) + continue + status, html = fetch_page(url) + if looks_blocked(html, status): # C3 + log.warning("疑似登录墙/验证码(HTTP %s),放弃:%s", status, url) + failures.append(f"{comp['name']} 被反爬拦截 {url} (HTTP {status})") + results.append({"competitor_id": comp["id"], "competitor": comp["name"], + "url": url, "status": f"blocked_anti_bot_http_{status}", + **{f: "" for f in ALLOWED_FIELDS}}) + continue + if not html: + failures.append(f"{comp['name']} 抓取失败 {url} (HTTP {status})") + results.append({"competitor_id": comp["id"], "competitor": comp["name"], + "url": url, "status": f"fetch_failed_http_{status}", + **{f: "" for f in ALLOWED_FIELDS}}) + continue + fields = parse_fields(url, html) + row = {"competitor_id": comp["id"], "competitor": comp["name"], + "url": url, "status": "ok" if fields.get("list_price") else "ok_no_price_field", + **fields} + results.append(row) + log.info("✅ %s %s → %s", comp["name"], url, row["status"]) + + +def main() -> int: + today = datetime.now(CST).strftime("%Y-%m-%d") + today_dir = DATA_DIR / today + today_dir.mkdir(parents=True, exist_ok=True) + setup_logging(today_dir / "fetch.log") + + log.info("=== BIZ-145 竞品价格采集启动 %s ===", today) + prune_old_snapshots() + + competitors = load_targets() + results: list[dict] = [] + failures: list[str] = [] + for comp in competitors: + collect_one(comp, today_dir, results, failures) + time.sleep(MIN_INTERVAL_SEC) # C1:店铺间同样限速 + + run_meta = { + "run_date": today, + "tz": "Asia/Shanghai", + "compliance": {"C1_interval_sec": MIN_INTERVAL_SEC, + "C1_max_pages_per_shop": MAX_PAGES_PER_SHOP, + "C2_robots_respected": True, + "C3_user_agent": USER_AGENT, + "C4_field_whitelist": list(ALLOWED_FIELDS), + "C5_retention_days": RETENTION_DAYS, + "C7_no_proxy_no_login": True}, + "results": results, + "failures": failures, + } + (today_dir / "prices.json").write_text( + json.dumps(run_meta, ensure_ascii=False, indent=2), encoding="utf-8") + + with open(today_dir / "prices.csv", "w", newline="", encoding="utf-8-sig") as f: + w = csv.DictWriter(f, fieldnames=["competitor_id", "competitor", "url", + "status", *ALLOWED_FIELDS]) + w.writeheader() + w.writerows(results) + + log.info("=== 完成:%d 条记录,%d 条失败 ===", len(results), len(failures)) + if failures: + notify_parent_issue("\n".join(f"- {x}" for x in failures)) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/monitoring/competitor-price/targets.yaml b/monitoring/competitor-price/targets.yaml new file mode 100644 index 0000000..c2a742c --- /dev/null +++ b/monitoring/competitor-price/targets.yaml @@ -0,0 +1,39 @@ +# 竞品价格监控目标清单(BIZ-145 / S2) +# 数据源:BIZ-144 v2 §1 竞品名单;模板字段:BIZ-144 v2 §3 +# ⚠️ sku_urls 目前为空 = 仅采集店铺首页公开信息。 +# 具体 SKU 页 URL 由数据 Owner(陆云帆)每周确认后补充到本文件; +# 白牌店铺每月 1 号由执行人按 BIZ-144 v2 §1.1 #5 规则选定后回填 url。 + +week_timezone: "Asia/Shanghai" + +competitors: + - id: senshan + name: 森山(森宇控股) + type: brand_flagship + shop_url: "https://senshan.tmall.com" + monitor_skus: ["铁皮枫斗", "铁皮石斛核心SKU"] + sku_urls: [] # TODO(owner): 填入具体 SKU 页面 URL + - id: conba + name: 康恩贝 + type: brand_flagship + shop_url: "https://conba.tmall.com" + monitor_skus: ["石斛/参茸线核心SKU"] + sku_urls: [] + - id: huqingyutang + name: 胡庆余堂 + type: brand_flagship + shop_url: "https://huqingyutang.tmall.com" + monitor_skus: ["石斛及传统滋补礼盒"] + sku_urls: [] + - id: shouxiangu + name: 寿仙谷 + type: brand_flagship + shop_url: "https://shouxian.tmall.com" + monitor_skus: ["铁皮石斛", "灵芝孢子粉"] + sku_urls: [] + - id: white_label_monthly + name: 平台头部白牌(每月轮换) + type: marketplace + shop_url: "" # 每月 1 号由执行人回填当月店铺 URL + monitor_skus: ["同规格散装石斛价格带下限"] + sku_urls: []