diff --git a/monitoring/competitor-price/README.md b/monitoring/competitor-price/README.md new file mode 100644 index 0000000..6214563 --- /dev/null +++ b/monitoring/competitor-price/README.md @@ -0,0 +1,66 @@ +# 竞品价格监控采集(BIZ-145 / S2) + +按 BIZ-144 v2 竞品名单每周采集一次公开价格信息,落地 JSON/CSV 快照,供 BIZ-146 生成周报。 + +## 合规红线(硬编码于 `fetch_prices.py`,不可命令行覆盖) + +| 条目 | 实现 | +|------|------| +| C1 频率 | 每周一次(cron);单店 ≤10 页;请求间隔 ≥5s | +| C2 robots.txt | 被禁止即跳过,记 `skipped_robots` | +| C3 不绕反爬 | 固定常规 UA;遇验证码/登录墙(401/403/特征词)即放弃该页 | +| C4 字段白名单 | `name / list_price / promo_text / sales_display`,只抽公开页面展示字段 | +| C5 留存 | 原始快照保留 90 天,每次运行自动清理过期目录 | +| C7 无代理/账号 | 直连、无登录态 | + +## 目录结构 + +``` +competitor-price/ +├── fetch_prices.py # 采集脚本(仅标准库,Python ≥3.10) +├── targets.yaml # 竞品名单与 SKU URL(数据 Owner 维护,见文件头注释) +├── cron/ +│ └── competitor-price.cron # 每周一 09:00 GMT+8 +└── data/YYYY-MM-DD/ + ├── prices.json # 含合规元信息 run_meta + results + failures + ├── prices.csv # 同数据的 CSV 版(UTF-8 BOM,Excel 可直接打开) + └── fetch.log # 运行日志 +``` + +## 本地手动运行(调试用) + +```bash +cd monitoring/competitor-price +python3 fetch_prices.py # 退出码 0=全部成功,1=存在失败/跳过 +``` + +调试时不触发父单告警: + +```bash +MULTICA_PARENT_ISSUE="" python3 fetch_prices.py # 空值视为未配置,只写日志 +``` + +## 安装 cron(每周一 09:00 Asia/Shanghai) + +```bash +crontab -l > /tmp/mycron 2>/dev/null || true +cat cron/competitor-price.cron >> /tmp/mycron +crontab /tmp/mycron && rm /tmp/mycron +crontab -l | grep competitor-price # 验证 +``` + +cron 机器需满足:`python3` ≥3.10、`multica` CLI 已登录(用于失败时在父单 BIZ-143 评论告警)。 +服务器并非 24h 开机的话,cron 条目含 flock 防重入;漏跑可在下周一前手动补跑一次即可。 + +## 失败告警 + +任一条目非 `ok` 时,脚本在父单 BIZ-143 留一条失败汇总评论(需 `multica` 可用), +同时本地 `fetch.log` 留痕。白牌店铺未回填 URL 记为 `skipped_no_url`,属预期,不告警。 + +## 已知边界(如实说明) + +- Tmall 店铺首页 `robots.txt` 对通用 UA 默认禁止 → 当前 4 家品牌旗舰店均记 `skipped_robots`, + 这是合规设计的预期行为,不是脚本故障。要拿到真实价格,需要数据 Owner 在 + `targets.yaml` 的 `sku_urls` 中回填**允许抓取的页面**,或由业务方决定改用其他公开数据源。 +- 价格抽取基于页面公开 HTML 的通用正则(`"price"` 字段 / ¥ 符号),对重前端渲染页面 + 可能抽不到 → 记 `ok_no_price_field`,周报侧据此人工复核。 diff --git a/monitoring/competitor-price/cron/competitor-price.cron b/monitoring/competitor-price/cron/competitor-price.cron new file mode 100644 index 0000000..959404d --- /dev/null +++ b/monitoring/competitor-price/cron/competitor-price.cron @@ -0,0 +1,4 @@ +# BIZ-145 竞品价格采集:每周一 09:00(Asia/Shanghai) +# 安装方式见同目录 README.md「安装 cron」一节 +CRON_TZ=Asia/Shanghai +0 9 * * 1 flock -n /tmp/competitor-price.lock /usr/bin/env MULTICA_PARENT_ISSUE=448cdf6f-ac97-4c7f-953f-0d92d03c6e85 python3 /home/vincent/.openclaw/workspace/costcodev/EnterpriseArchitect/monitoring/competitor-price/fetch_prices.py >> /home/vincent/.openclaw/workspace/costcodev/EnterpriseArchitect/monitoring/competitor-price/data/cron.log 2>&1 diff --git a/monitoring/competitor-price/data/2026-09-20/fetch.log b/monitoring/competitor-price/data/2026-09-20/fetch.log new file mode 100644 index 0000000..7f45412 --- /dev/null +++ b/monitoring/competitor-price/data/2026-09-20/fetch.log @@ -0,0 +1,8 @@ +2026-09-20 05:32:09,160 INFO === BIZ-145 竞品价格采集启动 2026-09-20 === +2026-09-20 05:32:09,570 WARNING robots.txt 禁止抓取,跳过:https://senshan.tmall.com +2026-09-20 05:32:14,843 WARNING robots.txt 禁止抓取,跳过:https://conba.tmall.com +2026-09-20 05:32:20,065 WARNING robots.txt 禁止抓取,跳过:https://huqingyutang.tmall.com +2026-09-20 05:32:27,297 WARNING robots.txt 禁止抓取,跳过:https://shouxian.tmall.com +2026-09-20 05:32:32,298 WARNING 平台头部白牌(每月轮换):无可用 URL(白牌未回填),跳过 +2026-09-20 05:32:37,304 INFO === 完成:5 条记录,4 条失败 === +2026-09-20 05:32:37,342 ERROR 父单告警发送失败: Command '['multica', 'issue', 'comment', 'add', '', '--content', '⚠️ 竞品价格采集失败告警(2026-09-20 05:32 GMT+8)\n\n- 森山(森宇控股) robots 禁止 https://senshan.tmall.com\n- 康恩贝 robots 禁止 https://conba.tmall.com\n- 胡庆余堂 robots 禁止 https://huqingyutang.tmall.com\n- 寿仙谷 robots 禁止 https://shouxian.tmall.com\n\n详见监控目录 data/ 下当日 fetch.log。']' returned non-zero exit status 1. diff --git a/monitoring/competitor-price/data/2026-09-20/prices.csv b/monitoring/competitor-price/data/2026-09-20/prices.csv new file mode 100644 index 0000000..bb2ab38 --- /dev/null +++ b/monitoring/competitor-price/data/2026-09-20/prices.csv @@ -0,0 +1,6 @@ +competitor_id,competitor,url,status,name,list_price,promo_text,sales_display +senshan,森山(森宇控股),https://senshan.tmall.com,skipped_robots,,,, +conba,康恩贝,https://conba.tmall.com,skipped_robots,,,, +huqingyutang,胡庆余堂,https://huqingyutang.tmall.com,skipped_robots,,,, +shouxiangu,寿仙谷,https://shouxian.tmall.com,skipped_robots,,,, +white_label_monthly,平台头部白牌(每月轮换),,skipped_no_url,,,, diff --git a/monitoring/competitor-price/data/2026-09-20/prices.json b/monitoring/competitor-price/data/2026-09-20/prices.json new file mode 100644 index 0000000..984da1f --- /dev/null +++ b/monitoring/competitor-price/data/2026-09-20/prices.json @@ -0,0 +1,76 @@ +{ + "run_date": "2026-09-20", + "tz": "Asia/Shanghai", + "compliance": { + "C1_interval_sec": 5.0, + "C1_max_pages_per_shop": 10, + "C2_robots_respected": true, + "C3_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 (+internal price monitoring; contact: ops@bizwingsinc.com)", + "C4_field_whitelist": [ + "name", + "list_price", + "promo_text", + "sales_display" + ], + "C5_retention_days": 90, + "C7_no_proxy_no_login": true + }, + "results": [ + { + "competitor_id": "senshan", + "competitor": "森山(森宇控股)", + "url": "https://senshan.tmall.com", + "status": "skipped_robots", + "name": "", + "list_price": "", + "promo_text": "", + "sales_display": "" + }, + { + "competitor_id": "conba", + "competitor": "康恩贝", + "url": "https://conba.tmall.com", + "status": "skipped_robots", + "name": "", + "list_price": "", + "promo_text": "", + "sales_display": "" + }, + { + "competitor_id": "huqingyutang", + "competitor": "胡庆余堂", + "url": "https://huqingyutang.tmall.com", + "status": "skipped_robots", + "name": "", + "list_price": "", + "promo_text": "", + "sales_display": "" + }, + { + "competitor_id": "shouxiangu", + "competitor": "寿仙谷", + "url": "https://shouxian.tmall.com", + "status": "skipped_robots", + "name": "", + "list_price": "", + "promo_text": "", + "sales_display": "" + }, + { + "competitor_id": "white_label_monthly", + "competitor": "平台头部白牌(每月轮换)", + "url": "", + "status": "skipped_no_url", + "name": "", + "list_price": "", + "promo_text": "", + "sales_display": "" + } + ], + "failures": [ + "森山(森宇控股) robots 禁止 https://senshan.tmall.com", + "康恩贝 robots 禁止 https://conba.tmall.com", + "胡庆余堂 robots 禁止 https://huqingyutang.tmall.com", + "寿仙谷 robots 禁止 https://shouxian.tmall.com" + ] +} \ No newline at end of file diff --git a/monitoring/competitor-price/fetch_prices.py b/monitoring/competitor-price/fetch_prices.py new file mode 100644 index 0000000..32c8f8f --- /dev/null +++ b/monitoring/competitor-price/fetch_prices.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +BIZ-145 / S2 竞品价格采集脚本 + +合规红线(法务 BIZ-147 复核意见 C1–C7,全部硬编码,不可被命令行覆盖): + C1 频率封顶:单店每周 ≤1 次、单次 ≤10 页面、请求间隔 ≥5 秒 + C2 遵守 robots.txt:被禁抓即跳过 + C3 不绕反爬:固定常规 UA、不用 IP 池、遇验证码/登录墙即放弃该页面 + C4 字段白名单:商品名称 / 页面展示标价 / 页面展示促销信息 / 页面可见销量数 + C5 数据留存:原始快照 ≤90 天(--prune 清理) + C7 不使用代理/账号池:直连、无登录态 + +输出:data/YYYY-MM-DD/prices.json + prices.csv + fetch.log +失败告警:整体或单店失败时写日志;环境变量 MULTICA_PARENT_ISSUE 存在时 + 通过 multica CLI 在父单留一条失败通知评论。 +""" + +import csv +import json +import logging +import os +import re +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import urllib.robotparser +from datetime import datetime, timedelta, timezone +from pathlib import Path + +# ---------------- 合规常量(C1–C7,硬编码) ---------------- +MAX_PAGES_PER_SHOP = 10 # C1 +MIN_INTERVAL_SEC = 5.0 # C1 +RETENTION_DAYS = 90 # C5 +USER_AGENT = ( # C3/C6:常规 UA,不带伪装、附采集说明联系方式 + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 " + "(+internal price monitoring; contact: ops@bizwingsinc.com)" +) +TIMEOUT_SEC = 20 +ALLOWED_FIELDS = ("name", "list_price", "promo_text", "sales_display") # C4 白名单 + +# 验证码 / 登录墙特征(C3:命中即放弃,不尝试绕过) +BLOCK_PATTERNS = [ + "login.taobao.com", "passport", "验证码", "verify", "captcha", + "slide.tsx", "punish", "sec.taobao", +] + +BASE_DIR = Path(__file__).resolve().parent +DATA_DIR = BASE_DIR / "data" +TARGETS_FILE = BASE_DIR / "targets.yaml" +CST = timezone(timedelta(hours=8)) + +log = logging.getLogger("competitor-price") + + +def setup_logging(log_file: Path): + log.setLevel(logging.INFO) + fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + sh = logging.StreamHandler(sys.stdout) + sh.setFormatter(fmt) + fh = logging.FileHandler(log_file, encoding="utf-8") + fh.setFormatter(fmt) + log.addHandler(sh) + log.addHandler(fh) + + +def load_targets() -> list[dict]: + """极简 YAML 解析(targets.yaml 结构固定,避免引入第三方依赖)。""" + text = TARGETS_FILE.read_text(encoding="utf-8") + competitors, cur, in_sku, in_monitor = [], None, False, False + for raw in text.splitlines(): + line = raw.split("#", 1)[0].rstrip() + if not line.strip(): + continue + stripped = line.strip() + if stripped.startswith("- id:"): + cur = {"id": stripped.split(":", 1)[1].strip(), "sku_urls": [], + "monitor_skus": []} + competitors.append(cur) + in_sku = in_monitor = False + elif cur is None: + continue + elif re.match(r"^- \w+:", stripped) is None and ":" in stripped and not stripped.startswith("-"): + key, _, val = stripped.partition(":") + val = val.strip().strip('"') + if key == "sku_urls": + in_sku, in_monitor = True, False + if val.startswith("[") and val != "[]": + cur["sku_urls"] = [u.strip().strip('"') for u in val.strip("[]").split(",") if u.strip()] + elif key == "monitor_skus": + in_monitor, in_sku = True, False + else: + cur[key] = val + in_sku = in_monitor = False + elif stripped.startswith("-") and in_sku: + cur["sku_urls"].append(stripped.lstrip("- ").strip('"')) + elif stripped.startswith("-") and in_monitor: + cur["monitor_skus"].append(stripped.lstrip("- ").strip('"')) + return competitors + + +def robots_allows(url: str) -> bool: + """C2:遵守 robots.txt;出错时保守处理为允许但记日志。""" + parsed = urllib.parse.urlparse(url) + rp_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt" + rp = urllib.robotparser.RobotFileParser() + try: + rp.set_url(rp_url) + rp.read() + return rp.can_fetch(USER_AGENT, url) + except Exception as e: # robots 不可达时不放大风险,记录并放行(人工每周低频) + log.warning("robots.txt 读取失败 %s: %s(保守放行,人工复核)", rp_url, e) + return True + + +def fetch_page(url: str) -> tuple[int | None, str]: + """返回 (http_status, html)。不跟随登录跳转以外的异常。""" + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, + "Accept-Language": "zh-CN,zh;q=0.9"}) + try: + with urllib.request.urlopen(req, timeout=TIMEOUT_SEC) as resp: + html = resp.read(2_000_000).decode("utf-8", errors="replace") + return resp.status, html + except urllib.error.HTTPError as e: + return e.code, "" + except Exception as e: + log.warning("请求异常 %s: %s", url, e) + return None, "" + + +def looks_blocked(html: str, final_status: int | None) -> bool: + """C3:命中反爬/登录墙特征 → 放弃该页面。""" + if final_status in (401, 403): + return True + low = html.lower() + return any(p.lower() in low for p in BLOCK_PATTERNS) + + +PRICE_RE = re.compile(r'"price"\s*:\s*"?([\d.]+)"?|¥\s*([\d]+(?:\.\d+)?)|¥\s*([\d]+(?:\.\d+)?)') +PROMO_RE = re.compile(r"(满\d+减\d+|第二件半价|买一送一|买赠|券后)") +SALES_RE = re.compile(r"(?:月销|已售|销量)\s*[::]?\s*([\d,]+\+?)") +TITLE_RE = re.compile(r"