scrape_single_page.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726
  1. """
  2. 单个 Facebook 主页完整采集
  3. - 连接已打开的 AdsPower 浏览器
  4. - 访问指定主页 URL
  5. - 采集完整信息
  6. - 回写到 Excel
  7. """
  8. import re
  9. import sys
  10. import time
  11. import random
  12. import argparse
  13. from pathlib import Path
  14. import sys
  15. sys.path.append(str(Path(__file__).resolve().parents[1]))
  16. from common.artifact_manager import resolve_artifact_path, create_backup_once
  17. from typing import Dict, Any
  18. # Windows 控制台输出中文/阿拉伯文时避免 GBK 编码错误
  19. if hasattr(sys.stdout, "reconfigure"):
  20. sys.stdout.reconfigure(encoding="utf-8")
  21. if hasattr(sys.stderr, "reconfigure"):
  22. sys.stderr.reconfigure(encoding="utf-8")
  23. from playwright.sync_api import sync_playwright
  24. try:
  25. from deep_translator import GoogleTranslator
  26. TRANSLATOR_AVAILABLE = True
  27. except ImportError:
  28. TRANSLATOR_AVAILABLE = False
  29. GoogleTranslator = None
  30. try:
  31. from ..common import append_records, enrich_record_with_brands, resolve_workbook_path
  32. from . import discovery_common as dc
  33. from .website_deep_scraper import (
  34. build_chinese_notes,
  35. classify_customer_type_cn,
  36. extract_company_website_from_facebook,
  37. scrape_public_website,
  38. summarize_business_cn,
  39. )
  40. except ImportError:
  41. import sys
  42. sys.path.insert(0, str(Path(__file__).parent.parent))
  43. from common import append_records, enrich_record_with_brands, resolve_workbook_path
  44. from scraper import discovery_common as dc
  45. from scraper.website_deep_scraper import (
  46. build_chinese_notes,
  47. classify_customer_type_cn,
  48. extract_company_website_from_facebook,
  49. scrape_public_website,
  50. summarize_business_cn,
  51. )
  52. def get_active_ws_endpoint(base_url: str, profile_id: str) -> str:
  53. """通过 AdsPower API 获取已打开浏览器的 ws endpoint"""
  54. import requests
  55. url = f"{base_url}/api/v1/browser/active"
  56. resp = requests.get(url, params={"user_id": profile_id}, timeout=10)
  57. resp.raise_for_status()
  58. data = resp.json()
  59. if data.get("code") != 0:
  60. raise RuntimeError(f"获取活动浏览器失败: {data}")
  61. ws = data["data"]["ws"]["puppeteer"]
  62. return ws
  63. def is_mostly_chinese(text: str) -> bool:
  64. """判断文本是否主要为中文"""
  65. if not text:
  66. return False
  67. chinese_chars = sum(1 for c in text if "\u4e00" <= c <= "\u9fff")
  68. return chinese_chars / max(len(text), 1) > 0.3
  69. def translate_to_chinese(text: str) -> str:
  70. """将文本翻译为中文;失败或已是中文时返回原文"""
  71. if not text or len(text.strip()) < 3:
  72. return text
  73. if is_mostly_chinese(text):
  74. return text
  75. if not TRANSLATOR_AVAILABLE:
  76. return text
  77. try:
  78. translated = GoogleTranslator(source="auto", target="zh-CN").translate(text)
  79. # 简单限流,避免 Google 免费接口触发频率限制
  80. time.sleep(0.3)
  81. return translated or text
  82. except Exception as exc:
  83. print(f"翻译失败,保留原文: {exc}")
  84. return text
  85. def extract_phone(text: str) -> str:
  86. """Extract a public phone/WhatsApp number from Facebook text across countries."""
  87. if not text:
  88. return ""
  89. value = str(text or "")
  90. explicit = re.search(
  91. r"(?:Téléphone|Telephone|电话|Tél|Tel|WhatsApp|Phone|Fone|FONE|Commercial|Comercial)[::]?\s*([+()\d][+()\d\s.-]{6,}\d)",
  92. value,
  93. re.I,
  94. )
  95. if explicit:
  96. return re.sub(r"\s+", " ", explicit.group(1)).strip(" .,-")
  97. patterns = [
  98. r'\+\d{1,3}[\s\-.]?(?:\(?\d{1,4}\)?[\s\-.]?){2,6}\d',
  99. r'\+212[\s\-]?\d[\s\-]?\d{3}[\s\-]?\d{2}[\s\-]?\d{2}[\s\-]?\d{2}',
  100. r'\+212[\s\-]?\d{3}[\s\-]?\d{2}[\s\-]?\d{2}[\s\-]?\d{2}',
  101. r'0\d[\s\-]?\d{4}[\s\-]?\d{4}',
  102. r'0\d{3}[\s\-]?\d{2}[\s\-]?\d{2}[\s\-]?\d{2}',
  103. ]
  104. for pattern in patterns:
  105. match = re.search(pattern, value)
  106. if match:
  107. return re.sub(r"\s+", " ", match.group(0)).strip(" .,-")
  108. return ""
  109. def extract_email(text: str) -> str:
  110. """从文本中提取邮箱"""
  111. if not text:
  112. return ""
  113. pattern = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
  114. matches = re.findall(pattern, text)
  115. excluded = ["@facebook.com", "@fb.com", "@example.com"]
  116. for m in matches:
  117. if all(e not in m.lower() for e in excluded):
  118. return m.strip()
  119. return ""
  120. def extract_followers(text: str) -> str:
  121. """提取粉丝数,支持英文、法文、中文(含万/千)"""
  122. if not text:
  123. return ""
  124. # 优先匹配中文“X 万位粉丝 / X 万粉丝”格式
  125. cn_match = re.search(r'(\d+[\d\s,.]*)\s*([万千]?)\s*(位粉丝|粉丝)', text, re.IGNORECASE)
  126. if cn_match:
  127. num = cn_match.group(1).strip()
  128. unit = cn_match.group(2).strip()
  129. suffix = cn_match.group(3).strip()
  130. # 合并单位与后缀,如“万位粉丝”“万粉丝”
  131. if unit:
  132. return f"{num}{unit}{suffix}"
  133. return f"{num} {suffix}"
  134. # 英文/法文格式:1.2k fans / 3M followers
  135. en_match = re.search(r'(\d+[\d\s,.]*[KkMm]?)\s*(followers|fans|abonnés|likes)', text, re.IGNORECASE)
  136. if en_match:
  137. return f"{en_match.group(1).strip()} {en_match.group(2).strip()}"
  138. return ""
  139. def _looks_like_time(text: str) -> bool:
  140. """判断文本是否像时间戳"""
  141. text_lower = text.lower()
  142. time_units = ["h", "hr", "hrs", "hour", "hours", "d", "day", "days", "w", "week",
  143. "m", "min", "mins", "month", "months", "y", "year", "years",
  144. "昨天", "今天", "刚刚", "分钟", "小时", "天", "周", "月", "年"]
  145. has_unit = any(unit in text_lower for unit in time_units)
  146. has_number = bool(re.search(r'\d', text))
  147. has_date_separator = bool(re.search(r'\d{1,4}[/-]\d{1,2}[/-]\d{1,4}', text))
  148. short_relative = len(text) <= 8 and has_number
  149. date_like = has_number and (has_unit or has_date_separator or len(text) < 25)
  150. return (has_unit and has_number) or short_relative or date_like
  151. def _extract_post_time(element) -> str:
  152. """尝试从帖子元素中提取发布时间"""
  153. time_text = ""
  154. try:
  155. time_selectors = [
  156. 'a[href*="posts"] span',
  157. 'a[href*="posts"]',
  158. 'span[dir="auto"] a[role="link"] span',
  159. 'abbr',
  160. 'span[aria-label]',
  161. 'a[role="link"] span',
  162. ]
  163. for sel in time_selectors:
  164. time_el = element.query_selector(sel)
  165. if time_el:
  166. txt = (time_el.get_attribute("aria-label") or time_el.inner_text() or "").strip()
  167. if txt and _looks_like_time(txt):
  168. time_text = txt
  169. break
  170. except Exception:
  171. pass
  172. return time_text
  173. def scrape_posts(page, max_posts: int = 8) -> list:
  174. """
  175. 滚动主页并提取最近帖子文本及发布时间
  176. 返回: [{"text": str, "time": str}, ...]
  177. """
  178. posts = []
  179. seen_texts = set()
  180. start_time = time.time()
  181. # 回到主页
  182. current_url = page.url
  183. if "/about" in current_url:
  184. home_url = current_url.split("/about")[0] + "/"
  185. try:
  186. page.goto(home_url, wait_until="domcontentloaded", timeout=30000)
  187. except Exception:
  188. pass
  189. time.sleep(random.uniform(2, 4))
  190. # 滚动几次,加载帖子
  191. for i in range(3):
  192. # 尝试多种帖子容器选择器
  193. selectors = [
  194. 'div[role="article"]',
  195. 'div[data-ad-preview="message"]',
  196. '[data-pagelet="ProfileTimeline"] div[role="article"]',
  197. ]
  198. for sel in selectors:
  199. elements = page.query_selector_all(sel)
  200. for el in elements:
  201. try:
  202. # 获取帖子文本:优先从 dir="auto" 的 div 中取
  203. text_els = el.query_selector_all('div[dir="auto"]')
  204. text_parts = []
  205. for tel in text_els:
  206. t = tel.inner_text().strip()
  207. if t and len(t) > 5:
  208. text_parts.append(t)
  209. text = " ".join(text_parts).strip()
  210. # 过滤:长度适中、不是导航文字、未重复
  211. if 20 < len(text) < 800 and text not in seen_texts:
  212. excluded = ["个人资料", "帖子", "简介", "提及", "好友", "照片", "视频", "Reels", "签到", "展开", "查看更多", "评论", "分享", "点赞"]
  213. if not any(e in text for e in excluded):
  214. post_time = _extract_post_time(el)
  215. posts.append({"text": text, "time": post_time})
  216. seen_texts.add(text)
  217. except Exception:
  218. continue
  219. if len(posts) >= max_posts:
  220. break
  221. # 单页面帖子采集总时间限制 45 秒
  222. if time.time() - start_time > 45:
  223. print("帖子采集时间超过 45 秒,提前结束")
  224. break
  225. # 滚动
  226. page.evaluate("window.scrollTo(0, window.scrollY + 800)")
  227. time.sleep(random.uniform(2, 4))
  228. return posts[:max_posts]
  229. def _assess_activity_level(latest_time: str) -> str:
  230. """根据最近发帖时间评估活跃度"""
  231. if not latest_time or latest_time == "未识别":
  232. return "活跃度:未识别"
  233. text = latest_time.lower()
  234. # 近期:小时、天、本周
  235. if any(u in text for u in ["h", "hr", "hour", "hours", "小时", "d", "day", "days", "天", "w", "week", "周"]):
  236. return "活跃度:高"
  237. # 较近期:月且 <=3
  238. month_match = re.search(r"(\d+)\s*(month|months|mo|月)", text)
  239. if month_match:
  240. months = int(month_match.group(1))
  241. if months <= 3:
  242. return "活跃度:中"
  243. # 其他具体日期或更久远
  244. return "活跃度:低或陈旧"
  245. def analyze_posts(posts: list, about_text: str = "") -> str:
  246. """
  247. 综合分析最近帖子内容,输出自然语言备注。
  248. 包含:主营业务判断、近期动态、关注方向、活跃度评估、最近发帖时间、最新帖子摘要。
  249. posts: [{"text": str, "time": str}, ...]
  250. """
  251. if not posts:
  252. return ""
  253. texts = [p["text"] for p in posts]
  254. all_text = " ".join(texts).lower()
  255. # 1. 主营业务判断
  256. business_signals = {
  257. "汽车诊断/维修": ["diagnostic", "diag", "défaut", "moteur", "voyant", "réparation", "panne", "ordinateur de bord", "valise", "scanner"],
  258. "二手车交易": ["occasion", "vente", "achat", "vendre", "à vendre", "prix", "km", "kilométrage"],
  259. "汽车销售/经销商": ["concessionnaire", "neuf", "showroom", "livraison", "commande", "véhicule neuf"],
  260. "租车服务": ["location", "louer", "rental", "à louer", "jour", "mois"],
  261. "汽车美容/改装": ["tuning", "jantes", "peinture", "covering", "cleaning", "detailing"],
  262. }
  263. business_scores = {}
  264. for biz, keywords in business_signals.items():
  265. score = sum(all_text.count(kw) for kw in keywords)
  266. if score > 0:
  267. business_scores[biz] = score
  268. main_business = ""
  269. if business_scores:
  270. main_business = max(business_scores, key=business_scores.get)
  271. # 2. 近期动态
  272. activity_signals = {
  273. "促销活动": ["promo", "promotion", "offre", "réduction", "discount", "prix spécial", "vente flash"],
  274. "服务展示": ["service", "intervention", "réparation", "diagnostic", "résultat", "avant/après"],
  275. "客户案例": ["client", "témoignage", "satisfait", "merci", "avis"],
  276. "日常内容": ["bonjour", "bonne journée", "week-end", "maroc", "casablanca"],
  277. }
  278. activity_scores = {}
  279. for act, keywords in activity_signals.items():
  280. score = sum(all_text.count(kw) for kw in keywords)
  281. if score > 0:
  282. activity_scores[act] = score
  283. recent_activities = sorted(activity_scores, key=activity_scores.get, reverse=True)[:2]
  284. # 3. 关注方向 / 高频词(翻译为中文)
  285. focus_map = {
  286. "diagnostic": "诊断",
  287. "voiture": "汽车",
  288. "auto": "汽车",
  289. "moteur": "发动机",
  290. "réparation": "维修",
  291. "occasion": "二手车",
  292. "prix": "价格",
  293. "casablanca": "卡萨布兰卡",
  294. "maroc": "摩洛哥",
  295. "service": "服务",
  296. "client": "客户",
  297. "promo": "促销",
  298. }
  299. focus_counts = {}
  300. for kw, cn in focus_map.items():
  301. count = all_text.count(kw)
  302. if count > 0:
  303. focus_counts[cn] = focus_counts.get(cn, 0) + count
  304. top_focus = sorted(focus_counts, key=focus_counts.get, reverse=True)[:5]
  305. # 4. 最近帖子时间 + 摘要
  306. latest_post = posts[0]
  307. latest_time = latest_post.get("time", "").strip()
  308. latest_text = latest_post["text"]
  309. latest_summary = latest_text[:120] + "..." if len(latest_text) > 120 else latest_text
  310. activity_level = _assess_activity_level(latest_time)
  311. # 组装备注
  312. parts = []
  313. if main_business:
  314. parts.append(f"主营业务判断:{main_business}")
  315. if recent_activities:
  316. parts.append(f"近期动态:{', '.join(recent_activities)}")
  317. if top_focus:
  318. parts.append(f"关注方向:{', '.join(top_focus)}")
  319. parts.append(f"{activity_level}(最近发帖时间:{latest_time or '未识别'})")
  320. parts.append(f"最新帖子摘要:{latest_summary}")
  321. return " | ".join(parts)
  322. def infer_dealer_type(text: str) -> str:
  323. """推断客户类型"""
  324. if not text:
  325. return "待判断"
  326. text_lower = text.lower()
  327. # 个人资料页信号
  328. personal_signals = [
  329. "personal information", "informations personnelles", "个人信息",
  330. "date of birth", "date de naissance", "生日",
  331. "works at", "travail chez", "études à", "studied at",
  332. "lives in", "habite à", "vit à", "relationship", "relation",
  333. "family members", "membres de la famille",
  334. ]
  335. if any(k in text_lower for k in personal_signals):
  336. return "个人用户"
  337. if any(k in text_lower for k in ["diagnostic", "diag", "diagnostique", "诊断"]):
  338. return "汽车诊断/维修"
  339. if any(k in text_lower for k in ["used car", "occasion", "二手车", "occaz"]):
  340. return "二手车商"
  341. if any(k in text_lower for k in ["dealer", "concessionnaire", "distributeur", "经销商"]):
  342. return "经销商"
  343. if any(k in text_lower for k in ["rental", "location", "租车"]):
  344. return "租车公司"
  345. if any(k in text_lower for k in ["repair", "garage", "维修"]):
  346. return "汽车维修"
  347. return "待判断"
  348. def scrape_page_record(
  349. page,
  350. page_url: str,
  351. city: str = "Casablanca",
  352. country: str = "摩洛哥",
  353. name: str = "",
  354. main_business: str = "",
  355. ) -> Dict[str, Any]:
  356. """使用已存在的 Playwright page 采集单个 Facebook 主页信息"""
  357. start_time = time.time()
  358. record = {
  359. "客户姓名/公司": "",
  360. "国家": country,
  361. "城市": city,
  362. "客户类型": "",
  363. "主页/链接": page_url,
  364. "公司官网": "",
  365. "联系人": "",
  366. "职位": "",
  367. "电话/WhatsApp": "",
  368. "邮箱": "",
  369. "主营业务": "",
  370. "建联状态": "未联系",
  371. "下次跟进": "",
  372. "备注": "",
  373. }
  374. try:
  375. # 访问主页
  376. page.goto(page_url, wait_until="domcontentloaded", timeout=30000)
  377. time.sleep(random.uniform(3, 5))
  378. # 先提取主页上的粉丝数
  379. home_page_text = ""
  380. try:
  381. home_page_text = page.locator("body").inner_text(timeout=10000)
  382. except Exception:
  383. pass
  384. followers_from_home = extract_followers(home_page_text)
  385. website_from_home = extract_company_website_from_facebook(page)
  386. if website_from_home:
  387. record["公司官网"] = website_from_home
  388. print(f"从主页联络资料提取到公司官网: {website_from_home}")
  389. home_phone = extract_phone(home_page_text)
  390. if home_phone:
  391. record["电话/WhatsApp"] = home_phone
  392. print(f"从主页联络资料提取到电话: {home_phone}")
  393. home_email = extract_email(home_page_text)
  394. if home_email:
  395. record["邮箱"] = home_email
  396. print(f"从主页联络资料提取到邮箱: {home_email}")
  397. if followers_from_home:
  398. print(f"从主页联络资料提取到粉丝数: {followers_from_home}")
  399. # 如果用户提供了名称,直接使用
  400. if name:
  401. record["客户姓名/公司"] = name
  402. else:
  403. # 尝试多种方式获取页面名称
  404. record["客户姓名/公司"] = ""
  405. name_selectors = [
  406. '[role="main"] h1',
  407. '[data-pagelet="ProfileActions"] h1',
  408. 'h1',
  409. ]
  410. for sel in name_selectors:
  411. try:
  412. el = page.query_selector(sel)
  413. if el:
  414. text = el.inner_text().strip()
  415. if text and len(text) < 100 and "Facebook" not in text and text not in ["个人资料", "帖子", "简介", "提及"]:
  416. record["客户姓名/公司"] = text
  417. break
  418. except Exception:
  419. continue
  420. # 如果 h1 都没拿到,从 title 或 URL 推断
  421. if not record["客户姓名/公司"]:
  422. title = page.title()
  423. name_from_title = title.split("|")[0].strip() if "|" in title else title.replace("Facebook", "").strip()
  424. if name_from_title and name_from_title not in ["个人资料", "帖子", "简介", "提及", "通知"] and len(name_from_title) < 100:
  425. record["客户姓名/公司"] = name_from_title
  426. else:
  427. from urllib.parse import urlparse
  428. path = urlparse(page_url).path.strip("/").split("/")[0]
  429. record["客户姓名/公司"] = path or "Unknown"
  430. # 主页联络资料读取完成后,再获取 About 信息
  431. about_url = f"{page_url.rstrip('/')}/about/"
  432. print(f"访问 About 页: {about_url}")
  433. page.goto(about_url, wait_until="domcontentloaded", timeout=30000)
  434. time.sleep(random.uniform(2, 4))
  435. page_text = ""
  436. try:
  437. page_text = page.locator("body").inner_text(timeout=15000)
  438. except Exception:
  439. pass
  440. about_website = extract_company_website_from_facebook(page)
  441. if about_website:
  442. if not record.get("公司官网"):
  443. record["公司官网"] = about_website
  444. elif about_website not in record["公司官网"]:
  445. record["公司官网"] = record["公司官网"] + ";" + about_website
  446. company_website = record.get("公司官网", "").split(";")[0].strip()
  447. website_result = {"email": "", "phone": "", "text": "", "business_summary": "", "evidence_notes": []}
  448. if company_website:
  449. try:
  450. print(f"深搜公司官网: {company_website}")
  451. website_result = scrape_public_website(page.context, company_website, max_pages=5)
  452. except Exception as exc:
  453. website_result = {"email": "", "phone": "", "text": "", "business_summary": "", "evidence_notes": [f"官网深搜失败:{str(exc)[:80]}"]}
  454. # 提取电话
  455. phone = extract_phone(page_text)
  456. if phone and not record.get("电话/WhatsApp"):
  457. record["电话/WhatsApp"] = phone
  458. # 提取邮箱,About 只补充空字段
  459. email = extract_email(page_text)
  460. if email and not record.get("邮箱"):
  461. record["邮箱"] = email
  462. if not record.get("邮箱") and website_result.get("email"):
  463. record["邮箱"] = website_result.get("email", "")
  464. if not record.get("电话/WhatsApp") and website_result.get("phone"):
  465. record["电话/WhatsApp"] = website_result.get("phone", "")
  466. # 如果用户提供了主营业务,直接使用
  467. if main_business:
  468. record["主营业务"] = main_business
  469. else:
  470. # 提取主营业务/描述
  471. description = ""
  472. lines = [line.strip() for line in page_text.split("\n") if line.strip()]
  473. # 优先找 "个人资料" 下方的描述句
  474. profile_index = -1
  475. for i, line in enumerate(lines):
  476. if line in ["个人资料", "About", "À propos", "简介"]:
  477. profile_index = i
  478. break
  479. if profile_index >= 0:
  480. for line in lines[profile_index+1:profile_index+10]:
  481. if len(line) > 10 and len(line) < 300:
  482. description = line
  483. break
  484. # 如果没找到,找包含关键词的完整描述句
  485. if not description:
  486. keywords = ["car", "auto", "voiture", "vehicle", "motor", "dealer", "occasion", "automotive", "diagnostic", "diag", "concessionnaire", "vente", "location"]
  487. best_line = ""
  488. for line in lines[:80]:
  489. lower_line = line.lower()
  490. if any(k in lower_line for k in keywords):
  491. if len(line) > len(best_line) and len(line) < 300:
  492. best_line = line
  493. if best_line:
  494. description = best_line
  495. # 兜底:用公司名称
  496. if not description:
  497. description = record["客户姓名/公司"]
  498. record["主营业务"] = translate_to_chinese(description)
  499. # 推断客户类型,官网深搜证据优先参与判断
  500. combined_business_text = "\n".join([home_page_text, page_text, website_result.get("text", "")])
  501. inferred_type = classify_customer_type_cn(combined_business_text)
  502. legacy_type = infer_dealer_type(page_text)
  503. record["客户类型"] = legacy_type if legacy_type == "个人用户" else inferred_type
  504. if not main_business:
  505. record["主营业务"] = summarize_business_cn(combined_business_text)
  506. # 备注:粉丝数 + 帖子分析
  507. # 优先使用主页提取的粉丝数,若主页没有则尝试 About 页
  508. followers = followers_from_home or extract_followers(page_text)
  509. post_analysis = ""
  510. try:
  511. print("正在浏览最近帖子并分析...")
  512. posts = scrape_posts(page, max_posts=8)
  513. post_analysis = analyze_posts(posts, about_text=page_text)
  514. print(f"采集到 {len(posts)} 条帖子")
  515. except Exception as e:
  516. print(f"帖子分析失败: {e}")
  517. translated_post_analysis = translate_to_chinese(post_analysis) if post_analysis else ""
  518. # 先把帖子分析写入备注,便于品牌检测提取上下文
  519. if translated_post_analysis:
  520. record["备注"] = translated_post_analysis
  521. # 品牌检测辅助
  522. record = enrich_record_with_brands(record)
  523. # 组装中文结构化备注:Facebook证据 + 官网证据 + 业务判断 + 联系方式证据
  524. detected = record.get("detected_brands", [])
  525. exclusivity = record.get("exclusivity_assessment", "")
  526. record["备注"] = build_chinese_notes(
  527. facebook_url=page_url,
  528. facebook_text="\n".join([home_page_text, page_text]),
  529. website_url=record.get("公司官网", ""),
  530. website_result=website_result,
  531. followers=followers,
  532. post_analysis=translated_post_analysis,
  533. detected_brands=detected if isinstance(detected, list) else [],
  534. exclusivity=exclusivity,
  535. )
  536. if record.get("备注"):
  537. record["备注"] = "证据采集顺序:主页联络资料 -> About -> 官网深搜 | " + record["备注"]
  538. print("采集完成:")
  539. for k, v in record.items():
  540. if v:
  541. print(f" {k}: {v}")
  542. except Exception as e:
  543. elapsed = time.time() - start_time
  544. print(f"采集过程异常({elapsed:.1f}s): {e}")
  545. if not record["客户姓名/公司"]:
  546. from urllib.parse import urlparse
  547. path = urlparse(page_url).path.strip("/").split("/")[0]
  548. record["客户姓名/公司"] = path or "Unknown"
  549. record["备注"] = (record.get("备注", "") + f" | 采集异常: {str(e)[:80]}").strip(" | ")
  550. return record
  551. def scrape_facebook_page(
  552. page_url: str,
  553. profile_id: str,
  554. city: str = "Casablanca",
  555. country: str = "摩洛哥",
  556. name: str = "",
  557. main_business: str = "",
  558. ads_power_url: str = "http://127.0.0.1:50325",
  559. ) -> Dict[str, Any]:
  560. """采集单个 Facebook 主页完整信息(自动管理 Playwright 生命周期)"""
  561. ws_endpoint = get_active_ws_endpoint(ads_power_url, profile_id)
  562. print(f"连接到已打开的浏览器: {ws_endpoint}")
  563. playwright = sync_playwright().start()
  564. browser = playwright.chromium.connect_over_cdp(ws_endpoint)
  565. context = browser.contexts[0] if browser.contexts else browser.new_context()
  566. page = context.new_page()
  567. print(f"新建标签页,访问: {page_url}")
  568. try:
  569. record = scrape_page_record(
  570. page=page,
  571. page_url=page_url,
  572. city=city,
  573. country=country,
  574. name=name,
  575. main_business=main_business,
  576. )
  577. finally:
  578. print("停止 Playwright,浏览器窗口保持打开")
  579. playwright.stop()
  580. return record
  581. def main():
  582. parser = argparse.ArgumentParser(description="单个 Facebook 主页完整采集并写入 Excel")
  583. parser.add_argument("--url", required=True, help="Facebook 主页 URL")
  584. parser.add_argument("--profile-id", required=True, help="AdsPower profile ID")
  585. parser.add_argument("--city", default="摩洛哥全国", help="城市或覆盖范围")
  586. parser.add_argument("--country", default="摩洛哥", help="国家")
  587. parser.add_argument("--name", default="", help="公司名称(脚本识别不准时手动指定)")
  588. parser.add_argument("--main-business", default="", help="主营业务(脚本识别不准时手动指定)")
  589. parser.add_argument("--ads-power-url", default="http://127.0.0.1:50325", help="AdsPower API URL")
  590. parser.add_argument("--excel", default="", help="建联表路径;不传时按项目优先级自动查找")
  591. parser.add_argument("--sheet", default="Facebook", help="Sheet 名")
  592. parser.add_argument("--write-excel", action="store_true", help="确认写入建联表;否则只输出 JSON 预览")
  593. parser.add_argument("--run-id", default="", help="Run ID used for runs/YYYYMMDD/<run_id>/ artifacts.")
  594. parser.add_argument("--output", default="single_page_scraped.json", help="JSON 输出")
  595. parser.add_argument("--no-excel", action="store_true", help="不写入 Excel,只输出 JSON")
  596. args = parser.parse_args()
  597. record = scrape_facebook_page(
  598. page_url=args.url,
  599. profile_id=args.profile_id,
  600. city=args.city,
  601. country=args.country,
  602. name=args.name,
  603. main_business=args.main_business,
  604. ads_power_url=args.ads_power_url,
  605. )
  606. # 保存 JSON
  607. import json
  608. output_path = resolve_artifact_path(args.output, kind="single_page_scrape", default_name=Path(args.output).name, run_id=args.run_id or None)
  609. output_path.parent.mkdir(parents=True, exist_ok=True)
  610. with open(output_path, "w", encoding="utf-8") as f:
  611. json.dump(record, f, ensure_ascii=False, indent=2)
  612. print(f"已保存 JSON: {args.output}")
  613. # 写入 Excel
  614. if args.write_excel and not args.no_excel:
  615. workbook_info = resolve_workbook_path(args.excel, create_from_template=True)
  616. args.excel = str(workbook_info["path"])
  617. print(f"Workbook for writing: {args.excel} ({workbook_info['source']})")
  618. result = append_records(
  619. excel_path=args.excel,
  620. sheet_name=args.sheet,
  621. records=[record],
  622. dedup_keys=["客户姓名/公司", "城市", "主页/链接"]
  623. )
  624. print(f"Excel write result: {result}")
  625. else:
  626. print("默认预览模式:未写入 Excel。确认要入表时再使用 --write-excel。")
  627. if __name__ == "__main__":
  628. main()