| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726 |
- """
- 单个 Facebook 主页完整采集
- - 连接已打开的 AdsPower 浏览器
- - 访问指定主页 URL
- - 采集完整信息
- - 回写到 Excel
- """
- import re
- import sys
- import time
- import random
- import argparse
- from pathlib import Path
- import sys
- sys.path.append(str(Path(__file__).resolve().parents[1]))
- from common.artifact_manager import resolve_artifact_path, create_backup_once
- from typing import Dict, Any
- # Windows 控制台输出中文/阿拉伯文时避免 GBK 编码错误
- if hasattr(sys.stdout, "reconfigure"):
- sys.stdout.reconfigure(encoding="utf-8")
- if hasattr(sys.stderr, "reconfigure"):
- sys.stderr.reconfigure(encoding="utf-8")
- from playwright.sync_api import sync_playwright
- try:
- from deep_translator import GoogleTranslator
- TRANSLATOR_AVAILABLE = True
- except ImportError:
- TRANSLATOR_AVAILABLE = False
- GoogleTranslator = None
- try:
- from ..common import append_records, enrich_record_with_brands, resolve_workbook_path
- from . import discovery_common as dc
- from .website_deep_scraper import (
- build_chinese_notes,
- classify_customer_type_cn,
- extract_company_website_from_facebook,
- scrape_public_website,
- summarize_business_cn,
- )
- except ImportError:
- import sys
- sys.path.insert(0, str(Path(__file__).parent.parent))
- from common import append_records, enrich_record_with_brands, resolve_workbook_path
- from scraper import discovery_common as dc
- from scraper.website_deep_scraper import (
- build_chinese_notes,
- classify_customer_type_cn,
- extract_company_website_from_facebook,
- scrape_public_website,
- summarize_business_cn,
- )
- def get_active_ws_endpoint(base_url: str, profile_id: str) -> str:
- """通过 AdsPower API 获取已打开浏览器的 ws endpoint"""
- import requests
- url = f"{base_url}/api/v1/browser/active"
- resp = requests.get(url, params={"user_id": profile_id}, timeout=10)
- resp.raise_for_status()
- data = resp.json()
- if data.get("code") != 0:
- raise RuntimeError(f"获取活动浏览器失败: {data}")
- ws = data["data"]["ws"]["puppeteer"]
- return ws
- def is_mostly_chinese(text: str) -> bool:
- """判断文本是否主要为中文"""
- if not text:
- return False
- chinese_chars = sum(1 for c in text if "\u4e00" <= c <= "\u9fff")
- return chinese_chars / max(len(text), 1) > 0.3
- def translate_to_chinese(text: str) -> str:
- """将文本翻译为中文;失败或已是中文时返回原文"""
- if not text or len(text.strip()) < 3:
- return text
- if is_mostly_chinese(text):
- return text
- if not TRANSLATOR_AVAILABLE:
- return text
- try:
- translated = GoogleTranslator(source="auto", target="zh-CN").translate(text)
- # 简单限流,避免 Google 免费接口触发频率限制
- time.sleep(0.3)
- return translated or text
- except Exception as exc:
- print(f"翻译失败,保留原文: {exc}")
- return text
- def extract_phone(text: str) -> str:
- """Extract a public phone/WhatsApp number from Facebook text across countries."""
- if not text:
- return ""
- value = str(text or "")
- explicit = re.search(
- r"(?:Téléphone|Telephone|电话|Tél|Tel|WhatsApp|Phone|Fone|FONE|Commercial|Comercial)[::]?\s*([+()\d][+()\d\s.-]{6,}\d)",
- value,
- re.I,
- )
- if explicit:
- return re.sub(r"\s+", " ", explicit.group(1)).strip(" .,-")
- patterns = [
- r'\+\d{1,3}[\s\-.]?(?:\(?\d{1,4}\)?[\s\-.]?){2,6}\d',
- r'\+212[\s\-]?\d[\s\-]?\d{3}[\s\-]?\d{2}[\s\-]?\d{2}[\s\-]?\d{2}',
- r'\+212[\s\-]?\d{3}[\s\-]?\d{2}[\s\-]?\d{2}[\s\-]?\d{2}',
- r'0\d[\s\-]?\d{4}[\s\-]?\d{4}',
- r'0\d{3}[\s\-]?\d{2}[\s\-]?\d{2}[\s\-]?\d{2}',
- ]
- for pattern in patterns:
- match = re.search(pattern, value)
- if match:
- return re.sub(r"\s+", " ", match.group(0)).strip(" .,-")
- return ""
- def extract_email(text: str) -> str:
- """从文本中提取邮箱"""
- if not text:
- return ""
- pattern = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
- matches = re.findall(pattern, text)
- excluded = ["@facebook.com", "@fb.com", "@example.com"]
- for m in matches:
- if all(e not in m.lower() for e in excluded):
- return m.strip()
- return ""
- def extract_followers(text: str) -> str:
- """提取粉丝数,支持英文、法文、中文(含万/千)"""
- if not text:
- return ""
- # 优先匹配中文“X 万位粉丝 / X 万粉丝”格式
- cn_match = re.search(r'(\d+[\d\s,.]*)\s*([万千]?)\s*(位粉丝|粉丝)', text, re.IGNORECASE)
- if cn_match:
- num = cn_match.group(1).strip()
- unit = cn_match.group(2).strip()
- suffix = cn_match.group(3).strip()
- # 合并单位与后缀,如“万位粉丝”“万粉丝”
- if unit:
- return f"{num}{unit}{suffix}"
- return f"{num} {suffix}"
- # 英文/法文格式:1.2k fans / 3M followers
- en_match = re.search(r'(\d+[\d\s,.]*[KkMm]?)\s*(followers|fans|abonnés|likes)', text, re.IGNORECASE)
- if en_match:
- return f"{en_match.group(1).strip()} {en_match.group(2).strip()}"
- return ""
- def _looks_like_time(text: str) -> bool:
- """判断文本是否像时间戳"""
- text_lower = text.lower()
- time_units = ["h", "hr", "hrs", "hour", "hours", "d", "day", "days", "w", "week",
- "m", "min", "mins", "month", "months", "y", "year", "years",
- "昨天", "今天", "刚刚", "分钟", "小时", "天", "周", "月", "年"]
- has_unit = any(unit in text_lower for unit in time_units)
- has_number = bool(re.search(r'\d', text))
- has_date_separator = bool(re.search(r'\d{1,4}[/-]\d{1,2}[/-]\d{1,4}', text))
- short_relative = len(text) <= 8 and has_number
- date_like = has_number and (has_unit or has_date_separator or len(text) < 25)
- return (has_unit and has_number) or short_relative or date_like
- def _extract_post_time(element) -> str:
- """尝试从帖子元素中提取发布时间"""
- time_text = ""
- try:
- time_selectors = [
- 'a[href*="posts"] span',
- 'a[href*="posts"]',
- 'span[dir="auto"] a[role="link"] span',
- 'abbr',
- 'span[aria-label]',
- 'a[role="link"] span',
- ]
- for sel in time_selectors:
- time_el = element.query_selector(sel)
- if time_el:
- txt = (time_el.get_attribute("aria-label") or time_el.inner_text() or "").strip()
- if txt and _looks_like_time(txt):
- time_text = txt
- break
- except Exception:
- pass
- return time_text
- def scrape_posts(page, max_posts: int = 8) -> list:
- """
- 滚动主页并提取最近帖子文本及发布时间
- 返回: [{"text": str, "time": str}, ...]
- """
- posts = []
- seen_texts = set()
- start_time = time.time()
- # 回到主页
- current_url = page.url
- if "/about" in current_url:
- home_url = current_url.split("/about")[0] + "/"
- try:
- page.goto(home_url, wait_until="domcontentloaded", timeout=30000)
- except Exception:
- pass
- time.sleep(random.uniform(2, 4))
- # 滚动几次,加载帖子
- for i in range(3):
- # 尝试多种帖子容器选择器
- selectors = [
- 'div[role="article"]',
- 'div[data-ad-preview="message"]',
- '[data-pagelet="ProfileTimeline"] div[role="article"]',
- ]
- for sel in selectors:
- elements = page.query_selector_all(sel)
- for el in elements:
- try:
- # 获取帖子文本:优先从 dir="auto" 的 div 中取
- text_els = el.query_selector_all('div[dir="auto"]')
- text_parts = []
- for tel in text_els:
- t = tel.inner_text().strip()
- if t and len(t) > 5:
- text_parts.append(t)
- text = " ".join(text_parts).strip()
- # 过滤:长度适中、不是导航文字、未重复
- if 20 < len(text) < 800 and text not in seen_texts:
- excluded = ["个人资料", "帖子", "简介", "提及", "好友", "照片", "视频", "Reels", "签到", "展开", "查看更多", "评论", "分享", "点赞"]
- if not any(e in text for e in excluded):
- post_time = _extract_post_time(el)
- posts.append({"text": text, "time": post_time})
- seen_texts.add(text)
- except Exception:
- continue
- if len(posts) >= max_posts:
- break
- # 单页面帖子采集总时间限制 45 秒
- if time.time() - start_time > 45:
- print("帖子采集时间超过 45 秒,提前结束")
- break
- # 滚动
- page.evaluate("window.scrollTo(0, window.scrollY + 800)")
- time.sleep(random.uniform(2, 4))
- return posts[:max_posts]
- def _assess_activity_level(latest_time: str) -> str:
- """根据最近发帖时间评估活跃度"""
- if not latest_time or latest_time == "未识别":
- return "活跃度:未识别"
- text = latest_time.lower()
- # 近期:小时、天、本周
- if any(u in text for u in ["h", "hr", "hour", "hours", "小时", "d", "day", "days", "天", "w", "week", "周"]):
- return "活跃度:高"
- # 较近期:月且 <=3
- month_match = re.search(r"(\d+)\s*(month|months|mo|月)", text)
- if month_match:
- months = int(month_match.group(1))
- if months <= 3:
- return "活跃度:中"
- # 其他具体日期或更久远
- return "活跃度:低或陈旧"
- def analyze_posts(posts: list, about_text: str = "") -> str:
- """
- 综合分析最近帖子内容,输出自然语言备注。
- 包含:主营业务判断、近期动态、关注方向、活跃度评估、最近发帖时间、最新帖子摘要。
- posts: [{"text": str, "time": str}, ...]
- """
- if not posts:
- return ""
- texts = [p["text"] for p in posts]
- all_text = " ".join(texts).lower()
- # 1. 主营业务判断
- business_signals = {
- "汽车诊断/维修": ["diagnostic", "diag", "défaut", "moteur", "voyant", "réparation", "panne", "ordinateur de bord", "valise", "scanner"],
- "二手车交易": ["occasion", "vente", "achat", "vendre", "à vendre", "prix", "km", "kilométrage"],
- "汽车销售/经销商": ["concessionnaire", "neuf", "showroom", "livraison", "commande", "véhicule neuf"],
- "租车服务": ["location", "louer", "rental", "à louer", "jour", "mois"],
- "汽车美容/改装": ["tuning", "jantes", "peinture", "covering", "cleaning", "detailing"],
- }
- business_scores = {}
- for biz, keywords in business_signals.items():
- score = sum(all_text.count(kw) for kw in keywords)
- if score > 0:
- business_scores[biz] = score
- main_business = ""
- if business_scores:
- main_business = max(business_scores, key=business_scores.get)
- # 2. 近期动态
- activity_signals = {
- "促销活动": ["promo", "promotion", "offre", "réduction", "discount", "prix spécial", "vente flash"],
- "服务展示": ["service", "intervention", "réparation", "diagnostic", "résultat", "avant/après"],
- "客户案例": ["client", "témoignage", "satisfait", "merci", "avis"],
- "日常内容": ["bonjour", "bonne journée", "week-end", "maroc", "casablanca"],
- }
- activity_scores = {}
- for act, keywords in activity_signals.items():
- score = sum(all_text.count(kw) for kw in keywords)
- if score > 0:
- activity_scores[act] = score
- recent_activities = sorted(activity_scores, key=activity_scores.get, reverse=True)[:2]
- # 3. 关注方向 / 高频词(翻译为中文)
- focus_map = {
- "diagnostic": "诊断",
- "voiture": "汽车",
- "auto": "汽车",
- "moteur": "发动机",
- "réparation": "维修",
- "occasion": "二手车",
- "prix": "价格",
- "casablanca": "卡萨布兰卡",
- "maroc": "摩洛哥",
- "service": "服务",
- "client": "客户",
- "promo": "促销",
- }
- focus_counts = {}
- for kw, cn in focus_map.items():
- count = all_text.count(kw)
- if count > 0:
- focus_counts[cn] = focus_counts.get(cn, 0) + count
- top_focus = sorted(focus_counts, key=focus_counts.get, reverse=True)[:5]
- # 4. 最近帖子时间 + 摘要
- latest_post = posts[0]
- latest_time = latest_post.get("time", "").strip()
- latest_text = latest_post["text"]
- latest_summary = latest_text[:120] + "..." if len(latest_text) > 120 else latest_text
- activity_level = _assess_activity_level(latest_time)
- # 组装备注
- parts = []
- if main_business:
- parts.append(f"主营业务判断:{main_business}")
- if recent_activities:
- parts.append(f"近期动态:{', '.join(recent_activities)}")
- if top_focus:
- parts.append(f"关注方向:{', '.join(top_focus)}")
- parts.append(f"{activity_level}(最近发帖时间:{latest_time or '未识别'})")
- parts.append(f"最新帖子摘要:{latest_summary}")
- return " | ".join(parts)
- def infer_dealer_type(text: str) -> str:
- """推断客户类型"""
- if not text:
- return "待判断"
- text_lower = text.lower()
- # 个人资料页信号
- personal_signals = [
- "personal information", "informations personnelles", "个人信息",
- "date of birth", "date de naissance", "生日",
- "works at", "travail chez", "études à", "studied at",
- "lives in", "habite à", "vit à", "relationship", "relation",
- "family members", "membres de la famille",
- ]
- if any(k in text_lower for k in personal_signals):
- return "个人用户"
- if any(k in text_lower for k in ["diagnostic", "diag", "diagnostique", "诊断"]):
- return "汽车诊断/维修"
- if any(k in text_lower for k in ["used car", "occasion", "二手车", "occaz"]):
- return "二手车商"
- if any(k in text_lower for k in ["dealer", "concessionnaire", "distributeur", "经销商"]):
- return "经销商"
- if any(k in text_lower for k in ["rental", "location", "租车"]):
- return "租车公司"
- if any(k in text_lower for k in ["repair", "garage", "维修"]):
- return "汽车维修"
- return "待判断"
- def scrape_page_record(
- page,
- page_url: str,
- city: str = "Casablanca",
- country: str = "摩洛哥",
- name: str = "",
- main_business: str = "",
- ) -> Dict[str, Any]:
- """使用已存在的 Playwright page 采集单个 Facebook 主页信息"""
- start_time = time.time()
- record = {
- "客户姓名/公司": "",
- "国家": country,
- "城市": city,
- "客户类型": "",
- "主页/链接": page_url,
- "公司官网": "",
- "联系人": "",
- "职位": "",
- "电话/WhatsApp": "",
- "邮箱": "",
- "主营业务": "",
- "建联状态": "未联系",
- "下次跟进": "",
- "备注": "",
- }
- try:
- # 访问主页
- page.goto(page_url, wait_until="domcontentloaded", timeout=30000)
- time.sleep(random.uniform(3, 5))
- # 先提取主页上的粉丝数
- home_page_text = ""
- try:
- home_page_text = page.locator("body").inner_text(timeout=10000)
- except Exception:
- pass
- followers_from_home = extract_followers(home_page_text)
- website_from_home = extract_company_website_from_facebook(page)
- if website_from_home:
- record["公司官网"] = website_from_home
- print(f"从主页联络资料提取到公司官网: {website_from_home}")
- home_phone = extract_phone(home_page_text)
- if home_phone:
- record["电话/WhatsApp"] = home_phone
- print(f"从主页联络资料提取到电话: {home_phone}")
- home_email = extract_email(home_page_text)
- if home_email:
- record["邮箱"] = home_email
- print(f"从主页联络资料提取到邮箱: {home_email}")
- if followers_from_home:
- print(f"从主页联络资料提取到粉丝数: {followers_from_home}")
- # 如果用户提供了名称,直接使用
- if name:
- record["客户姓名/公司"] = name
- else:
- # 尝试多种方式获取页面名称
- record["客户姓名/公司"] = ""
- name_selectors = [
- '[role="main"] h1',
- '[data-pagelet="ProfileActions"] h1',
- 'h1',
- ]
- for sel in name_selectors:
- try:
- el = page.query_selector(sel)
- if el:
- text = el.inner_text().strip()
- if text and len(text) < 100 and "Facebook" not in text and text not in ["个人资料", "帖子", "简介", "提及"]:
- record["客户姓名/公司"] = text
- break
- except Exception:
- continue
- # 如果 h1 都没拿到,从 title 或 URL 推断
- if not record["客户姓名/公司"]:
- title = page.title()
- name_from_title = title.split("|")[0].strip() if "|" in title else title.replace("Facebook", "").strip()
- if name_from_title and name_from_title not in ["个人资料", "帖子", "简介", "提及", "通知"] and len(name_from_title) < 100:
- record["客户姓名/公司"] = name_from_title
- else:
- from urllib.parse import urlparse
- path = urlparse(page_url).path.strip("/").split("/")[0]
- record["客户姓名/公司"] = path or "Unknown"
- # 主页联络资料读取完成后,再获取 About 信息
- about_url = f"{page_url.rstrip('/')}/about/"
- print(f"访问 About 页: {about_url}")
- page.goto(about_url, wait_until="domcontentloaded", timeout=30000)
- time.sleep(random.uniform(2, 4))
- page_text = ""
- try:
- page_text = page.locator("body").inner_text(timeout=15000)
- except Exception:
- pass
- about_website = extract_company_website_from_facebook(page)
- if about_website:
- if not record.get("公司官网"):
- record["公司官网"] = about_website
- elif about_website not in record["公司官网"]:
- record["公司官网"] = record["公司官网"] + ";" + about_website
- company_website = record.get("公司官网", "").split(";")[0].strip()
- website_result = {"email": "", "phone": "", "text": "", "business_summary": "", "evidence_notes": []}
- if company_website:
- try:
- print(f"深搜公司官网: {company_website}")
- website_result = scrape_public_website(page.context, company_website, max_pages=5)
- except Exception as exc:
- website_result = {"email": "", "phone": "", "text": "", "business_summary": "", "evidence_notes": [f"官网深搜失败:{str(exc)[:80]}"]}
- # 提取电话
- phone = extract_phone(page_text)
- if phone and not record.get("电话/WhatsApp"):
- record["电话/WhatsApp"] = phone
- # 提取邮箱,About 只补充空字段
- email = extract_email(page_text)
- if email and not record.get("邮箱"):
- record["邮箱"] = email
- if not record.get("邮箱") and website_result.get("email"):
- record["邮箱"] = website_result.get("email", "")
- if not record.get("电话/WhatsApp") and website_result.get("phone"):
- record["电话/WhatsApp"] = website_result.get("phone", "")
- # 如果用户提供了主营业务,直接使用
- if main_business:
- record["主营业务"] = main_business
- else:
- # 提取主营业务/描述
- description = ""
- lines = [line.strip() for line in page_text.split("\n") if line.strip()]
- # 优先找 "个人资料" 下方的描述句
- profile_index = -1
- for i, line in enumerate(lines):
- if line in ["个人资料", "About", "À propos", "简介"]:
- profile_index = i
- break
- if profile_index >= 0:
- for line in lines[profile_index+1:profile_index+10]:
- if len(line) > 10 and len(line) < 300:
- description = line
- break
- # 如果没找到,找包含关键词的完整描述句
- if not description:
- keywords = ["car", "auto", "voiture", "vehicle", "motor", "dealer", "occasion", "automotive", "diagnostic", "diag", "concessionnaire", "vente", "location"]
- best_line = ""
- for line in lines[:80]:
- lower_line = line.lower()
- if any(k in lower_line for k in keywords):
- if len(line) > len(best_line) and len(line) < 300:
- best_line = line
- if best_line:
- description = best_line
- # 兜底:用公司名称
- if not description:
- description = record["客户姓名/公司"]
- record["主营业务"] = translate_to_chinese(description)
- # 推断客户类型,官网深搜证据优先参与判断
- combined_business_text = "\n".join([home_page_text, page_text, website_result.get("text", "")])
- inferred_type = classify_customer_type_cn(combined_business_text)
- legacy_type = infer_dealer_type(page_text)
- record["客户类型"] = legacy_type if legacy_type == "个人用户" else inferred_type
- if not main_business:
- record["主营业务"] = summarize_business_cn(combined_business_text)
- # 备注:粉丝数 + 帖子分析
- # 优先使用主页提取的粉丝数,若主页没有则尝试 About 页
- followers = followers_from_home or extract_followers(page_text)
- post_analysis = ""
- try:
- print("正在浏览最近帖子并分析...")
- posts = scrape_posts(page, max_posts=8)
- post_analysis = analyze_posts(posts, about_text=page_text)
- print(f"采集到 {len(posts)} 条帖子")
- except Exception as e:
- print(f"帖子分析失败: {e}")
- translated_post_analysis = translate_to_chinese(post_analysis) if post_analysis else ""
- # 先把帖子分析写入备注,便于品牌检测提取上下文
- if translated_post_analysis:
- record["备注"] = translated_post_analysis
- # 品牌检测辅助
- record = enrich_record_with_brands(record)
- # 组装中文结构化备注:Facebook证据 + 官网证据 + 业务判断 + 联系方式证据
- detected = record.get("detected_brands", [])
- exclusivity = record.get("exclusivity_assessment", "")
- record["备注"] = build_chinese_notes(
- facebook_url=page_url,
- facebook_text="\n".join([home_page_text, page_text]),
- website_url=record.get("公司官网", ""),
- website_result=website_result,
- followers=followers,
- post_analysis=translated_post_analysis,
- detected_brands=detected if isinstance(detected, list) else [],
- exclusivity=exclusivity,
- )
- if record.get("备注"):
- record["备注"] = "证据采集顺序:主页联络资料 -> About -> 官网深搜 | " + record["备注"]
- print("采集完成:")
- for k, v in record.items():
- if v:
- print(f" {k}: {v}")
- except Exception as e:
- elapsed = time.time() - start_time
- print(f"采集过程异常({elapsed:.1f}s): {e}")
- if not record["客户姓名/公司"]:
- from urllib.parse import urlparse
- path = urlparse(page_url).path.strip("/").split("/")[0]
- record["客户姓名/公司"] = path or "Unknown"
- record["备注"] = (record.get("备注", "") + f" | 采集异常: {str(e)[:80]}").strip(" | ")
- return record
- def scrape_facebook_page(
- page_url: str,
- profile_id: str,
- city: str = "Casablanca",
- country: str = "摩洛哥",
- name: str = "",
- main_business: str = "",
- ads_power_url: str = "http://127.0.0.1:50325",
- ) -> Dict[str, Any]:
- """采集单个 Facebook 主页完整信息(自动管理 Playwright 生命周期)"""
- ws_endpoint = get_active_ws_endpoint(ads_power_url, profile_id)
- print(f"连接到已打开的浏览器: {ws_endpoint}")
- playwright = sync_playwright().start()
- browser = playwright.chromium.connect_over_cdp(ws_endpoint)
- context = browser.contexts[0] if browser.contexts else browser.new_context()
- page = context.new_page()
- print(f"新建标签页,访问: {page_url}")
- try:
- record = scrape_page_record(
- page=page,
- page_url=page_url,
- city=city,
- country=country,
- name=name,
- main_business=main_business,
- )
- finally:
- print("停止 Playwright,浏览器窗口保持打开")
- playwright.stop()
- return record
- def main():
- parser = argparse.ArgumentParser(description="单个 Facebook 主页完整采集并写入 Excel")
- parser.add_argument("--url", required=True, help="Facebook 主页 URL")
- parser.add_argument("--profile-id", required=True, help="AdsPower profile ID")
- parser.add_argument("--city", default="摩洛哥全国", help="城市或覆盖范围")
- parser.add_argument("--country", default="摩洛哥", help="国家")
- parser.add_argument("--name", default="", help="公司名称(脚本识别不准时手动指定)")
- parser.add_argument("--main-business", default="", help="主营业务(脚本识别不准时手动指定)")
- parser.add_argument("--ads-power-url", default="http://127.0.0.1:50325", help="AdsPower API URL")
- parser.add_argument("--excel", default="", help="建联表路径;不传时按项目优先级自动查找")
- parser.add_argument("--sheet", default="Facebook", help="Sheet 名")
- parser.add_argument("--write-excel", action="store_true", help="确认写入建联表;否则只输出 JSON 预览")
- parser.add_argument("--run-id", default="", help="Run ID used for runs/YYYYMMDD/<run_id>/ artifacts.")
- parser.add_argument("--output", default="single_page_scraped.json", help="JSON 输出")
- parser.add_argument("--no-excel", action="store_true", help="不写入 Excel,只输出 JSON")
- args = parser.parse_args()
- record = scrape_facebook_page(
- page_url=args.url,
- profile_id=args.profile_id,
- city=args.city,
- country=args.country,
- name=args.name,
- main_business=args.main_business,
- ads_power_url=args.ads_power_url,
- )
- # 保存 JSON
- import json
- output_path = resolve_artifact_path(args.output, kind="single_page_scrape", default_name=Path(args.output).name, run_id=args.run_id or None)
- output_path.parent.mkdir(parents=True, exist_ok=True)
- with open(output_path, "w", encoding="utf-8") as f:
- json.dump(record, f, ensure_ascii=False, indent=2)
- print(f"已保存 JSON: {args.output}")
- # 写入 Excel
- if args.write_excel and not args.no_excel:
- workbook_info = resolve_workbook_path(args.excel, create_from_template=True)
- args.excel = str(workbook_info["path"])
- print(f"Workbook for writing: {args.excel} ({workbook_info['source']})")
- result = append_records(
- excel_path=args.excel,
- sheet_name=args.sheet,
- records=[record],
- dedup_keys=["客户姓名/公司", "城市", "主页/链接"]
- )
- print(f"Excel write result: {result}")
- else:
- print("默认预览模式:未写入 Excel。确认要入表时再使用 --write-excel。")
- if __name__ == "__main__":
- main()
|