wuling-overseas-dealer-expansion-skill/scripts/common/deduplicator.py

"""
去重工具模块
"""
from typing import List, Dict, Any


def deduplicate_records(
    records: List[Dict[str, Any]],
    keys: List[str] = None
) -> List[Dict[str, Any]]:
    """
    根据指定 key 去重
    默认 key: 客户姓名/公司 + 城市 + 主页/链接
    """
    if keys is None:
        keys = ["客户姓名/公司", "城市", "主页/链接"]

    seen = set()
    unique_records = []

    for record in records:
        key_values = []
        for key in keys:
            value = str(record.get(key, "")).strip().lower()
            key_values.append(value)
        signature = "|".join(key_values)

        if signature and signature not in seen:
            seen.add(signature)
            unique_records.append(record)
        elif not signature:
            # 空签名也保留，避免误删
            unique_records.append(record)

    return unique_records


def merge_records(
    existing: List[Dict[str, Any]],
    new: List[Dict[str, Any]],
    keys: List[str] = None
) -> List[Dict[str, Any]]:
    """合并两组记录并去重"""
    combined = existing + new
    return deduplicate_records(combined, keys)