"""
pip install nodriver --upgrade
pip install curl-cffi --upgrade
pip install ai-cloudscraper --upgrade
pip install never-primp --upgrade
pip install pydoll-python --upgrade
pip install flaresolverr-cli --upgrade
pip install primp --upgrade
pip install scrapling --upgrade
pip install playwright-stealth

"""


import asyncio
import nodriver as uc
from lxml import etree
import re
import requests
from lxml import html
import json
from urllib.parse import urlparse
from urllib.parse import urlencode
import json5
import cloudscraper
from bs4 import BeautifulSoup
import ast




async def curl_cffi_main():
    from curl_cffi.requests import AsyncSession
    """ 下载失败
    from curl_cffi.requests import BrowserType
    # 过滤出名称中包含 'chrome' 的枚举值
    chrome_versions = [b.name for b in BrowserType if 'chrome' in b.name.lower()]
    print("当前 curl_cffi 支持的 Chrome 版本0:")
    for version in chrome_versions:
        print(f"  - {version}")
    :return:
    """
    url = "https://www.amazon.com/Cordless-Cleaners-Motorized-Touchscreen-Self-Standing/dp/B0G64ZJ5MQ"
    async with AsyncSession(impersonate="chrome146") as session:
        response = await session.get(url, timeout=30)
        print(f"状态码: {response.status_code}")
        print(f"内容长度: {len(response.text)} 字符")
        print(f"前800字符预览: {response.text}")


def scrapling_main():
    r""" 安装繁琐
    https://chenxutan.com/d/3538.html
    1. 基础安装
     pip install scrapling
    2. 安装浏览器自动化依赖（Playwright Chromium）
    pip install "scrapling[fetchers]"
    scrapling install  # 自动下载 Chromium、Camoufox 反指纹套件

    3. Docker 用户
    docker pull d4vinci/scrapling:latest

    pip install patchright -i https://pypi.tuna.tsinghua.edu.cn/simple
    pip install msgspec -i https://pypi.tuna.tsinghua.edu.cn/simple
    pip install browserforge -i https://pypi.tuna.tsinghua.edu.cn/simple
    patchright install --force chrome  # 下载 Patchright 专用的 Chrome 浏览器二进制文件（一个独立的、可被自动化控制的版本）。
    C:\Users\YHCX\AppData\Local\ms-playwright\winldd-1007
    :return:
    """
    from scrapling.fetchers import StealthyFetcher
    # 开启自适应模式
    StealthyFetcher.adaptive = True
    url = "https://www.amazon.com/Cordless-Cleaners-Motorized-Touchscreen-Self-Standing/dp/B0G64ZJ5MQ"
    page = StealthyFetcher.fetch(url,
        headless=False,  # 先用有头模式测试，成功后再改 True
        network_idle=True,
        timeout=60,  # Amazon 建议适当加长超时
        # 可选参数，增加稳定性
        # viewport={"width": 1920, "height": 1080},
        # wait_for_timeout=5000,
    )

    if page and page.status_code == 200:
        print(f"✅ 抓取成功！状态码: {page.status_code}")
        print(f"页面标题: {page.title}")
        print(page.html[:1000])  # 打印前1000字符查看内容
    else:
        print(f"❌ 抓取失败。状态码: {page.status_code if page else 'None'}")


def build_universal_url(original_url, domain="www.amazon.com", link_type="dp"):
    """ 长链接变成短链接
    :param original_url:
    :param domain:
    :param link_type:
    :return:
    original_url = "https://www.amazon.com/Cordless-Cleaners-Motorized-Touchscreen-Self-Standing/dp/B0G64ZJ5MQ"
    universal_url = build_universal_url(original_url)
    print(universal_url) # 输出: https://www.amazon.com/dp/B0G64ZJ5MQ
    """
    import re
    # 提取 ASIN (不区分大小写)
    asin_match = re.search(r'/dp/([A-Z0-9]{10})', original_url, re.IGNORECASE)
    if asin_match:
        asin = asin_match.group(1)
        if link_type == "gp":
            return f"https://{domain}/gp/product/{asin}"
        else: # 默认返回 dp 链接
            return f"https://{domain}/dp/{asin}"
    else:
        raise ValueError("Invalid Amazon URL: ASIN not found")




def extract_asin(url: str) -> str or None:
    """
    从亚马逊产品 URL 中提取 ASIN（标准 10 位字母数字码）
    支持的 URL 格式：
    - https://www.amazon.com/dp/B00EXAMPLE
    - https://www.amazon.com/Product-Name/dp/B00EXAMPLE/ref=...
    - https://www.amazon.co.uk/gp/product/B00EXAMPLE
    - https://www.amazon.de/product/B00EXAMPLE
    - https://www.amazon.com/gp/offer-listing/B00EXAMPLE
    - https://www.amazon.com/exec/obidos/asin/B00EXAMPLE
    - https://smile.amazon.com/dp/B00EXAMPLE
    - 查询参数: ?asin=B00EXAMPLE 或 ?ASIN=B00EXAMPLE
    参数:
        url (str): 亚马逊商品链接
    返回:
        str or None: 匹配到的 ASIN（大写），未找到则返回 None
    """
    # 统一转为字符串，去除首尾空白
    url = url.strip()
    # 1. 优先匹配路径模式（常见且精确）
    # 模式包括：/dp/, /gp/product/, /product/, /exec/obidos/asin/, /gp/offer-listing/
    # 匹配规则：路径后跟 10 个字母数字（大小写均可），后面遇到非字母数字停止
    # 使用非捕获分组，忽略大小写
    patterns = [
        r'/dp/([A-Z0-9]{10})',                # /dp/ASIN
        r'/gp/product/([A-Z0-9]{10})',        # /gp/product/ASIN
        r'/product/([A-Z0-9]{10})',           # /product/ASIN
        r'/exec/obidos/asin/([A-Z0-9]{10})',  # 老式链接
        r'/gp/offer-listing/([A-Z0-9]{10})',  # 优惠列表页
        r'/gp/aw/d/([A-Z0-9]{10})',           # 移动端
        r'/dp/([A-Z0-9]{10})(?=[/?]|$)',      # 更精准的结束
    ]
    for pattern in patterns:
        match = re.search(pattern, url, re.IGNORECASE)
        if match:
            asin = match.group(1).upper()
            # 额外校验长度是否为 10（确保不是截断）
            if len(asin) == 10:
                return asin

    # 2. 尝试匹配通用的 /dp/ 后跟任意 10 个字母数字（包括可能的小写）
    # 这个方法作为后备，但上面的模式已经覆盖
    match = re.search(r'/dp/([A-Z0-9]{10})', url, re.IGNORECASE)
    if match:
        asin = match.group(1).upper()
        if len(asin) == 10:
            return asin

    # 3. 尝试查询参数 asin= 或 ASIN=
    match = re.search(r'[?&]asin=([A-Z0-9]{10})', url, re.IGNORECASE)
    if match:
        asin = match.group(1).upper()
        if len(asin) == 10:
            return asin

    # 4. 尝试从任意位置匹配 10 个大写字母数字（限定边界）
    # 这种方法风险较高，可能误匹配，仅在以上方法失效时使用
    # 并且要求在前后不是字母数字（防止匹配到其他码）
    match = re.search(r'(?<![A-Z0-9])([A-Z0-9]{10})(?![A-Z0-9])', url, re.IGNORECASE)
    if match:
        asin = match.group(1).upper()
        if len(asin) == 10:
            return asin
    return None



def get_headers():
    import random
    from browserforge.headers import HeaderGenerator
    from browserforge.headers import Browser
    # generate_headers = HeaderGenerator()
    # headers = generate_headers.generate()
    # print(headers)

    browsers = [
        Browser(name='chrome', min_version=135, max_version=148),
        Browser(name='firefox', min_version=144),
        Browser(name='edge', max_version=140, http_version=1),
    ]
    random.shuffle(browsers)
    headers = HeaderGenerator(browser=browsers)
    return headers.generate()


def save_html(response, path='./amazon.html'):
    if len(response) < 20000:
        print(f'请求失败--{len(response)}字符')
        return
    with open(path, 'w', encoding='utf-8') as f:
        f.write(response)



def parse_with_ast(text: str) -> dict:
    """
    使用 ast.literal_eval 解析，需要先清理成 Python 字面量
    """
    try:
        # 清理步骤
        cleaned = text.strip()
        # 1. 去掉 JS 的 + 字符串拼接
        cleaned = re.sub(r'"\s*\+\s*"', '', cleaned)
        cleaned = re.sub(r"'\s*\+\s*'", '', cleaned)
        # 2. 处理可能的尾随逗号
        cleaned = re.sub(r',(\s*[}\]])', r'\1', cleaned)
        # 3. 把 true/false/null 转成 Python 的 True/False/None
        cleaned = cleaned.replace('true', 'True')
        cleaned = cleaned.replace('false', 'False')
        cleaned = cleaned.replace('null', 'None')
        # 4. 使用 ast 安全解析
        data = ast.literal_eval(cleaned)
        print("✅ ast 解析成功！")
        return data
    except Exception as e:
        print("❌ ast 解析失败:", e)
        # 打印出错位置附近内容
        pos = getattr(e, 'pos', None) or getattr(e, 'offset', 100)
        print("附近内容:", cleaned[max(0, pos - 200):pos + 200])
        raise


def jsonrepair(text):
    import json_repair
    try:
        # 清理步骤
        cleaned = text.strip()
        # 1. 去掉 JS 的 + 字符串拼接
        cleaned = re.sub(r'"\s*\+\s*"', '', cleaned)
        cleaned = re.sub(r"'\s*\+\s*'", '', cleaned)
        # 2. 处理可能的尾随逗号
        cleaned = re.sub(r',(\s*[}\]])', r'\1', cleaned)
        # 3. 把 true/false/null 转成 Python 的 True/False/None
        cleaned = cleaned.replace('true', 'True')
        cleaned = cleaned.replace('false', 'False')
        cleaned = cleaned.replace('null', 'None')
        # 4. 使用 ast 安全解析
        data = json_repair.loads(cleaned)
        print("✅ json_repair 解析成功！")
        return data
    except Exception as e:
        print("❌ json_repair 解析失败:", e)
        # 打印出错位置附近内容
        pos = getattr(e, 'pos', None) or getattr(e, 'offset', 100)
        print("附近内容:", cleaned[max(0, pos - 200):pos + 200])
        raise


def fastjsonrepair(text):
    import fast_json_repair
    try:
        # 清理步骤
        cleaned = text.strip()
        # 1. 去掉 JS 的 + 字符串拼接
        cleaned = re.sub(r'"\s*\+\s*"', '', cleaned)
        cleaned = re.sub(r"'\s*\+\s*'", '', cleaned)
        # 2. 处理可能的尾随逗号
        cleaned = re.sub(r',(\s*[}\]])', r'\1', cleaned)
        # 3. 把 true/false/null 转成 Python 的 True/False/None
        cleaned = cleaned.replace('true', 'True')
        cleaned = cleaned.replace('false', 'False')
        cleaned = cleaned.replace('null', 'None')
        # 4. 使用 ast 安全解析
        data = fast_json_repair.loads(cleaned)
        print("✅ json_repair 解析成功！")
        return data
    except Exception as e:
        print("❌ json_repair 解析失败:", e)
        # 打印出错位置附近内容
        pos = getattr(e, 'pos', None) or getattr(e, 'offset', 100)
        print("附近内容:", cleaned[max(0, pos - 200):pos + 200])
        raise


def parse_with_json5(text: str) -> dict:
    """
    使用 json5 解析宽松的 JS 对象字符串
    """
    try:
        # 清理步骤
        cleaned = text.strip()
        # 1. 去掉 JS 的 + 字符串拼接
        cleaned = re.sub(r'"\s*\+\s*"', '', cleaned)
        cleaned = re.sub(r"'\s*\+\s*'", '', cleaned)
        # 2. 处理可能的尾随逗号
        cleaned = re.sub(r',(\s*[}\]])', r'\1', cleaned)
        # 3. 把 true/false/null 转成 Python 的 True/False/None
        cleaned = cleaned.replace('true', 'True')
        cleaned = cleaned.replace('false', 'False')
        cleaned = cleaned.replace('null', 'None')
        # json5 能直接处理大部分 JS 风格的对象
        data = json5.loads(cleaned)
        print("✅ json5 解析成功！")
        return data
    except Exception as e:
        print("❌ json5 解析失败:", e)
        return {}


def extract_data(path='./amazon2.html', url=''):
    if not url:
        url = "https://www.amazon.com/Cordless-Cleaners-Motorized-Touchscreen-Self-Standing/dp/B0G64ZJ5MQ"

    with open(path, 'r', encoding='utf-8') as f:
        html_text = f.read()

    # html = etree.HTML(html_text)

    data = {}
    # 当前 ASIN
    asin_match = re.search(r'/dp/([A-Z0-9]{10})', url)
    current_asin = asin_match.group(1) if asin_match else None
    data['current_asin'] = current_asin

    # ==================== 提取 dataToReturn ====================
    # 更宽松的匹配（支持多行和大对象）
    pattern = r'var\s+dataToReturn\s*=\s*(\{[\s\S]*?\}\s*);'
    match = re.search(pattern, html_text, re.DOTALL)

    if match:
        json_str = match.group(1)
        # data_to_return = clean_js_to_json(json_str)
        data_to_return0 = parse_with_json5(json_str)
        data_to_return1 = parse_with_ast(json_str)
        data_to_return2 = jsonrepair(json_str)
        data_to_return3 = fastjsonrepair(json_str)
        data_to_return = data_to_return1 or data_to_return2 or data_to_return3 or data_to_return0
        print("提取成功！parentAsin =", data_to_return.get("parentAsin"))
        print("currentAsin =", data_to_return.get("currentAsin"))
        print("dimensionToAsinMap =", data_to_return.get("dimensionToAsinMap"))
        # print("data_to_return:", data_to_return)
        # data['dataToReturn'] = data_to_return  # 保留完整数据供后续使用
        # ==================== 2. 从 dataToReturn 中提取关键字段 ====================
        # parentAsin
        data['parentAsin'] = data_to_return.get('parentAsin') or data_to_return.get('asin')
    else:
        data_to_return = False

    if not data.get('parentAsin'):
        data['parentAsin'] = re.findall(r'"parentAsin":\s?"([\w]+)",', html_text)[0]

    # variations
    # 所有子 ASIN（变体）
    if data_to_return:
        child_asins = []
        dim_map = data_to_return.get('dimensionToAsinMap', {})
        if isinstance(dim_map, dict):
            child_asins.extend(dim_map.values())
        # 其他可能的 asin 列表
        for key in ['asinList', 'childAsins', 'variations']:
            if key in data_to_return:
                val = data_to_return[key]
                if isinstance(val, list):
                    child_asins.extend(val)
                elif isinstance(val, str):
                    child_asins.extend(re.findall(r'[A-Z0-9]{10}', val))
        data['child_asins'] = list(set([a for a in child_asins if re.match(r'^[A-Z0-9]{10}$', a)])) or [current_asin]

    if not data.get('child_asins'):
        child_asins = re.findall(r'"asinsInCollapsedView":\s?\[(.*?)\]}', html_text)
        child_asins1 = re.findall(r'"dimensionToAsinMap"\s?:\s?\{(.*?)\},', html_text)
        if child_asins1:
            child_asins = list(json5.loads('{'+child_asins1[0]+'}').values())
        print(f'child_asins:{child_asins} child_asins1:{child_asins1}')

    tree = html.fromstring(html_text)
    # data['child_asins'] = tree.xpath('//ul[contains(@class, "a-unordered-list")]/li/@data-asin')

    # ==================== 3. 其他字段（保留原有 + 增强） ====================
    global_ratings0 = tree.xpath('//span[@data-hook="total-review-count"]//text()')
    global_ratings1 = tree.xpath('//div[@id="centerCol"]//a[@id="acrCustomerReviewLink"]/span[@id="acrCustomerReviewText"]/@aria-label')
    global_ratings = global_ratings0 or global_ratings1
    global_ratings = ''.join(global_ratings).replace('global ratings', '').replace('\n', '').strip()
    print(f'global_ratings:{global_ratings} global_ratings1:{global_ratings1}')
    data['global_ratings'] = global_ratings

    # star
    histogramTable = tree.xpath('//ul[@id="histogramTable"]/li')
    star_dict = {}
    for li in histogramTable:
        text0 = li.xpath('.//a[contains(@class, "a-size-base a-link-normal")]/@aria-label')
        text1 = li.xpath('.//span[contains(@class, "a-size-base")]/@aria-label')
        text = text0 or text1
        percent = text[0].split('percent of reviews have')[0].strip()
        stars = text[0].split('percent of reviews have')[1].replace('stars', '').strip()
        star_dict[stars] = percent
    print(f'star_dict: {star_dict}')
    data['star_dict'] = star_dict

    star = tree.xpath('//div[@id="centerCol"]//div[@id="averageCustomerReviews"]//span[@id="acrPopover"]/@title')
    data['star'] = star

    # 标题
    title = (tree.xpath('//span[@id="productTitle"]/text()') or
             tree.xpath('//h1[contains(@class,"product-title")]/text()') or
             tree.xpath('//title/text()'))
    data['title'] = title[0].strip() if title else None

    # Brand
    brand = tree.xpath('//table[contains(@class, "a-normal a-spacing")]//tr[@class="a-spacing-small po-brand"]/td/span[contains(text(),"Brand")]/following::td[1]//text()')
    data['brand'] = ' '.join(b.strip() for b in brand if b.strip()) or None
    print(f'brand:{''.join(brand).strip()}')
    
    # BSR 排名 Best Sellers Rank
    bsr = tree.xpath('//table[contains(@class, "prodDetTable")]//tr/th[contains(@class, "prodDetSectionEntry") and contains(text(), "Best Sellers Rank")]/following::td[1]//text()')
    data['BSR'] = ' '.join(b.strip() for b in bsr if b.strip())

    # 价格相关 corePriceDisplay_desktop_feature_div
    # list_price0 = tree.xpath('//p[@id="pqv-price-list-price"]//text()')
    list_price1 = tree.xpath('//div[@class="offersConsistencyEnabled"]//div[contains(@id, "apex_desktop") and not(contains(@style, "display:none"))]//div[@id="corePriceDisplay_desktop_feature_div"]//div[@class="a-section a-spacing-small aok-align-center"]//span[contains(@class, "aok-offscreen")]//text()')
    list_price2 = tree.xpath('//div[contains(@id, "apex_desktop") and not(contains(@style, "display:none"))]//div[@id="corePriceDisplay_desktop_feature_div"]//div[@class="a-section a-spacing-small aok-align-center"]//span[contains(@class, "aok-offscreen")]//text()')
    # 原价
    list_price3 = tree.xpath('//div[contains(@id,"apex_desktop_prime")]//div[contains(@class, "apex-core-price-identifier")]/span[@id="apex-pricetopay-accessibility-label"]//text()')
    list_price = list_price1 or list_price2 or list_price3

    # 折扣价格  # offersConsistencyEnabled 标签可能不存在
    savings0 = tree.xpath('//div[@id="apex_desktop"]//div[@id="corePriceDisplay_desktop_feature_div"]//span[@id="apex-pricetopay-accessibility-label"]//text()')
    savings = tree.xpath('//div[@id="centerCol"]//div[contains(@id, "apex_desktop_prime") and not(contains(@style, "display:none"))]//div[@id="corePriceDisplay_desktop_feature_div"]//span[@id="apex-pricetopay-accessibility-label"]//text()')

    savings1 = savings or savings0
    savings1 = [i for i in savings1 if i.replace('\n', '').strip()]
    print(f'savings1:{savings1}')
    pqv_price = savings1[0].split('with')[0].strip()
    savings = savings1[0].split('with')[1].strip() if 'with' in savings1[0] else None
    coupon = tree.xpath('//p[@id="pqv-price-coupon-message"]/text()')
    print(f'list_price: {list_price} pqv_price: {pqv_price} savings:{savings} coupon:{coupon}')
    data['list_price'] = ''.join([i.replace('List Price:', '').strip() for i in list_price if i.strip()])
    data['pqv_price'] = savings1[0].split('with')[0].strip()
    data['savings'] = savings
    data['coupon'] = ''.join([i.strip() for i in coupon if i.strip()])

    # Prime Member Price
    PrimeMemberPrice = tree.xpath('//div[@id="rightCol"]//div[@data-csa-c-buying-option-type="PRIME_SAVINGS_UPSELL"]//div[contains(@class, "core-price-identifier")]/span[contains(@class, "apex-pricetopay-value")]/span[contains(@class, "a-offscreen")]/text()')
    print(f'PrimeMemberPrice:{PrimeMemberPrice}')
    PrimeMemberPrice = ''.join([i.strip() for i in PrimeMemberPrice if i.strip()])
    data['PrimeMemberPrice'] = PrimeMemberPrice

    # 折扣 / Limited time deal
    # //div[contains(@id, "apex_desktop_prime") and not(contains(@style, "display:none"))]//div[contains(@id, "promoPrice")]//div[contains(@id, "reinvent_price_desktop_prime") and not(contains(@style, "display:none"))]//span[@class="promoPriceBlockMessage"]//div[contains(@class, "a-alert-content")]/text()
    checkout0 = tree.xpath('//div[(contains(@id, "reinvent_price_desktop") or contains(@id, "apex_desktop")) and not(contains(@style, "display:none"))]//span[@data-csa-c-owner="PromotionsDiscovery"]//div[contains(@class, "a-alert-content")]/text()')
    checkout1 = tree.xpath('//div[contains(@id, "apex_desktop_prime") and not(contains(@style, "display:none"))]//div[contains(@id, "promoPrice")]//div[contains(@id, "reinvent_price_desktop_prime") and not(contains(@style, "display:none"))]//span[@class="promoPriceBlockMessage"]//div[contains(@class, "a-alert-content")]/text()')
    checkout = checkout1 or checkout0
    deal = tree.xpath('//div[contains(@id, "apex_desktop") and not(contains(@style, "display:none"))]//span[@id="dealBadgeSupportingText"]/span//text()')


    # promoPriceBlockMessage_feature_div // maplePriceblockAmabot_feature_div
    # //div[contains(@id, "apex_desktop_prime") and not(contains(@style, "display:none"))]//div[contains(@id, "reinvent_price_desktop_prime") and not(contains(@style, "display:none"))]//span[@class="promoPriceBlockMessage"]//span[@data-csa-c-owner="PromotionsDiscovery"]
    promotions0 = tree.xpath('//div[@id="apex_desktop"]//div[(contains(@id, "reinvent_price_desktop_prime") or @data-csa-c-slot-id="apex_dp_center_column") and not(contains(@style, "display:none"))]//span[@class="promoPriceBlockMessage"]//span[@data-csa-c-owner="PromotionsDiscovery"]')
    promotions1 = tree.xpath('//div[contains(@id, "apex_desktop_prime") and not(contains(@style, "display:none"))]//div[contains(@id, "promoPrice")]//div[(contains(@id, "reinvent_price_desktop_prime") or @data-csa-c-slot-id="apex_dp_center_column") and not(contains(@style, "display:none"))]//span[@class="promoPriceBlockMessage"]//span[@data-csa-c-owner="PromotionsDiscovery"]')
    promotions = promotions1 or promotions0

    Promotion = []
    promotion_dic = {}
    for promotion in promotions:
        label = promotion.xpath('./label[contains(@id, "greenBadgepctch")]/text()')
        label = [i.strip() for i in label if i.strip()]
        if not label:
            continue
        span_text = promotion.xpath('./span[contains(@id, "promoMessagepctch")]/text()')
        span_text = [i.strip() for i in span_text if i.strip()]
        print(f'label_span_text: {label+span_text}')
        label_span_text = ' '.join(label+span_text)
        if promotion_dic.get(label_span_text):
            continue
        else:
            promotion_dic[label_span_text] = 1
        Promotion.append(label_span_text)

    print(f'checkout:{checkout} deal:{deal}')
    data['checkout_code'] = ''.join(set([i.strip() for i in checkout if i.strip()])).replace('   ', ' ')
    data['deal'] = ''.join(set([i.strip() for i in deal if i.strip()]))
    data['Promotion'] = [i.strip() for i in Promotion if i.strip()]

    # Ships from / Sold by
    ships_from0 = tree.xpath('//div[@id="fulfillerInfoFeature_feature_div"]//div[@offer-display-feature-name="desktop-fulfiller-info"]//span[contains(text(),"Ships from")]/following::div[1][@offer-display-feature-name="desktop-fulfiller-info"]//text()')
    ships_from1 = tree.xpath('//div[contains(@id, "newAccordionRow")]//div[contains(@id, "shipFromSoldBy")]/div[@id="sfsb_accordion_head"]//span[contains(text(), "Ships from")]/following::span[1]/text()')
    ships_from = [i.replace('Ships from', '').replace('\n', '').strip() for i in ships_from0+ships_from1 if i.replace('Ships from', '').replace('\n', '').strip()]
    sold_by0 = tree.xpath('//div[@id="merchantInfoFeature_feature_div"]//div[@offer-display-feature-name="desktop-merchant-info"]//span[contains(text(), "Sold by")]/following::div[1]//a[@id="sellerProfileTriggerId"]//text()')
    sold_by1 = tree.xpath('//div[contains(@id, "newAccordionRow")]//div[contains(@id, "shipFromSoldBy")]/div[@id="sfsb_accordion_head"]//span[contains(text(), "Sold by")]/following::span[1]/text()')
    sold_by = [i.replace('Sold by', '').replace('\n', '').strip() for i in sold_by0+sold_by1 if i.replace('Sold by', '').replace('\n', '').strip()]

    # Deliver to
    deliver_to = tree.xpath(
        '//div[@id="nav-global-location-slot"]//a[contains(@id,"nav-global-location") or contains(text(),"Deliver to")]//span//text()')
    deliver_to = ''.join([i.replace('Deliver to', '').replace('\n', '').strip() for i in deliver_to if i.replace('Deliver to', '').replace('\n', '').strip()])
    data['ships_from'] = ''.join(set(ships_from))
    data['sold_by'] = ''.join(set(sold_by))
    data['deliver_to'] = deliver_to.replace('\u200c', '')
    print(f'ships_from:{ships_from} sold_by:{sold_by} deliver_to:{deliver_to}')


    # print(data)
    # Bought in past month
    bought_in_past_month = tree.xpath('//div[@id="socialProofingAsinFaceout_feature_div"]//span[@id="social-proofing-faceout-title-tk_bought"]//text()')
    data['bought_in_past_month'] = ' '.join([i.strip() for i in bought_in_past_month if i.strip()])

    # 橱窗图
    data_json = re.findall(r"var data\s?=\s?(\{[\s\S]*?\});\s*return", html_text)
    if data_json:
        data_json = fastjsonrepair(data_json[0])
        print(f'data_json:{data_json}')
        images = [i['hiRes'] for i in data_json['colorImages']['initial']]
        print('images:', images)
    else:
        images = tree.xpath('//div[@id="main-image-container"]//ul[contains(@class, "desktop-media-mainView")]/li//div[contains(@class, "imgTagWrapper")]/*/@data-old-hires')
        print('images:', images)
    data['images'] = images

    # parseJSON = re.findall(r"jQuery\.parseJSON\('(.*?)'\)", html_text)
    # if parseJSON:
    #     parseJSON = fastjsonrepair(parseJSON[0])
    #     videos = parseJSON['videos']

    # From the brand
    Fromthebrand = tree.xpath('//div[contains(@class, "aplus-brand-story-hero")]//div[@class="apm-brand-story-background-image"]//img/@data-src')
    print('Fromthebrand:', Fromthebrand)

    # Product description
    print('Product description')
    div_list = tree.xpath('//div[@id="aplus_feature_div"]//div[@id="aplus"]//div[contains(@class, "desktop celwidget")]/div[contains(@class, "aplus-content-wrapper")]//div[contains(@class, "aplus-premium")]')
    for div in div_list:
        class_name = div.xpath('./@class')[0]
        print('class_name:', class_name)
        h1_title = div.xpath('.//h1/text()')
        if 'u16 flex vacuum cleaner' in ''.join(h1_title):
            print('error:', etree.tostring(div))
        print(f'h1_title:{h1_title}')
        if 'video aplus-premium' in class_name:
            productdescription = tree.xpath(
                '//div[contains(@class, "hero-video")]//div[contains(@class, "video-container")]//div[@data-csa-c-component="aplus-vse-video-widget"]/script[@type="a-state"]/text()')
            if productdescription:
                productdescription = fastjsonrepair(productdescription[0])
                print(f'productdescription:{productdescription}')
                videoUrl = productdescription['videoUrl']
                print(f'videoUrl:{videoUrl}')
        elif 'video-text aplus-premium' in class_name:
            video_text = div.xpath('.//div[contains(@class, "text-panel-container")]//text()')
            video_text = [i.replace('\n', '').strip() for i in video_text if i.replace('\n', '').strip()]
            print(f'video_text:{video_text}')
            video_json = div.xpath('.//div[@data-csa-c-component="aplus-vse-video-widget"]/script/text()')
            if video_json:
                video_json = fastjsonrepair(video_json[0])
                print(f'video_json:{video_json}')
                videoUrl1 = video_json['videoUrl']
                print(f'videoUrl1:{videoUrl1}')
        elif 'images aplus-premium' in class_name:
            text_list = div.xpath('.//div[contains(@class, "premium-aplus-column")]')
            for i in text_list:
                text_1 = i.xpath('.//text()')
                img = i.xpath('.//div[contains(@class, "column-image")]/img/@data-src')
                text_1 = [l.replace('\n', '').strip() for l in text_1 if l.replace('\n', '').strip()]
                print(f'text_1: {text_1}')
                print(f'img: {img}')
        else:
            image_url0 = div.xpath('.//div[contains(@class, "background-image") or contains(@class, "aplus-card-image")]/img/@src')
            image_url2 = div.xpath('.//div[contains(@class, "background-image") or contains(@class, "aplus-card-image")]/img/@data-src')
            image_url = [u for u in image_url0+image_url2 if not u.endswith('.gif')]
            print(f'image_url:{image_url}')
        print('*'*100)

    print('data:', data)
    return data





def cloudscraper_main(url):
    """ 可以成功请求
    :param url:
    :return:
    """
    import cloudscraper
    # 创建爬虫实例（开启 turbo_mode 提升性能）
    scraper = cloudscraper.create_scraper(
        browser={'browser': 'chrome', 'platform': 'windows', 'mobile': False},
        turbo_mode=True
    )

    # ==================== 1. 先设置邮编 ====================
    zip_code = "10001"
    set_headers = {
        "Accept-Language": "en-US,en;q=0.9",
        "Accept-Encoding": "gzip, deflate, br",
        "Content-Type": "application/x-www-form-urlencoded",
        "x-requested-with": "XMLHttpRequest",
        "Referer": "https://www.amazon.com/",
        "Origin": "https://www.amazon.com",
    }

    set_payload = {
        "locationType": "LOCATION_INPUT",
        "zipCode": zip_code,
        "storeContext": "generic",      # 商品页常用 generic
        "deviceType": "web",
        "pageType": "Detail",           # 商品详情页用 Detail，首页可改成 Gateway
        "actionSource": "glow",
    }

    print("正在设置邮编...")
    email_url = "https://www.amazon.com/gp/delivery/ajax/address-change.html"
    encoded_data = urlencode(set_payload)
    resp_set = scraper.post(email_url, data=encoded_data, headers=set_headers, timeout=90)
    print(f"设置邮编状态码: {resp_set.status_code}")
    if resp_set.status_code == 200:
        print("✅ 邮编设置成功（cookies 已更新）")
    else:
        print("❌ 设置失败:", resp_set.text[:300])

    # ==================== 2. 再访问商品页面 ====================
    # url = "https://www.amazon.com/Cordless-Cleaners-Motorized-Touchscreen-Self-Standing/dp/B0G64ZJ5MQ"
    # url = 'https://www.amazon.com/Ultenic-180%C2%B0Bendable-Anti-Tangle-U16-Flex/dp/B0CZNTRLJB'
    # url = 'https://www.amazon.com/Electric-Mason-Vacuum-Sealer-Regular/dp/B0BDDX27WT'
    # url = 'https://www.amazon.com/Cordless-Cleaners-Foldable-AUTO-Mode-Fragrance/dp/B0FT2LTJRF'
    # url = 'https://www.amazon.com/SelePow-VP06-Electric-Cordless-Handheld/dp/B0GC3GZD2D'

    print("\n正在访问商品页面...")
    response = scraper.get(url, timeout=120)

    print(f"请求状态码: {response.status_code}")
    print(f"内容长度: {len(response.text)} 字符")

    # 简单提取标题
    if '<title>' in response.text:
        title = response.text.split('<title>')[1].split('</title>')[0].strip()
        print(f"页面标题: {title}")
    else:
        print("未找到标题")

    # 可选：检查页面中是否显示对应邮编
    if f"Deliver to" in response.text or zip_code in response.text:
        print(f"✅ 页面中可能已显示 {zip_code} 的配送信息")

    asin = extract_asin(url)
    save_html(response.content.decode(), f'./{asin}.html')
    extract_data(path=f'./{asin}.html', url=url)



async def nodriver_main(url):
    """ 可以成功请求
    :param url:
    :return:
    """
    # 浏览器参数 自动化采集
    browser_args = ['--no-first-run', '--window-size=1020,1080', '--disable-infobars',
                    '--disable-blink-features=AutomationControlled']
    browser = await uc.start(
        browser_args=browser_args,
        headless=False
    )

    page = await browser.get("https://www.amazon.com/")
    await asyncio.sleep(5)

    zip_code = "10001"  # ← 修改为你想要的邮编
    # ==================== 设置邮编 ====================
    try:
        print("正在打开位置选择弹窗...")
        # 点击顶部 Deliver to / 位置按钮（最常用选择器）
        await asyncio.sleep(5)
        for i in range(35):
            result = await page.evaluate(
            '''{var temp = document.querySelector("a#nav-global-location-popover-link");if(temp){temp.click()}else{"0000"}}''',
            await_promise = True)
            result1 = await page.evaluate(
                '''{var temp = document.querySelector("#nav-global-location-data-modal-action");if(temp){temp.click()}else{"0000"}}''',
                await_promise=True)
            aria_hidden = await page.evaluate(
                '''{var temp = document.querySelector('div[data-action="a-popover-floating-close"]>div[data-action="a-popover-a11y"]');if(temp){temp.getAttribute('aria-hidden')}else{"0000"}}''',
                await_promise=True)
            print(f'result:{result} result1:{result1} aria_hidden: {aria_hidden}')
            if aria_hidden == 'false':
                break
            if result != '0000':
                await asyncio.sleep(5)
                if i > 5:
                    break
            await asyncio.sleep(1)

        await asyncio.sleep(2)  # 等待弹窗出现

        # 输入邮编
        input_selector = 'input#GLUXZipUpdateInput, input[placeholder*="zip code" i], input[name="zipCode"]'
        zip_input = await page.select(input_selector)
        if zip_input:
            await zip_input.send_keys(zip_code)
            print(f"已输入邮编: {zip_code}")
        else:
            print("未找到邮编输入框，尝试其他方式...")
        await asyncio.sleep(2)  # 等待弹窗出现

        # 点击 Apply / Continue / Update 按钮
        apply_btn = await page.select('div#GLUXZipInputSection span#GLUXZipUpdate input[class="a-button-input"]')
        await apply_btn.click()
        print("已点击提交按钮")

        # 点击完成
        for i in range(10):
            result2 = await page.evaluate(
            '''{var temp = document.querySelector('div.a-popover-footer span[data-action="GLUXConfirmAction"] input#GLUXConfirmClose');if(temp){temp.click()}else{"0000"}}''',
            await_promise = True)
            print(f'result2: {result2}')
            if result2 != '0000':
                await asyncio.sleep(2)
                break
            await asyncio.sleep(1)
        print(f"✅ 邮编 {zip_code} 设置完成")
    except Exception as e:
        print("设置邮编过程中出错:", e)

    # ==================== 访问目标商品页 ====================

    # url = "https://www.amazon.com/Cordless-Cleaners-Motorized-Touchscreen-Self-Standing/dp/B0G64ZJ5MQ"
    # url = 'https://www.amazon.com/Ultenic-180%C2%B0Bendable-Anti-Tangle-U16-Flex/dp/B0CZNTRLJB'
    # url = 'https://www.amazon.com/Cordless-Cleaners-Motorized-Touchscreen-Self-Standing/dp/B0G64ZJ5MQ'
    # 打开目标商品页面
    page = await browser.get(url)

    # 判断标签是否存在,执行js不用retrun,若存在返回标签的value
    for i in range(20):
        result3 = await page.evaluate(
        '''{var temp = document.querySelector("button.a-button-text");if(temp){temp.click()}else{"0000"}}''',
        await_promise = True)
        print('result3:', result3)
        if result3 != '0000':
            await asyncio.sleep(5)
            break
        await asyncio.sleep(1)

    # 获取页面源码
    html_content = await page.get_content()
    print("页面内容长度:", len(html_content))
    if zip_code in html_content:
        print(f"✅ 页面中检测到邮编 {zip_code}")

    asin = extract_asin(url)
    save_html(html_content, f'./{asin}.html')
    extract_data(path=f'./{asin}.html', url=url)


async def pydoll_main(url=None):
    """ 自动化 获取不稳定
    :return:
    """
    from pydoll.browser import Chrome
    from pydoll.browser.requests import Request
    if not url:
        url = "https://www.amazon.com/Cordless-Cleaners-Motorized-Touchscreen-Self-Standing/dp/B0G64ZJ5MQ"
    async with Chrome() as browser:
        # 启动浏览器并打开一个标签页
        tab = await browser.start(headless = False)
        await tab.go_to(url, timeout=120)

        for i in range(20):
            result = await tab.execute_script(
                '''{var temp = document.querySelector("button.a-button-text");if(temp){temp.click()}else{"0000"}}''',
                await_promise=True)
            print('result:', result)
            if result != '0000':
                await asyncio.sleep(5)
                break
            await asyncio.sleep(1)

        # 等待页面加载完成
        # await tab.wait_for_element("div#dp-container", timeout=15)

        # 获取页面标题
        title = await tab.execute_script("return document.title;")
        print(f"页面标题: {title}")

        # 获取页面HTML
        # html = await tab.execute_script("return document.documentElement.outerHTML;")
        # print(html)
        html = await tab.page_source
        print(html)

        # 或者使用pydoll内置的请求类，继承浏览器会话状态
        # req = Request(tab)
        # json_response = await req.request("GET", "https://www.amazon.com/Cordless-Cleaners-Motorized-Touchscreen-Self-Standing/dp/B0G64ZJ5MQ")
        # print(json_response)  # 打印请求的完整响应对象


def primp_main(url=None):
    """ 不理想
     url = 'https://github.com/deedy5/primp/blob/main/README.md'
    text = requests.get(url).text
    :return:
    """
    import primp
    # 创建客户端，并选择一款较新的Chrome浏览器进行伪装
    client = primp.Client(impersonate="chrome")
    # 添加一些请求头，模拟更真实的浏览器行为
    # headers = get_headers()
    headers = {
        "Accept-Language": "en-US,en;q=0.9",
        "Accept-Encoding": "gzip, deflate, br",
        "Content-Type": "application/x-www-form-urlencoded",  # 重要
        "x-requested-with": "XMLHttpRequest",
        # 可选：添加 Referer 等，更接近真实浏览器
        "Referer": "https://www.amazon.com/",
    }

    set_zip_payload = {
        "locationType": "LOCATION_INPUT",
        "zipCode": "10001",  # 你要设置的邮编
        "storeContext": "generic",  # 或根据页面调整，如 "office-products"
        "deviceType": "web",
        "pageType": "Detail",  # 商品详情页用 Detail，首页可试 Gateway
        "actionSource": "glow",
    }
    resp_set = client.post(
        "https://www.amazon.com/gp/delivery/ajax/address-change.html",
        headers=headers,
        data=set_zip_payload,  # 用 data（form 表单），不要用 json
        timeout=30
    )
    print("设置邮编状态码:", resp_set.status_code)
    if resp_set.status_code == 200:
        print("邮编设置成功（或已生效）")
    else:
        print("响应内容:", resp_set.text[:500])  # 查看错误信息

    if not url:
        # url = "https://www.amazon.com/Cordless-Cleaners-Motorized-Touchscreen-Self-Standing/dp/B0G64ZJ5MQ"
        url = 'https://www.amazon.com/Ultenic-180%C2%B0Bendable-Anti-Tangle-U16-Flex/dp/B0CZNTRLJB'
    resp = client.get(url, headers=headers, timeout=120)
    print(resp.headers)
    print(f"状态码: {resp.status_code}")
    print(f"请求URL: {resp.url}")
    print(f"内容长度: {len(resp.text)} 字符")
    # print(resp.text)
    asin = extract_asin(url)
    save_html(resp.content.decode(), f'./{asin}.html')
    extract_data(path=f'./{asin}.html', url=url)


def neverprimp(url=None):
    """ 可以获取amazon的详情
     url = 'https://github.com/deedy5/primp/blob/main/README.md'
    text = requests.get(url).text
    primp
    最早的版本，能模仿浏览器指纹（headers + TLS/JA3/JA4/HTTP2）
    可用，但可能不是最新优化★★★☆☆
    pp-prim
    加强了指纹模仿（headers + TLS 指纹）
    较好，但功能相对基础★★★★☆
    never-primp
    基于原primp用Rust重构，号称最快的Python HTTP客户端，保留了强大浏览器伪装能力
    最佳选择★★★★★
    """
    import never_primp
    # 创建客户端，并选择一款较新的Chrome浏览器进行伪装
    client = never_primp.Client(impersonate="chrome",
                                impersonate_os="windows",
                                timeout=30.0,
                                )
    # 添加一些请求头，模拟更真实的浏览器行为
    # headers = get_headers()
    headers = {
        "Accept-Language": "en-US,en;q=0.9",
        "Accept-Encoding": "gzip, deflate, br",
        "Content-Type": "application/x-www-form-urlencoded",  # 重要
        "x-requested-with": "XMLHttpRequest",
        # 可选：添加 Referer 等，更接近真实浏览器
        "Referer": "https://www.amazon.com/",
    }

    set_zip_payload = {
        "locationType": "LOCATION_INPUT",
        "zipCode": "10001",  # 你要设置的邮编
        "storeContext": "generic",  # 或根据页面调整，如 "office-products"
        "deviceType": "web",
        "pageType": "Detail",  # 商品详情页用 Detail，首页可试 Gateway
        "actionSource": "glow",
    }
    resp_set = client.post(
        "https://www.amazon.com/gp/delivery/ajax/address-change.html",
        headers=headers,
        data=set_zip_payload,  # 用 data（form 表单），不要用 json
        timeout=60
    )
    print("设置邮编状态码:", resp_set.status_code)
    if resp_set.status_code == 200:
        print("邮编设置成功（或已生效）")
    else:
        print("响应内容:", resp_set.text[:500])  # 查看错误信息

    if not url:
        # url = "https://www.amazon.com/Cordless-Cleaners-Motorized-Touchscreen-Self-Standing/dp/B0G64ZJ5MQ"
        # url = 'https://www.amazon.com/Ultenic-180%C2%B0Bendable-Anti-Tangle-U16-Flex/dp/B0CZNTRLJB'
        # url = 'https://www.amazon.com/dp/B0GGH51B6V'
        # url = 'https://www.amazon.com/SelePow-Electric-Mason-Jar-Vacuum-Sealer-Regular-Indicator-Fermentation/dp/B0CGH244RN'
        # url = 'https://www.amazon.com/SelePow-Electric-Mason-Jar-Vacuum-Sealer-Regular-Indicator-Fermentation/dp/B0CGHJK5J2/ref=sr_1_1?crid=2VFOIK1HI957E&dib=eyJ2IjoiMSJ9.kQXw0WOpJ6j_qmQ7Ey4IAg.oRSE8ODqWoGPwNJIsXfxB-ZW7Vz7dgBYoHTQPmCx-q8&dib_tag=se&keywords=B0CGH244RN&nsdOptOutParam=true&qid=1782106843&sprefix=b0cgh244rn%2Caps%2C871&sr=8-1&th=1'
        url = 'https://www.amazon.com/Cordless-Cleaners-Motorized-Touchscreen-Self-Standing/dp/B0G64ZJ5MQ/ref=sr_1_1?crid=10HQJSCL41YMT&dib=eyJ2IjoiMSJ9.C7--VVpCHkEXFhOQSVWmAw.D29-ZUHo4LzgYE0EwHQbN94xktlwwSWbdkfC_GWhVXE&dib_tag=se&keywords=B0G64ZJ5MQ&nsdOptOutParam=true&qid=1782118730&sprefix=b0g64zj5mq%2Caps%2C679&sr=8-1&th=1'

    resp = client.get(url, headers=headers, timeout=120)
    print(resp.headers)
    print(f"状态码: {resp.status_code}")
    print(f"请求URL: {resp.url}")
    print(f"内容长度: {len(resp.text)} 字符")
    # print(resp.text)
    asin = extract_asin(url)
    save_html(resp.content.decode(), f'./{asin}.html')
    extract_data(path=f'./{asin}.html', url=url)







if __name__ == "__main__":
    # url = 'https://www.amazon.com/Cordless-Cleaners-Motorized-Touchscreen-Self-Standing/dp/B0G64ZJ5MQ'
    # url = 'https://www.amazon.com/Kids-Sunglasses-Bulk-Party-Favors/dp/B0CRGQMFYB'
    # url = 'https://www.amazon.com/SelePow-Electric-Mason-Jar-Vacuum-Sealer-Regular-Indicator-Fermentation/dp/B0CGH244RN'
    # url = 'https://www.amazon.com/Cordless-Cleaners-Foldable-AUTO-Mode-Fragrance/dp/B0FT2LTJRF'
    # url = 'https://www.amazon.com/Electric-Mason-Vacuum-Sealer-Regular/dp/B0BDDX27WT'
    # url = 'https://www.amazon.com/Ultenic-180%C2%B0Bendable-Anti-Tangle-U16-Flex/dp/B0CZNTRLJB'
    # url = 'https://www.amazon.com/SelePow-VP06-Electric-Cordless-Handheld/dp/B0GC3GZD2D'
    # url = 'https://www.amazon.com/Cordless-Cleaners-Upgraded-Extendable-Charging/dp/B0GGH51B6V'
    # url = 'https://www.amazon.com/OLIKER-Sunglasses-24Pack-Protection-Favors/dp/B0BW92YFNL'
    # url = 'https://www.amazon.com/Party-Favors-Kids-Sunglasses-Bulk/dp/B0FQPB6YXQ'
    # url = 'https://www.amazon.com/Froman-Kids-Sunglasses-Bulk-Protection/dp/B0GBXJMVVL'
    url = 'https://www.amazon.com/Cordless-Cleaners-Foldable-AUTO-Mode-Fragrance/dp/B0FT2LTJRF'
    # asyncio.run(curl_cffi_main())
    # asyncio.run(pydoll_main())
    # uc.loop().run_until_complete(nodriver_main())


    cloudscraper_main(url)
    # primp_main(url)
    # neverprimp(url)