import requests
from bs4 import BeautifulSoup
import json
import re
import time

BASE_URL = "https://ribenyan.com"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,ja;q=0.7",
}

def fetch_page(url):
    try:
        response = requests.get(url, headers=HEADERS, timeout=30)
        response.encoding = 'utf-8'
        return response.text
    except Exception as e:
        print(f"Error fetching {url}: {e}")
        return None

def get_category_links(html):
    soup = BeautifulSoup(html, 'html.parser')
    categories = []
    
    accordion = soup.find('div', id='goodstypeaccordion')
    if accordion:
        links = accordion.find_all('a', class_='list-group-item')
        for link in links:
            href = link.get('href', '')
            text = link.get_text(strip=True)
            if href and 'm=goods&a=list' in href:
                if href.startswith('./'):
                    href = BASE_URL + '/' + href[2:]
                elif href.startswith('index.php'):
                    href = BASE_URL + '/' + href
                elif not href.startswith('http'):
                    href = BASE_URL + '/' + href
                categories.append({'name': text, 'url': href})
    
    return categories

def parse_products(html):
    products = []
    soup = BeautifulSoup(html, 'html.parser')
    
    rows = soup.find_all('div', class_='d-flex')
    
    product_rows = []
    for row in rows:
        classes = row.get('class', [])
        if 'py-2' in classes and 'border-bottom' in classes:
            product_rows.append(row)
    
    for row in product_rows:
        try:
            product = {}
            
            id_div = row.find('div', class_='position-absolute')
            if id_div and 'top-0' in id_div.get('class', []) and 'end-0' in id_div.get('class', []):
                product['external_id'] = id_div.get_text(strip=True)
            
            img = row.find('img', class_='img-thumbnail')
            if img:
                src = img.get('src', '')
                if src:
                    if src.startswith('./'):
                        src = BASE_URL + '/' + src[2:]
                    elif not src.startswith('http'):
                        src = BASE_URL + src
                    product['image_url'] = src
            
            p_tags = row.find_all('p')
            name_found = False
            for p in p_tags:
                classes = p.get('class', [])
                if 'mb-1' in classes and 'text-muted' not in classes and 'text-body-tertiary' not in classes and 'text-info' not in classes:
                    if not name_found:
                        product['name_cn'] = p.get_text(strip=True)
                        name_found = True
                elif 'mb-1' in classes and 'text-muted' in classes:
                    product['name_jp'] = p.get_text(strip=True)
                elif 'text-body-tertiary' in classes or 'text-info' in classes:
                    text = p.get_text(strip=True)
                    if text and '日元' not in text and '整大包' not in text and '限购' not in text:
                        product['description'] = text
            
            if not product.get('description'):
                product['description'] = ''
            
            price_p = row.find('p', class_='mb-3')
            if price_p:
                price_text = price_p.get_text(strip=True)
                price_match = re.search(r'(\d+)', price_text)
                if price_match:
                    product['price_jpy'] = int(price_match.group(1))
            
            if product.get('name_cn') or product.get('name_jp'):
                products.append(product)
            
        except Exception as e:
            continue
    
    return products

def categorize_product(product, category_name=None):
    if category_name:
        return category_name
    
    name = product.get('name_cn', '') + ' ' + product.get('name_jp', '')
    name_lower = name.lower()
    
    if 'iqos' in name_lower or 'terea' in name_lower or 'iluma' in name_lower or 'sentia' in name_lower or 'virto' in name_lower or 'neo' in name_lower:
        return 'IQOS烟弹'
    elif 'シャグ' in name or 'shag' in name_lower or '手卷' in name or '烟丝' in name:
        return '手卷烟丝'
    elif 'シガー' in name or 'cigar' in name_lower or '雪茄' in name:
        return '雪茄'
    elif 'スーパースリム' in name or 'super slim' in name_lower or '超细' in name or '细支' in name:
        return '超细烟'
    elif 'メンソール' in name or 'menthol' in name_lower or '薄荷' in name or '爆珠' in name:
        return '薄荷烟'
    else:
        return '卷烟'

def main():
    print("开始采集 ribenyan.com 所有商品数据...")
    
    html = fetch_page(BASE_URL)
    if not html:
        print("无法获取首页数据")
        return
    
    categories = get_category_links(html)
    print(f"发现 {len(categories)} 个分类")
    
    all_products = []
    seen_ids = set()
    
    for i, cat in enumerate(categories):
        print(f"\n[{i+1}/{len(categories)}] 采集分类: {cat['name']}")
        
        cat_html = fetch_page(cat['url'])
        if not cat_html:
            continue
        
        products = parse_products(cat_html)
        
        for p in products:
            p['category'] = categorize_product(p, cat['name'])
            p['price_cny'] = round(p.get('price_jpy', 0) * 0.048, 2)
            
            pid = p.get('external_id')
            if pid and pid not in seen_ids:
                seen_ids.add(pid)
                all_products.append(p)
        
        print(f"  获取 {len(products)} 个商品，累计 {len(all_products)} 个")
        time.sleep(0.5)
    
    output_file = 'ribenyan_products.json'
    with open(output_file, 'w', encoding='utf-8') as f:
        json.dump(all_products, f, ensure_ascii=False, indent=2)
    
    print(f"\n采集完成，共获取 {len(all_products)} 个商品")
    print(f"数据已保存到 {output_file}")
    
    cat_stats = {}
    for p in all_products:
        cat = p.get('category', '其他')
        if cat not in cat_stats:
            cat_stats[cat] = 0
        cat_stats[cat] += 1
    
    print("\n分类统计:")
    for cat, count in sorted(cat_stats.items(), key=lambda x: -x[1]):
        print(f"  {cat}: {count} 个")

if __name__ == "__main__":
    main()