182 lines
6.5 KiB
Python
182 lines
6.5 KiB
Python
import requests
|
|
from bs4 import BeautifulSoup
|
|
import json
|
|
import re
|
|
import time
|
|
|
|
BASE_URL = "https://ribenyan.com"
|
|
HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,ja;q=0.7",
|
|
}
|
|
|
|
def fetch_page(url):
|
|
try:
|
|
response = requests.get(url, headers=HEADERS, timeout=30)
|
|
response.encoding = 'utf-8'
|
|
return response.text
|
|
except Exception as e:
|
|
print(f"Error fetching {url}: {e}")
|
|
return None
|
|
|
|
def get_category_links(html):
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
categories = []
|
|
|
|
accordion = soup.find('div', id='goodstypeaccordion')
|
|
if accordion:
|
|
links = accordion.find_all('a', class_='list-group-item')
|
|
for link in links:
|
|
href = link.get('href', '')
|
|
text = link.get_text(strip=True)
|
|
if href and 'm=goods&a=list' in href:
|
|
if href.startswith('./'):
|
|
href = BASE_URL + '/' + href[2:]
|
|
elif href.startswith('index.php'):
|
|
href = BASE_URL + '/' + href
|
|
elif not href.startswith('http'):
|
|
href = BASE_URL + '/' + href
|
|
categories.append({'name': text, 'url': href})
|
|
|
|
return categories
|
|
|
|
def parse_products(html):
|
|
products = []
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
|
|
rows = soup.find_all('div', class_='d-flex')
|
|
|
|
product_rows = []
|
|
for row in rows:
|
|
classes = row.get('class', [])
|
|
if 'py-2' in classes and 'border-bottom' in classes:
|
|
product_rows.append(row)
|
|
|
|
for row in product_rows:
|
|
try:
|
|
product = {}
|
|
|
|
id_div = row.find('div', class_='position-absolute')
|
|
if id_div and 'top-0' in id_div.get('class', []) and 'end-0' in id_div.get('class', []):
|
|
product['external_id'] = id_div.get_text(strip=True)
|
|
|
|
img = row.find('img', class_='img-thumbnail')
|
|
if img:
|
|
src = img.get('src', '')
|
|
if src:
|
|
if src.startswith('./'):
|
|
src = BASE_URL + '/' + src[2:]
|
|
elif not src.startswith('http'):
|
|
src = BASE_URL + src
|
|
product['image_url'] = src
|
|
|
|
p_tags = row.find_all('p')
|
|
name_found = False
|
|
for p in p_tags:
|
|
classes = p.get('class', [])
|
|
if 'mb-1' in classes and 'text-muted' not in classes and 'text-body-tertiary' not in classes and 'text-info' not in classes:
|
|
if not name_found:
|
|
product['name_cn'] = p.get_text(strip=True)
|
|
name_found = True
|
|
elif 'mb-1' in classes and 'text-muted' in classes:
|
|
product['name_jp'] = p.get_text(strip=True)
|
|
elif 'text-body-tertiary' in classes or 'text-info' in classes:
|
|
text = p.get_text(strip=True)
|
|
if text and '日元' not in text and '整大包' not in text and '限购' not in text:
|
|
product['description'] = text
|
|
|
|
if not product.get('description'):
|
|
product['description'] = ''
|
|
|
|
price_p = row.find('p', class_='mb-3')
|
|
if price_p:
|
|
price_text = price_p.get_text(strip=True)
|
|
price_match = re.search(r'(\d+)', price_text)
|
|
if price_match:
|
|
product['price_jpy'] = int(price_match.group(1))
|
|
|
|
if product.get('name_cn') or product.get('name_jp'):
|
|
products.append(product)
|
|
|
|
except Exception as e:
|
|
continue
|
|
|
|
return products
|
|
|
|
def categorize_product(product, category_name=None):
|
|
if category_name:
|
|
return category_name
|
|
|
|
name = product.get('name_cn', '') + ' ' + product.get('name_jp', '')
|
|
name_lower = name.lower()
|
|
|
|
if 'iqos' in name_lower or 'terea' in name_lower or 'iluma' in name_lower or 'sentia' in name_lower or 'virto' in name_lower or 'neo' in name_lower:
|
|
return 'IQOS烟弹'
|
|
elif 'シャグ' in name or 'shag' in name_lower or '手卷' in name or '烟丝' in name:
|
|
return '手卷烟丝'
|
|
elif 'シガー' in name or 'cigar' in name_lower or '雪茄' in name:
|
|
return '雪茄'
|
|
elif 'スーパースリム' in name or 'super slim' in name_lower or '超细' in name or '细支' in name:
|
|
return '超细烟'
|
|
elif 'メンソール' in name or 'menthol' in name_lower or '薄荷' in name or '爆珠' in name:
|
|
return '薄荷烟'
|
|
else:
|
|
return '卷烟'
|
|
|
|
def main():
|
|
print("开始采集 ribenyan.com 所有商品数据...")
|
|
|
|
html = fetch_page(BASE_URL)
|
|
if not html:
|
|
print("无法获取首页数据")
|
|
return
|
|
|
|
categories = get_category_links(html)
|
|
print(f"发现 {len(categories)} 个分类")
|
|
|
|
all_products = []
|
|
seen_ids = set()
|
|
|
|
for i, cat in enumerate(categories):
|
|
print(f"\n[{i+1}/{len(categories)}] 采集分类: {cat['name']}")
|
|
|
|
cat_html = fetch_page(cat['url'])
|
|
if not cat_html:
|
|
continue
|
|
|
|
products = parse_products(cat_html)
|
|
|
|
for p in products:
|
|
p['category'] = categorize_product(p, cat['name'])
|
|
p['price_cny'] = round(p.get('price_jpy', 0) * 0.048, 2)
|
|
|
|
pid = p.get('external_id')
|
|
if pid and pid not in seen_ids:
|
|
seen_ids.add(pid)
|
|
all_products.append(p)
|
|
|
|
print(f" 获取 {len(products)} 个商品，累计 {len(all_products)} 个")
|
|
time.sleep(0.5)
|
|
|
|
output_file = 'ribenyan_products.json'
|
|
with open(output_file, 'w', encoding='utf-8') as f:
|
|
json.dump(all_products, f, ensure_ascii=False, indent=2)
|
|
|
|
print(f"\n采集完成，共获取 {len(all_products)} 个商品")
|
|
print(f"数据已保存到 {output_file}")
|
|
|
|
cat_stats = {}
|
|
for p in all_products:
|
|
cat = p.get('category', '其他')
|
|
if cat not in cat_stats:
|
|
cat_stats[cat] = 0
|
|
cat_stats[cat] += 1
|
|
|
|
print("\n分类统计:")
|
|
for cat, count in sorted(cat_stats.items(), key=lambda x: -x[1]):
|
|
print(f" {cat}: {count} 个")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|