Initial commit: 商品售卖网站
This commit is contained in:
@@ -0,0 +1,181 @@
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
|
||||
BASE_URL = "https://ribenyan.com"
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,ja;q=0.7",
|
||||
}
|
||||
|
||||
def fetch_page(url):
|
||||
try:
|
||||
response = requests.get(url, headers=HEADERS, timeout=30)
|
||||
response.encoding = 'utf-8'
|
||||
return response.text
|
||||
except Exception as e:
|
||||
print(f"Error fetching {url}: {e}")
|
||||
return None
|
||||
|
||||
def get_category_links(html):
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
categories = []
|
||||
|
||||
accordion = soup.find('div', id='goodstypeaccordion')
|
||||
if accordion:
|
||||
links = accordion.find_all('a', class_='list-group-item')
|
||||
for link in links:
|
||||
href = link.get('href', '')
|
||||
text = link.get_text(strip=True)
|
||||
if href and 'm=goods&a=list' in href:
|
||||
if href.startswith('./'):
|
||||
href = BASE_URL + '/' + href[2:]
|
||||
elif href.startswith('index.php'):
|
||||
href = BASE_URL + '/' + href
|
||||
elif not href.startswith('http'):
|
||||
href = BASE_URL + '/' + href
|
||||
categories.append({'name': text, 'url': href})
|
||||
|
||||
return categories
|
||||
|
||||
def parse_products(html):
|
||||
products = []
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
rows = soup.find_all('div', class_='d-flex')
|
||||
|
||||
product_rows = []
|
||||
for row in rows:
|
||||
classes = row.get('class', [])
|
||||
if 'py-2' in classes and 'border-bottom' in classes:
|
||||
product_rows.append(row)
|
||||
|
||||
for row in product_rows:
|
||||
try:
|
||||
product = {}
|
||||
|
||||
id_div = row.find('div', class_='position-absolute')
|
||||
if id_div and 'top-0' in id_div.get('class', []) and 'end-0' in id_div.get('class', []):
|
||||
product['external_id'] = id_div.get_text(strip=True)
|
||||
|
||||
img = row.find('img', class_='img-thumbnail')
|
||||
if img:
|
||||
src = img.get('src', '')
|
||||
if src:
|
||||
if src.startswith('./'):
|
||||
src = BASE_URL + '/' + src[2:]
|
||||
elif not src.startswith('http'):
|
||||
src = BASE_URL + src
|
||||
product['image_url'] = src
|
||||
|
||||
p_tags = row.find_all('p')
|
||||
name_found = False
|
||||
for p in p_tags:
|
||||
classes = p.get('class', [])
|
||||
if 'mb-1' in classes and 'text-muted' not in classes and 'text-body-tertiary' not in classes and 'text-info' not in classes:
|
||||
if not name_found:
|
||||
product['name_cn'] = p.get_text(strip=True)
|
||||
name_found = True
|
||||
elif 'mb-1' in classes and 'text-muted' in classes:
|
||||
product['name_jp'] = p.get_text(strip=True)
|
||||
elif 'text-body-tertiary' in classes or 'text-info' in classes:
|
||||
text = p.get_text(strip=True)
|
||||
if text and '日元' not in text and '整大包' not in text and '限购' not in text:
|
||||
product['description'] = text
|
||||
|
||||
if not product.get('description'):
|
||||
product['description'] = ''
|
||||
|
||||
price_p = row.find('p', class_='mb-3')
|
||||
if price_p:
|
||||
price_text = price_p.get_text(strip=True)
|
||||
price_match = re.search(r'(\d+)', price_text)
|
||||
if price_match:
|
||||
product['price_jpy'] = int(price_match.group(1))
|
||||
|
||||
if product.get('name_cn') or product.get('name_jp'):
|
||||
products.append(product)
|
||||
|
||||
except Exception as e:
|
||||
continue
|
||||
|
||||
return products
|
||||
|
||||
def categorize_product(product, category_name=None):
|
||||
if category_name:
|
||||
return category_name
|
||||
|
||||
name = product.get('name_cn', '') + ' ' + product.get('name_jp', '')
|
||||
name_lower = name.lower()
|
||||
|
||||
if 'iqos' in name_lower or 'terea' in name_lower or 'iluma' in name_lower or 'sentia' in name_lower or 'virto' in name_lower or 'neo' in name_lower:
|
||||
return 'IQOS烟弹'
|
||||
elif 'シャグ' in name or 'shag' in name_lower or '手卷' in name or '烟丝' in name:
|
||||
return '手卷烟丝'
|
||||
elif 'シガー' in name or 'cigar' in name_lower or '雪茄' in name:
|
||||
return '雪茄'
|
||||
elif 'スーパースリム' in name or 'super slim' in name_lower or '超细' in name or '细支' in name:
|
||||
return '超细烟'
|
||||
elif 'メンソール' in name or 'menthol' in name_lower or '薄荷' in name or '爆珠' in name:
|
||||
return '薄荷烟'
|
||||
else:
|
||||
return '卷烟'
|
||||
|
||||
def main():
|
||||
print("开始采集 ribenyan.com 所有商品数据...")
|
||||
|
||||
html = fetch_page(BASE_URL)
|
||||
if not html:
|
||||
print("无法获取首页数据")
|
||||
return
|
||||
|
||||
categories = get_category_links(html)
|
||||
print(f"发现 {len(categories)} 个分类")
|
||||
|
||||
all_products = []
|
||||
seen_ids = set()
|
||||
|
||||
for i, cat in enumerate(categories):
|
||||
print(f"\n[{i+1}/{len(categories)}] 采集分类: {cat['name']}")
|
||||
|
||||
cat_html = fetch_page(cat['url'])
|
||||
if not cat_html:
|
||||
continue
|
||||
|
||||
products = parse_products(cat_html)
|
||||
|
||||
for p in products:
|
||||
p['category'] = categorize_product(p, cat['name'])
|
||||
p['price_cny'] = round(p.get('price_jpy', 0) * 0.048, 2)
|
||||
|
||||
pid = p.get('external_id')
|
||||
if pid and pid not in seen_ids:
|
||||
seen_ids.add(pid)
|
||||
all_products.append(p)
|
||||
|
||||
print(f" 获取 {len(products)} 个商品，累计 {len(all_products)} 个")
|
||||
time.sleep(0.5)
|
||||
|
||||
output_file = 'ribenyan_products.json'
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_products, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"\n采集完成，共获取 {len(all_products)} 个商品")
|
||||
print(f"数据已保存到 {output_file}")
|
||||
|
||||
cat_stats = {}
|
||||
for p in all_products:
|
||||
cat = p.get('category', '其他')
|
||||
if cat not in cat_stats:
|
||||
cat_stats[cat] = 0
|
||||
cat_stats[cat] += 1
|
||||
|
||||
print("\n分类统计:")
|
||||
for cat, count in sorted(cat_stats.items(), key=lambda x: -x[1]):
|
||||
print(f" {cat}: {count} 个")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user