sale/backend/internal/crawler/ribenyan.go

package crawler

import (
	"fmt"
	"log"
	"net/http"
	"strings"
	"time"

	"sale/internal/models"

	"github.com/PuerkitoBio/goquery"
	"gorm.io/gorm"
)

type RibenyanCrawler struct {
	db *gorm.DB
}

func NewRibenyanCrawler(db *gorm.DB) *RibenyanCrawler {
	return &RibenyanCrawler{db: db}
}

func (c *RibenyanCrawler) CrawlArticles() error {
	log.Println("开始采集 ribenyan.com 文章...")

	// 访问首页获取文章列表
	resp, err := http.Get("https://ribenyan.com")
	if err != nil {
		return fmt.Errorf("请求失败：%v", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("响应状态码：%d", resp.StatusCode)
	}

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		return fmt.Errorf("解析 HTML 失败：%v", err)
	}

	// 查找文章列表（根据实际网站结构调整选择器）
	articles := make([]models.Article, 0)
	
	doc.Find("article, .post, .article, .news-item").Each(func(i int, s *goquery.Selection) {
		if i >= 20 { // 最多采集 20 篇
			return
		}

		// 获取标题
		title := ""
		s.Find("h1, h2, h3, .post-title, .article-title").First().Each(func(j int, ts *goquery.Selection) {
			title = strings.TrimSpace(ts.Text())
		})

		if title == "" {
			return
		}

		// 获取链接（暂时不使用）
		s.Find("a").First().Each(func(j int, ls *goquery.Selection) {
			_, exists := ls.Attr("href")
			_ = exists
		})

		// 获取封面图
		coverImage := ""
		s.Find("img, .post-thumbnail, .article-cover").First().Each(func(j int, imgs *goquery.Selection) {
			src, exists := imgs.Attr("src")
			if exists {
				if strings.HasPrefix(src, "http") {
					coverImage = src
				} else {
					coverImage = "https://ribenyan.com" + src
				}
			}
		})

		// 获取摘要
		summary := ""
		s.Find(".post-excerpt, .article-excerpt, .summary, p").First().Each(func(j int, ps *goquery.Selection) {
			summary = strings.TrimSpace(ps.Text())
			if len(summary) > 500 {
				summary = summary[:500] + "..."
			}
		})

		// 获取轮播图（如果有）
		carousel := ""
		s.Find(".carousel, .slider, .gallery").First().Each(func(j int, cs *goquery.Selection) {
			// 收集轮播图中的图片
			var images []string
			cs.Find("img").Each(func(k int, imgs *goquery.Selection) {
				src, exists := imgs.Attr("src")
				if exists {
					if strings.HasPrefix(src, "http") {
						images = append(images, src)
					} else {
						images = append(images, "https://ribenyan.com"+src)
					}
				}
			})
			if len(images) > 0 {
				carousel = strings.Join(images, ",")
			}
		})

		article := models.Article{
			Title:       title,
			Summary:     summary,
			CoverImage:  coverImage,
			Carousel:    carousel,
			IsPublished: true,
			IsPinned:    i == 0, // 第一篇置顶
			SortOrder:   i,
			CreatedAt:   time.Now(),
			UpdatedAt:   time.Now(),
		}

		articles = append(articles, article)
	})

	// 批量插入数据库
	if len(articles) > 0 {
		log.Printf("准备插入 %d 篇文章", len(articles))
		
		// 检查是否已存在，避免重复
		var existingIDs []uint
		c.db.Model(&models.Article{}).Where("title IN ?", getTitles(articles)).Pluck("id", &existingIDs)
		
		// 过滤掉已存在的文章
		newArticles := make([]models.Article, 0)
		for _, article := range articles {
			exists := false
			for _, id := range existingIDs {
				if article.ID == id {
					exists = true
					break
				}
			}
			if !exists {
				newArticles = append(newArticles, article)
			}
		}

		if len(newArticles) > 0 {
			if err := c.db.Create(&newArticles).Error; err != nil {
				return fmt.Errorf("插入失败：%v", err)
			}
			log.Printf("成功插入 %d 篇文章", len(newArticles))
		} else {
			log.Println("没有新文章需要插入")
		}
	} else {
		log.Println("未找到文章")
	}

	return nil
}

func getTitles(articles []models.Article) []string {
	titles := make([]string, len(articles))
	for i, article := range articles {
		titles[i] = article.Title
	}
	return titles
}

// 单个文章详情采集（可选）
func (c *RibenyanCrawler) CrawlArticleDetail(url string) (*models.Article, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, fmt.Errorf("请求失败：%v", err)
	}
	defer resp.Body.Close()

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("解析 HTML 失败：%v", err)
	}

	article := &models.Article{
		IsPublished: true,
		CreatedAt:   time.Now(),
		UpdatedAt:   time.Now(),
	}

	// 获取标题
	doc.Find("h1, .post-title, .article-title").First().Each(func(i int, s *goquery.Selection) {
		article.Title = strings.TrimSpace(s.Text())
	})

	// 获取内容
	var contentBuilder strings.Builder
	doc.Find("article, .post-content, .article-content, .entry-content").First().Each(func(i int, s *goquery.Selection) {
		html, _ := s.Html()
		contentBuilder.WriteString(html)
	})
	article.Content = contentBuilder.String()

	// 获取封面图
	doc.Find("img, .post-thumbnail").First().Each(func(i int, s *goquery.Selection) {
		src, _ := s.Attr("src")
		if strings.HasPrefix(src, "http") {
			article.CoverImage = src
		} else {
			article.CoverImage = "https://ribenyan.com" + src
		}
	})

	// 获取摘要
	doc.Find(".post-excerpt, .article-excerpt, .summary").First().Each(func(i int, s *goquery.Selection) {
		article.Summary = strings.TrimSpace(s.Text())
		if len(article.Summary) > 500 {
			article.Summary = article.Summary[:500] + "..."
		}
	})

	return article, nil
}