package crawler import ( "fmt" "log" "net/http" "strings" "time" "sale/internal/models" "github.com/PuerkitoBio/goquery" "gorm.io/gorm" ) type RibenyanCrawler struct { db *gorm.DB } func NewRibenyanCrawler(db *gorm.DB) *RibenyanCrawler { return &RibenyanCrawler{db: db} } func (c *RibenyanCrawler) CrawlArticles() error { log.Println("开始采集 ribenyan.com 文章...") // 访问首页获取文章列表 resp, err := http.Get("https://ribenyan.com") if err != nil { return fmt.Errorf("请求失败:%v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("响应状态码:%d", resp.StatusCode) } doc, err := goquery.NewDocumentFromReader(resp.Body) if err != nil { return fmt.Errorf("解析 HTML 失败:%v", err) } // 查找文章列表(根据实际网站结构调整选择器) articles := make([]models.Article, 0) doc.Find("article, .post, .article, .news-item").Each(func(i int, s *goquery.Selection) { if i >= 20 { // 最多采集 20 篇 return } // 获取标题 title := "" s.Find("h1, h2, h3, .post-title, .article-title").First().Each(func(j int, ts *goquery.Selection) { title = strings.TrimSpace(ts.Text()) }) if title == "" { return } // 获取链接(暂时不使用) s.Find("a").First().Each(func(j int, ls *goquery.Selection) { _, exists := ls.Attr("href") _ = exists }) // 获取封面图 coverImage := "" s.Find("img, .post-thumbnail, .article-cover").First().Each(func(j int, imgs *goquery.Selection) { src, exists := imgs.Attr("src") if exists { if strings.HasPrefix(src, "http") { coverImage = src } else { coverImage = "https://ribenyan.com" + src } } }) // 获取摘要 summary := "" s.Find(".post-excerpt, .article-excerpt, .summary, p").First().Each(func(j int, ps *goquery.Selection) { summary = strings.TrimSpace(ps.Text()) if len(summary) > 500 { summary = summary[:500] + "..." } }) // 获取轮播图(如果有) carousel := "" s.Find(".carousel, .slider, .gallery").First().Each(func(j int, cs *goquery.Selection) { // 收集轮播图中的图片 var images []string cs.Find("img").Each(func(k int, imgs *goquery.Selection) { src, exists := imgs.Attr("src") if exists { if strings.HasPrefix(src, "http") { images = append(images, src) } else { images = append(images, "https://ribenyan.com"+src) } } }) if len(images) > 0 { carousel = strings.Join(images, ",") } }) article := models.Article{ Title: title, Summary: summary, CoverImage: coverImage, Carousel: carousel, IsPublished: true, IsPinned: i == 0, // 第一篇置顶 SortOrder: i, CreatedAt: time.Now(), UpdatedAt: time.Now(), } articles = append(articles, article) }) // 批量插入数据库 if len(articles) > 0 { log.Printf("准备插入 %d 篇文章", len(articles)) // 检查是否已存在,避免重复 var existingIDs []uint c.db.Model(&models.Article{}).Where("title IN ?", getTitles(articles)).Pluck("id", &existingIDs) // 过滤掉已存在的文章 newArticles := make([]models.Article, 0) for _, article := range articles { exists := false for _, id := range existingIDs { if article.ID == id { exists = true break } } if !exists { newArticles = append(newArticles, article) } } if len(newArticles) > 0 { if err := c.db.Create(&newArticles).Error; err != nil { return fmt.Errorf("插入失败:%v", err) } log.Printf("成功插入 %d 篇文章", len(newArticles)) } else { log.Println("没有新文章需要插入") } } else { log.Println("未找到文章") } return nil } func getTitles(articles []models.Article) []string { titles := make([]string, len(articles)) for i, article := range articles { titles[i] = article.Title } return titles } // 单个文章详情采集(可选) func (c *RibenyanCrawler) CrawlArticleDetail(url string) (*models.Article, error) { resp, err := http.Get(url) if err != nil { return nil, fmt.Errorf("请求失败:%v", err) } defer resp.Body.Close() doc, err := goquery.NewDocumentFromReader(resp.Body) if err != nil { return nil, fmt.Errorf("解析 HTML 失败:%v", err) } article := &models.Article{ IsPublished: true, CreatedAt: time.Now(), UpdatedAt: time.Now(), } // 获取标题 doc.Find("h1, .post-title, .article-title").First().Each(func(i int, s *goquery.Selection) { article.Title = strings.TrimSpace(s.Text()) }) // 获取内容 var contentBuilder strings.Builder doc.Find("article, .post-content, .article-content, .entry-content").First().Each(func(i int, s *goquery.Selection) { html, _ := s.Html() contentBuilder.WriteString(html) }) article.Content = contentBuilder.String() // 获取封面图 doc.Find("img, .post-thumbnail").First().Each(func(i int, s *goquery.Selection) { src, _ := s.Attr("src") if strings.HasPrefix(src, "http") { article.CoverImage = src } else { article.CoverImage = "https://ribenyan.com" + src } }) // 获取摘要 doc.Find(".post-excerpt, .article-excerpt, .summary").First().Each(func(i int, s *goquery.Selection) { article.Summary = strings.TrimSpace(s.Text()) if len(article.Summary) > 500 { article.Summary = article.Summary[:500] + "..." } }) return article, nil }