feat: add carousel support and ribenyan crawler

2026-04-29 08:35:25 +00:00
parent 6ddb8f58ad
commit 2334e29a58
4 changed files with 241 additions and 0 deletions
@@ -0,0 +1,225 @@
+package crawler
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+
+	"sale/internal/models"
+
+	"github.com/PuerkitoBio/goquery"
+	"gorm.io/gorm"
+)
+
+type RibenyanCrawler struct {
+	db *gorm.DB
+}
+
+func NewRibenyanCrawler(db *gorm.DB) *RibenyanCrawler {
+	return &RibenyanCrawler{db: db}
+}
+
+func (c *RibenyanCrawler) CrawlArticles() error {
+	log.Println("开始采集 ribenyan.com 文章...")
+
+	// 访问首页获取文章列表
+	resp, err := http.Get("https://ribenyan.com")
+	if err != nil {
+		return fmt.Errorf("请求失败：%v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("响应状态码：%d", resp.StatusCode)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return fmt.Errorf("解析 HTML 失败：%v", err)
+	}
+
+	// 查找文章列表（根据实际网站结构调整选择器）
+	articles := make([]models.Article, 0)
+	
+	doc.Find("article, .post, .article, .news-item").Each(func(i int, s *goquery.Selection) {
+		if i >= 20 { // 最多采集 20 篇
+			return
+		}
+
+		// 获取标题
+		title := ""
+		s.Find("h1, h2, h3, .post-title, .article-title").First().Each(func(j int, ts *goquery.Selection) {
+			title = strings.TrimSpace(ts.Text())
+		})
+
+		if title == "" {
+			return
+		}
+
+		// 获取链接
+		link := ""
+		s.Find("a").First().Each(func(j int, ls *goquery.Selection) {
+			href, exists := ls.Attr("href")
+			if exists && !strings.HasPrefix(href, "http") {
+				link = "https://ribenyan.com" + href
+			} else if exists {
+				link = href
+			}
+		})
+
+		// 获取封面图
+		coverImage := ""
+		s.Find("img, .post-thumbnail, .article-cover").First().Each(func(j int, imgs *goquery.Selection) {
+			src, exists := imgs.Attr("src")
+			if exists {
+				if strings.HasPrefix(src, "http") {
+					coverImage = src
+				} else {
+					coverImage = "https://ribenyan.com" + src
+				}
+			}
+		})
+
+		// 获取摘要
+		summary := ""
+		s.Find(".post-excerpt, .article-excerpt, .summary, p").First().Each(func(j int, ps *goquery.Selection) {
+			summary = strings.TrimSpace(ps.Text())
+			if len(summary) > 500 {
+				summary = summary[:500] + "..."
+			}
+		})
+
+		// 获取轮播图（如果有）
+		carousel := ""
+		s.Find(".carousel, .slider, .gallery").First().Each(func(j int, cs *goquery.Selection) {
+			// 收集轮播图中的图片
+			var images []string
+			cs.Find("img").Each(func(k int, imgs *goquery.Selection) {
+				src, exists := imgs.Attr("src")
+				if exists {
+					if strings.HasPrefix(src, "http") {
+						images = append(images, src)
+					} else {
+						images = append(images, "https://ribenyan.com"+src)
+					}
+				}
+			})
+			if len(images) > 0 {
+				carousel = strings.Join(images, ",")
+			}
+		})
+
+		article := models.Article{
+			Title:       title,
+			Summary:     summary,
+			CoverImage:  coverImage,
+			Carousel:    carousel,
+			IsPublished: true,
+			IsPinned:    i == 0, // 第一篇置顶
+			SortOrder:   i,
+			CreatedAt:   time.Now(),
+			UpdatedAt:   time.Now(),
+		}
+
+		articles = append(articles, article)
+	})
+
+	// 批量插入数据库
+	if len(articles) > 0 {
+		log.Printf("准备插入 %d 篇文章", len(articles))
+		
+		// 检查是否已存在，避免重复
+		var existingIDs []uint
+		c.db.Model(&models.Article{}).Where("title IN ?", getTitles(articles)).Pluck("id", &existingIDs)
+		
+		// 过滤掉已存在的文章
+		newArticles := make([]models.Article, 0)
+		for _, article := range articles {
+			exists := false
+			for _, id := range existingIDs {
+				if article.ID == id {
+					exists = true
+					break
+				}
+			}
+			if !exists {
+				newArticles = append(newArticles, article)
+			}
+		}
+
+		if len(newArticles) > 0 {
+			if err := c.db.Create(&newArticles).Error; err != nil {
+				return fmt.Errorf("插入失败：%v", err)
+			}
+			log.Printf("成功插入 %d 篇文章", len(newArticles))
+		} else {
+			log.Println("没有新文章需要插入")
+		}
+	} else {
+		log.Println("未找到文章")
+	}
+
+	return nil
+}
+
+func getTitles(articles []models.Article) []string {
+	titles := make([]string, len(articles))
+	for i, article := range articles {
+		titles[i] = article.Title
+	}
+	return titles
+}
+
+// 单个文章详情采集（可选）
+func (c *RibenyanCrawler) CrawlArticleDetail(url string) (*models.Article, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("请求失败：%v", err)
+	}
+	defer resp.Body.Close()
+
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("解析 HTML 失败：%v", err)
+	}
+
+	article := &models.Article{
+		IsPublished: true,
+		CreatedAt:   time.Now(),
+		UpdatedAt:   time.Now(),
+	}
+
+	// 获取标题
+	doc.Find("h1, .post-title, .article-title").First().Each(func(i int, s *goquery.Selection) {
+		article.Title = strings.TrimSpace(s.Text())
+	})
+
+	// 获取内容
+	var contentBuilder strings.Builder
+	doc.Find("article, .post-content, .article-content, .entry-content").First().Each(func(i int, s *goquery.Selection) {
+		contentBuilder.WriteString(s.Html())
+	})
+	article.Content = contentBuilder.String()
+
+	// 获取封面图
+	doc.Find("img, .post-thumbnail").First().Each(func(i int, s *goquery.Selection) {
+		src, _ := s.Attr("src")
+		if strings.HasPrefix(src, "http") {
+			article.CoverImage = src
+		} else {
+			article.CoverImage = "https://ribenyan.com" + src
+		}
+	})
+
+	// 获取摘要
+	doc.Find(".post-excerpt, .article-excerpt, .summary").First().Each(func(i int, s *goquery.Selection) {
+		article.Summary = strings.TrimSpace(s.Text())
+		if len(article.Summary) > 500 {
+			article.Summary = article.Summary[:500] + "..."
+		}
+	})
+
+	return article, nil
+}