From 2334e29a58b5a502f5a675bbd85fb2bbc3bd7684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E8=90=9D?= Date: Wed, 29 Apr 2026 08:35:25 +0000 Subject: [PATCH] feat: add carousel support and ribenyan crawler --- backend/internal/api/handlers/article.go | 13 ++ backend/internal/api/routes/routes.go | 2 + backend/internal/crawler/ribenyan.go | 225 +++++++++++++++++++++++ backend/internal/models/article.go | 1 + 4 files changed, 241 insertions(+) create mode 100644 backend/internal/crawler/ribenyan.go diff --git a/backend/internal/api/handlers/article.go b/backend/internal/api/handlers/article.go index d1f5315..47cf5d5 100644 --- a/backend/internal/api/handlers/article.go +++ b/backend/internal/api/handlers/article.go @@ -5,6 +5,7 @@ import ( "net/http" "strconv" + "sale/internal/crawler" "sale/internal/models" "sale/internal/schemas" "sale/internal/utils" @@ -162,3 +163,15 @@ func (h *ArticleHandler) TogglePin(c *gin.Context) { utils.DB.Model(&article).Update("is_pinned", !article.IsPinned) c.JSON(http.StatusOK, gin.H{"data": article}) } + +// CrawlRibenyan 采集 ribenyan.com 文章 +func (h *ArticleHandler) CrawlRibenyan(c *gin.Context) { + crawler := crawler.NewRibenyanCrawler(utils.DB) + + if err := crawler.CrawlArticles(); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, gin.H{"message": "Crawl completed successfully"}) +} diff --git a/backend/internal/api/routes/routes.go b/backend/internal/api/routes/routes.go index 5ca5e5d..cfc33af 100644 --- a/backend/internal/api/routes/routes.go +++ b/backend/internal/api/routes/routes.go @@ -47,6 +47,8 @@ func SetupRoutes(r *gin.Engine) { { articles.GET("", articleHandler.List) articles.GET("/:id", articleHandler.GetByID) + // 采集 API（管理员权限） + articles.POST("/crawl-ribenyan", articleHandler.CrawlRibenyan) } api.GET("/categories", categoryHandler.List) diff --git a/backend/internal/crawler/ribenyan.go b/backend/internal/crawler/ribenyan.go new file mode 100644 index 0000000..ecff9e3 --- /dev/null +++ b/backend/internal/crawler/ribenyan.go @@ -0,0 +1,225 @@ +package crawler + +import ( + "fmt" + "log" + "net/http" + "strings" + "time" + + "sale/internal/models" + + "github.com/PuerkitoBio/goquery" + "gorm.io/gorm" +) + +type RibenyanCrawler struct { + db *gorm.DB +} + +func NewRibenyanCrawler(db *gorm.DB) *RibenyanCrawler { + return &RibenyanCrawler{db: db} +} + +func (c *RibenyanCrawler) CrawlArticles() error { + log.Println("开始采集 ribenyan.com 文章...") + + // 访问首页获取文章列表 + resp, err := http.Get("https://ribenyan.com") + if err != nil { + return fmt.Errorf("请求失败：%v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("响应状态码：%d", resp.StatusCode) + } + + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return fmt.Errorf("解析 HTML 失败：%v", err) + } + + // 查找文章列表（根据实际网站结构调整选择器） + articles := make([]models.Article, 0) + + doc.Find("article, .post, .article, .news-item").Each(func(i int, s *goquery.Selection) { + if i >= 20 { // 最多采集 20 篇 + return + } + + // 获取标题 + title := "" + s.Find("h1, h2, h3, .post-title, .article-title").First().Each(func(j int, ts *goquery.Selection) { + title = strings.TrimSpace(ts.Text()) + }) + + if title == "" { + return + } + + // 获取链接 + link := "" + s.Find("a").First().Each(func(j int, ls *goquery.Selection) { + href, exists := ls.Attr("href") + if exists && !strings.HasPrefix(href, "http") { + link = "https://ribenyan.com" + href + } else if exists { + link = href + } + }) + + // 获取封面图 + coverImage := "" + s.Find("img, .post-thumbnail, .article-cover").First().Each(func(j int, imgs *goquery.Selection) { + src, exists := imgs.Attr("src") + if exists { + if strings.HasPrefix(src, "http") { + coverImage = src + } else { + coverImage = "https://ribenyan.com" + src + } + } + }) + + // 获取摘要 + summary := "" + s.Find(".post-excerpt, .article-excerpt, .summary, p").First().Each(func(j int, ps *goquery.Selection) { + summary = strings.TrimSpace(ps.Text()) + if len(summary) > 500 { + summary = summary[:500] + "..." + } + }) + + // 获取轮播图（如果有） + carousel := "" + s.Find(".carousel, .slider, .gallery").First().Each(func(j int, cs *goquery.Selection) { + // 收集轮播图中的图片 + var images []string + cs.Find("img").Each(func(k int, imgs *goquery.Selection) { + src, exists := imgs.Attr("src") + if exists { + if strings.HasPrefix(src, "http") { + images = append(images, src) + } else { + images = append(images, "https://ribenyan.com"+src) + } + } + }) + if len(images) > 0 { + carousel = strings.Join(images, ",") + } + }) + + article := models.Article{ + Title: title, + Summary: summary, + CoverImage: coverImage, + Carousel: carousel, + IsPublished: true, + IsPinned: i == 0, // 第一篇置顶 + SortOrder: i, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + + articles = append(articles, article) + }) + + // 批量插入数据库 + if len(articles) > 0 { + log.Printf("准备插入 %d 篇文章", len(articles)) + + // 检查是否已存在，避免重复 + var existingIDs []uint + c.db.Model(&models.Article{}).Where("title IN ?", getTitles(articles)).Pluck("id", &existingIDs) + + // 过滤掉已存在的文章 + newArticles := make([]models.Article, 0) + for _, article := range articles { + exists := false + for _, id := range existingIDs { + if article.ID == id { + exists = true + break + } + } + if !exists { + newArticles = append(newArticles, article) + } + } + + if len(newArticles) > 0 { + if err := c.db.Create(&newArticles).Error; err != nil { + return fmt.Errorf("插入失败：%v", err) + } + log.Printf("成功插入 %d 篇文章", len(newArticles)) + } else { + log.Println("没有新文章需要插入") + } + } else { + log.Println("未找到文章") + } + + return nil +} + +func getTitles(articles []models.Article) []string { + titles := make([]string, len(articles)) + for i, article := range articles { + titles[i] = article.Title + } + return titles +} + +// 单个文章详情采集（可选） +func (c *RibenyanCrawler) CrawlArticleDetail(url string) (*models.Article, error) { + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("请求失败：%v", err) + } + defer resp.Body.Close() + + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return nil, fmt.Errorf("解析 HTML 失败：%v", err) + } + + article := &models.Article{ + IsPublished: true, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + + // 获取标题 + doc.Find("h1, .post-title, .article-title").First().Each(func(i int, s *goquery.Selection) { + article.Title = strings.TrimSpace(s.Text()) + }) + + // 获取内容 + var contentBuilder strings.Builder + doc.Find("article, .post-content, .article-content, .entry-content").First().Each(func(i int, s *goquery.Selection) { + contentBuilder.WriteString(s.Html()) + }) + article.Content = contentBuilder.String() + + // 获取封面图 + doc.Find("img, .post-thumbnail").First().Each(func(i int, s *goquery.Selection) { + src, _ := s.Attr("src") + if strings.HasPrefix(src, "http") { + article.CoverImage = src + } else { + article.CoverImage = "https://ribenyan.com" + src + } + }) + + // 获取摘要 + doc.Find(".post-excerpt, .article-excerpt, .summary").First().Each(func(i int, s *goquery.Selection) { + article.Summary = strings.TrimSpace(s.Text()) + if len(article.Summary) > 500 { + article.Summary = article.Summary[:500] + "..." + } + }) + + return article, nil +} diff --git a/backend/internal/models/article.go b/backend/internal/models/article.go index 27631ca..b30d0b6 100644 --- a/backend/internal/models/article.go +++ b/backend/internal/models/article.go @@ -12,6 +12,7 @@ type Article struct { Content string `gorm:"type:text;not null" json:"content"` Summary string `gorm:"size:500" json:"summary"` CoverImage string `gorm:"size:500" json:"cover_image"` + Carousel string `gorm:"size:500" json:"carousel"` IsPinned bool `gorm:"default:false" json:"is_pinned"` IsPublished bool `gorm:"default:true" json:"is_published"` SortOrder int `gorm:"default:0" json:"sort_order"`