feat: add carousel support and ribenyan crawler
This commit is contained in:
小萝
@@ -5,6 +5,7 @@ import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"sale/internal/crawler"
|
||||
"sale/internal/models"
|
||||
"sale/internal/schemas"
|
||||
"sale/internal/utils"
|
||||
@@ -162,3 +163,15 @@ func (h *ArticleHandler) TogglePin(c *gin.Context) {
|
||||
utils.DB.Model(&article).Update("is_pinned", !article.IsPinned)
|
||||
c.JSON(http.StatusOK, gin.H{"data": article})
|
||||
}
|
||||
|
||||
// CrawlRibenyan 采集 ribenyan.com 文章
|
||||
func (h *ArticleHandler) CrawlRibenyan(c *gin.Context) {
|
||||
crawler := crawler.NewRibenyanCrawler(utils.DB)
|
||||
|
||||
if err := crawler.CrawlArticles(); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"message": "Crawl completed successfully"})
|
||||
}
|
||||
|
||||
@@ -47,6 +47,8 @@ func SetupRoutes(r *gin.Engine) {
|
||||
{
|
||||
articles.GET("", articleHandler.List)
|
||||
articles.GET("/:id", articleHandler.GetByID)
|
||||
// 采集 API（管理员权限）
|
||||
articles.POST("/crawl-ribenyan", articleHandler.CrawlRibenyan)
|
||||
}
|
||||
|
||||
api.GET("/categories", categoryHandler.List)
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
package crawler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sale/internal/models"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type RibenyanCrawler struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
func NewRibenyanCrawler(db *gorm.DB) *RibenyanCrawler {
|
||||
return &RibenyanCrawler{db: db}
|
||||
}
|
||||
|
||||
func (c *RibenyanCrawler) CrawlArticles() error {
|
||||
log.Println("开始采集 ribenyan.com 文章...")
|
||||
|
||||
// 访问首页获取文章列表
|
||||
resp, err := http.Get("https://ribenyan.com")
|
||||
if err != nil {
|
||||
return fmt.Errorf("请求失败：%v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("响应状态码：%d", resp.StatusCode)
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("解析 HTML 失败：%v", err)
|
||||
}
|
||||
|
||||
// 查找文章列表（根据实际网站结构调整选择器）
|
||||
articles := make([]models.Article, 0)
|
||||
|
||||
doc.Find("article, .post, .article, .news-item").Each(func(i int, s *goquery.Selection) {
|
||||
if i >= 20 { // 最多采集 20 篇
|
||||
return
|
||||
}
|
||||
|
||||
// 获取标题
|
||||
title := ""
|
||||
s.Find("h1, h2, h3, .post-title, .article-title").First().Each(func(j int, ts *goquery.Selection) {
|
||||
title = strings.TrimSpace(ts.Text())
|
||||
})
|
||||
|
||||
if title == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// 获取链接
|
||||
link := ""
|
||||
s.Find("a").First().Each(func(j int, ls *goquery.Selection) {
|
||||
href, exists := ls.Attr("href")
|
||||
if exists && !strings.HasPrefix(href, "http") {
|
||||
link = "https://ribenyan.com" + href
|
||||
} else if exists {
|
||||
link = href
|
||||
}
|
||||
})
|
||||
|
||||
// 获取封面图
|
||||
coverImage := ""
|
||||
s.Find("img, .post-thumbnail, .article-cover").First().Each(func(j int, imgs *goquery.Selection) {
|
||||
src, exists := imgs.Attr("src")
|
||||
if exists {
|
||||
if strings.HasPrefix(src, "http") {
|
||||
coverImage = src
|
||||
} else {
|
||||
coverImage = "https://ribenyan.com" + src
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// 获取摘要
|
||||
summary := ""
|
||||
s.Find(".post-excerpt, .article-excerpt, .summary, p").First().Each(func(j int, ps *goquery.Selection) {
|
||||
summary = strings.TrimSpace(ps.Text())
|
||||
if len(summary) > 500 {
|
||||
summary = summary[:500] + "..."
|
||||
}
|
||||
})
|
||||
|
||||
// 获取轮播图（如果有）
|
||||
carousel := ""
|
||||
s.Find(".carousel, .slider, .gallery").First().Each(func(j int, cs *goquery.Selection) {
|
||||
// 收集轮播图中的图片
|
||||
var images []string
|
||||
cs.Find("img").Each(func(k int, imgs *goquery.Selection) {
|
||||
src, exists := imgs.Attr("src")
|
||||
if exists {
|
||||
if strings.HasPrefix(src, "http") {
|
||||
images = append(images, src)
|
||||
} else {
|
||||
images = append(images, "https://ribenyan.com"+src)
|
||||
}
|
||||
}
|
||||
})
|
||||
if len(images) > 0 {
|
||||
carousel = strings.Join(images, ",")
|
||||
}
|
||||
})
|
||||
|
||||
article := models.Article{
|
||||
Title: title,
|
||||
Summary: summary,
|
||||
CoverImage: coverImage,
|
||||
Carousel: carousel,
|
||||
IsPublished: true,
|
||||
IsPinned: i == 0, // 第一篇置顶
|
||||
SortOrder: i,
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
}
|
||||
|
||||
articles = append(articles, article)
|
||||
})
|
||||
|
||||
// 批量插入数据库
|
||||
if len(articles) > 0 {
|
||||
log.Printf("准备插入 %d 篇文章", len(articles))
|
||||
|
||||
// 检查是否已存在，避免重复
|
||||
var existingIDs []uint
|
||||
c.db.Model(&models.Article{}).Where("title IN ?", getTitles(articles)).Pluck("id", &existingIDs)
|
||||
|
||||
// 过滤掉已存在的文章
|
||||
newArticles := make([]models.Article, 0)
|
||||
for _, article := range articles {
|
||||
exists := false
|
||||
for _, id := range existingIDs {
|
||||
if article.ID == id {
|
||||
exists = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !exists {
|
||||
newArticles = append(newArticles, article)
|
||||
}
|
||||
}
|
||||
|
||||
if len(newArticles) > 0 {
|
||||
if err := c.db.Create(&newArticles).Error; err != nil {
|
||||
return fmt.Errorf("插入失败：%v", err)
|
||||
}
|
||||
log.Printf("成功插入 %d 篇文章", len(newArticles))
|
||||
} else {
|
||||
log.Println("没有新文章需要插入")
|
||||
}
|
||||
} else {
|
||||
log.Println("未找到文章")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getTitles(articles []models.Article) []string {
|
||||
titles := make([]string, len(articles))
|
||||
for i, article := range articles {
|
||||
titles[i] = article.Title
|
||||
}
|
||||
return titles
|
||||
}
|
||||
|
||||
// 单个文章详情采集（可选）
|
||||
func (c *RibenyanCrawler) CrawlArticleDetail(url string) (*models.Article, error) {
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("请求失败：%v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("解析 HTML 失败：%v", err)
|
||||
}
|
||||
|
||||
article := &models.Article{
|
||||
IsPublished: true,
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
}
|
||||
|
||||
// 获取标题
|
||||
doc.Find("h1, .post-title, .article-title").First().Each(func(i int, s *goquery.Selection) {
|
||||
article.Title = strings.TrimSpace(s.Text())
|
||||
})
|
||||
|
||||
// 获取内容
|
||||
var contentBuilder strings.Builder
|
||||
doc.Find("article, .post-content, .article-content, .entry-content").First().Each(func(i int, s *goquery.Selection) {
|
||||
contentBuilder.WriteString(s.Html())
|
||||
})
|
||||
article.Content = contentBuilder.String()
|
||||
|
||||
// 获取封面图
|
||||
doc.Find("img, .post-thumbnail").First().Each(func(i int, s *goquery.Selection) {
|
||||
src, _ := s.Attr("src")
|
||||
if strings.HasPrefix(src, "http") {
|
||||
article.CoverImage = src
|
||||
} else {
|
||||
article.CoverImage = "https://ribenyan.com" + src
|
||||
}
|
||||
})
|
||||
|
||||
// 获取摘要
|
||||
doc.Find(".post-excerpt, .article-excerpt, .summary").First().Each(func(i int, s *goquery.Selection) {
|
||||
article.Summary = strings.TrimSpace(s.Text())
|
||||
if len(article.Summary) > 500 {
|
||||
article.Summary = article.Summary[:500] + "..."
|
||||
}
|
||||
})
|
||||
|
||||
return article, nil
|
||||
}
|
||||
@@ -12,6 +12,7 @@ type Article struct {
|
||||
Content string `gorm:"type:text;not null" json:"content"`
|
||||
Summary string `gorm:"size:500" json:"summary"`
|
||||
CoverImage string `gorm:"size:500" json:"cover_image"`
|
||||
Carousel string `gorm:"size:500" json:"carousel"`
|
||||
IsPinned bool `gorm:"default:false" json:"is_pinned"`
|
||||
IsPublished bool `gorm:"default:true" json:"is_published"`
|
||||
SortOrder int `gorm:"default:0" json:"sort_order"`
|
||||
|
||||
Reference in New Issue
Block a user