hxuanyu/BingPaper
1
0
Fork 0
mirror of https://git.fightbot.fun/hxuanyu/BingPaper.git synced 2026-02-15 10:29:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

增加多地区每日图片抓取能力

Browse Source
This commit is contained in:
hxuanyu
2026-01-30 13:33:40 +08:00
parent b69db53f0a
commit 93690e10d3
24 changed files with 980 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
internal/service/fetcher/fetcher.go
View File

@@ -55,7 +55,30 @@ func NewFetcher() *Fetcher {
func (f *Fetcher) Fetch(ctx context.Context, n int) error {
util.Logger.Info("Starting fetch task", zap.Int("n", n))
url := fmt.Sprintf("%s?format=js&idx=0&n=%d&uhd=1&mkt=%s", config.BingAPIBase, n, config.BingMkt)
regions := config.GetConfig().Fetcher.Regions
if len(regions) == 0 {
regions = []string{config.GetConfig().GetDefaultMkt()}
}
for _, mkt := range regions {
util.Logger.Info("Fetching images for region", zap.String("mkt", mkt))
// 调用两次 API 获取最多两周的数据
// 第一次 idx=0&n=8 (今天起往回数 8 张)
if err := f.fetchByMkt(ctx, mkt, 0, 8); err != nil {
util.Logger.Error("Failed to fetch images", zap.String("mkt", mkt), zap.Int("idx", 0), zap.Error(err))
}
// 第二次 idx=7&n=8 (7天前起往回数 8 张，与第一次有重叠，确保不漏)
if err := f.fetchByMkt(ctx, mkt, 7, 8); err != nil {
util.Logger.Error("Failed to fetch images", zap.String("mkt", mkt), zap.Int("idx", 7), zap.Error(err))
}
}
util.Logger.Info("Fetch task completed")
return nil
}
func (f *Fetcher) fetchByMkt(ctx context.Context, mkt string, idx int, n int) error {
url := fmt.Sprintf("%s?format=js&idx=%d&n=%d&uhd=1&mkt=%s", config.BingAPIBase, idx, n, mkt)
util.Logger.Debug("Requesting Bing API", zap.String("url", url))
resp, err := f.httpClient.Get(url)
if err != nil {
@@ -70,29 +93,28 @@ func (f *Fetcher) Fetch(ctx context.Context, n int) error {
return err
}
util.Logger.Info("Fetched images from Bing", zap.Int("count", len(bingResp.Images)))
util.Logger.Info("Fetched images from Bing", zap.String("mkt", mkt), zap.Int("count", len(bingResp.Images)))
for _, bingImg := range bingResp.Images {
if err := f.processImage(ctx, bingImg); err != nil {
util.Logger.Error("Failed to process image", zap.String("date", bingImg.Enddate), zap.Error(err))
if err := f.processImage(ctx, bingImg, mkt); err != nil {
util.Logger.Error("Failed to process image", zap.String("date", bingImg.Enddate), zap.String("mkt", mkt), zap.Error(err))
}
}
util.Logger.Info("Fetch task completed")
return nil
}
func (f *Fetcher) processImage(ctx context.Context, bingImg BingImage) error {
func (f *Fetcher) processImage(ctx context.Context, bingImg BingImage, mkt string) error {
dateStr := fmt.Sprintf("%s-%s-%s", bingImg.Enddate[0:4], bingImg.Enddate[4:6], bingImg.Enddate[6:8])
// 幂等检查
var existing model.Image
if err := repo.DB.Where("date = ?", dateStr).First(&existing).Error; err == nil {
util.Logger.Info("Image already exists, skipping", zap.String("date", dateStr))
if err := repo.DB.Where("date = ? AND mkt = ?", dateStr, mkt).First(&existing).Error; err == nil {
util.Logger.Info("Image already exists, skipping", zap.String("date", dateStr), zap.String("mkt", mkt))
return nil
}
util.Logger.Info("Processing new image", zap.String("date", dateStr), zap.String("title", bingImg.Title))
util.Logger.Info("Processing new image", zap.String("date", dateStr), zap.String("mkt", mkt), zap.String("title", bingImg.Title))
// UHD 探测
imgURL, variantName := f.probeUHD(bingImg.URLBase)
@@ -113,6 +135,7 @@ func (f *Fetcher) processImage(ctx context.Context, bingImg BingImage) error {
// 创建 DB 记录
dbImg := model.Image{
Date: dateStr,
Mkt: mkt,
Title: bingImg.Title,
Copyright: bingImg.Copyright,
CopyrightLink: bingImg.CopyrightLink,
@@ -124,7 +147,7 @@ func (f *Fetcher) processImage(ctx context.Context, bingImg BingImage) error {
}
if err := repo.DB.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "date"}},
Columns: []clause.Column{{Name: "date"}, {Name: "mkt"}},
DoNothing: true,
}).Create(&dbImg).Error; err != nil {
util.Logger.Error("Failed to create image record", zap.Error(err))
@@ -134,7 +157,7 @@ func (f *Fetcher) processImage(ctx context.Context, bingImg BingImage) error {
// 再次检查 dbImg.ID 是否被填充，如果没有被填充（说明由于冲突未插入），则需要查询出已有的 ID
if dbImg.ID == 0 {
var existing model.Image
if err := repo.DB.Where("date = ?", dateStr).First(&existing).Error; err != nil {
if err := repo.DB.Where("date = ? AND mkt = ?", dateStr, mkt).First(&existing).Error; err != nil {
util.Logger.Error("Failed to query existing image record after conflict", zap.Error(err))
return err
}
@@ -188,7 +211,7 @@ func (f *Fetcher) processImage(ctx context.Context, bingImg BingImage) error {
// 保存今日额外文件
today := time.Now().Format("2006-01-02")
if dateStr == today && config.GetConfig().Feature.WriteDailyFiles {
f.saveDailyFiles(srcImg, imgData)
f.saveDailyFiles(srcImg, imgData, mkt)
}
return nil
@@ -213,7 +236,7 @@ func (f *Fetcher) downloadImage(url string) ([]byte, error) {
}
func (f *Fetcher) saveVariant(ctx context.Context, img *model.Image, variant, format string, data []byte) error {
key := fmt.Sprintf("%s/%s_%s.%s", img.Date, img.Date, variant, format)
key := fmt.Sprintf("%s/%s/%s_%s.%s", img.Mkt, img.Date, img.Date, variant, format)
contentType := "image/jpeg"
if format == "webp" {
contentType = "image/webp"
@@ -239,20 +262,21 @@ func (f *Fetcher) saveVariant(ctx context.Context, img *model.Image, variant, fo
}).Create(&vRecord).Error
}
func (f *Fetcher) saveDailyFiles(srcImg image.Image, originalData []byte) {
util.Logger.Info("Saving daily files")
func (f *Fetcher) saveDailyFiles(srcImg image.Image, originalData []byte, mkt string) {
util.Logger.Info("Saving daily files", zap.String("mkt", mkt))
localRoot := config.GetConfig().Storage.Local.Root
if localRoot == "" {
localRoot = "data"
}
if err := os.MkdirAll(localRoot, 0755); err != nil {
util.Logger.Error("Failed to create directory", zap.String("path", localRoot), zap.Error(err))
mktDir := filepath.Join(localRoot, mkt)
if err := os.MkdirAll(mktDir, 0755); err != nil {
util.Logger.Error("Failed to create directory", zap.String("path", mktDir), zap.Error(err))
return
}
// daily.jpeg (quality 100)
jpegPath := filepath.Join(localRoot, "daily.jpeg")
jpegPath := filepath.Join(mktDir, "daily.jpeg")
fJpeg, err := os.Create(jpegPath)
if err != nil {
util.Logger.Error("Failed to create daily.jpeg", zap.Error(err))
@@ -262,8 +286,21 @@ func (f *Fetcher) saveDailyFiles(srcImg image.Image, originalData []byte) {
}
// original.jpeg (quality 100)
originalPath := filepath.Join(localRoot, "original.jpeg")
originalPath := filepath.Join(mktDir, "original.jpeg")
if err := os.WriteFile(originalPath, originalData, 0644); err != nil {
util.Logger.Error("Failed to write original.jpeg", zap.Error(err))
}
// 同时也保留一份在根目录下（兼容旧逻辑，且作为默认地区图片）
// 如果是默认地区或者是第一个抓取的地区，可以覆盖根目录的文件
if mkt == config.GetConfig().GetDefaultMkt() {
jpegPathRoot := filepath.Join(localRoot, "daily.jpeg")
fJpegRoot, err := os.Create(jpegPathRoot)
if err == nil {
jpeg.Encode(fJpegRoot, srcImg, &jpeg.Options{Quality: 100})
fJpegRoot.Close()
}
originalPathRoot := filepath.Join(localRoot, "original.jpeg")
os.WriteFile(originalPathRoot, originalData, 0644)
}
}