优化按需抓取逻辑:改为异步处理以提升性能,并为相关接口新增 202 状态支持

This commit is contained in:
2026-01-30 15:49:51 +08:00
parent 8ef66b2cb1
commit fb636b9450
3 changed files with 51 additions and 23 deletions

View File

@@ -48,11 +48,16 @@ type ImageMetaResp struct {
// @Param format query string false "格式 (jpg)" default(jpg) // @Param format query string false "格式 (jpg)" default(jpg)
// @Produce image/jpeg // @Produce image/jpeg
// @Success 200 {file} binary // @Success 200 {file} binary
// @Success 202 {object} map[string]string "按需抓取任务已启动"
// @Failure 404 {object} map[string]string "图片未找到,响应体包含具体原因" // @Failure 404 {object} map[string]string "图片未找到,响应体包含具体原因"
// @Router /image/today [get] // @Router /image/today [get]
func GetToday(c *gin.Context) { func GetToday(c *gin.Context) {
mkt := c.Query("mkt") mkt := c.Query("mkt")
img, err := image.GetTodayImage(mkt) img, err := image.GetTodayImage(mkt)
if err == image.ErrFetchStarted {
c.JSON(http.StatusAccepted, gin.H{"message": fmt.Sprintf("On-demand fetch started for region [%s]. Please try again later.", mkt)})
return
}
if err != nil { if err != nil {
sendImageNotFound(c, mkt) sendImageNotFound(c, mkt)
return return
@@ -67,11 +72,16 @@ func GetToday(c *gin.Context) {
// @Param mkt query string false "地区编码 (如 zh-CN, en-US)" // @Param mkt query string false "地区编码 (如 zh-CN, en-US)"
// @Produce json // @Produce json
// @Success 200 {object} ImageMetaResp // @Success 200 {object} ImageMetaResp
// @Success 202 {object} map[string]string "按需抓取任务已启动"
// @Failure 404 {object} map[string]string "图片未找到,响应体包含具体原因" // @Failure 404 {object} map[string]string "图片未找到,响应体包含具体原因"
// @Router /image/today/meta [get] // @Router /image/today/meta [get]
func GetTodayMeta(c *gin.Context) { func GetTodayMeta(c *gin.Context) {
mkt := c.Query("mkt") mkt := c.Query("mkt")
img, err := image.GetTodayImage(mkt) img, err := image.GetTodayImage(mkt)
if err == image.ErrFetchStarted {
c.JSON(http.StatusAccepted, gin.H{"message": fmt.Sprintf("On-demand fetch started for region [%s]. Please try again later.", mkt)})
return
}
if err != nil { if err != nil {
sendImageNotFound(c, mkt) sendImageNotFound(c, mkt)
return return
@@ -89,11 +99,16 @@ func GetTodayMeta(c *gin.Context) {
// @Param format query string false "格式" default(jpg) // @Param format query string false "格式" default(jpg)
// @Produce image/jpeg // @Produce image/jpeg
// @Success 200 {file} binary // @Success 200 {file} binary
// @Success 202 {object} map[string]string "按需抓取任务已启动"
// @Failure 404 {object} map[string]string "图片未找到,响应体包含具体原因" // @Failure 404 {object} map[string]string "图片未找到,响应体包含具体原因"
// @Router /image/random [get] // @Router /image/random [get]
func GetRandom(c *gin.Context) { func GetRandom(c *gin.Context) {
mkt := c.Query("mkt") mkt := c.Query("mkt")
img, err := image.GetRandomImage(mkt) img, err := image.GetRandomImage(mkt)
if err == image.ErrFetchStarted {
c.JSON(http.StatusAccepted, gin.H{"message": fmt.Sprintf("On-demand fetch started for region [%s]. Please try again later.", mkt)})
return
}
if err != nil { if err != nil {
sendImageNotFound(c, mkt) sendImageNotFound(c, mkt)
return return
@@ -108,11 +123,16 @@ func GetRandom(c *gin.Context) {
// @Param mkt query string false "地区编码 (如 zh-CN, en-US)" // @Param mkt query string false "地区编码 (如 zh-CN, en-US)"
// @Produce json // @Produce json
// @Success 200 {object} ImageMetaResp // @Success 200 {object} ImageMetaResp
// @Success 202 {object} map[string]string "按需抓取任务已启动"
// @Failure 404 {object} map[string]string "图片未找到,响应体包含具体原因" // @Failure 404 {object} map[string]string "图片未找到,响应体包含具体原因"
// @Router /image/random/meta [get] // @Router /image/random/meta [get]
func GetRandomMeta(c *gin.Context) { func GetRandomMeta(c *gin.Context) {
mkt := c.Query("mkt") mkt := c.Query("mkt")
img, err := image.GetRandomImage(mkt) img, err := image.GetRandomImage(mkt)
if err == image.ErrFetchStarted {
c.JSON(http.StatusAccepted, gin.H{"message": fmt.Sprintf("On-demand fetch started for region [%s]. Please try again later.", mkt)})
return
}
if err != nil { if err != nil {
sendImageNotFound(c, mkt) sendImageNotFound(c, mkt)
return return
@@ -131,12 +151,17 @@ func GetRandomMeta(c *gin.Context) {
// @Param format query string false "格式" default(jpg) // @Param format query string false "格式" default(jpg)
// @Produce image/jpeg // @Produce image/jpeg
// @Success 200 {file} binary // @Success 200 {file} binary
// @Success 202 {object} map[string]string "按需抓取任务已启动"
// @Failure 404 {object} map[string]string "图片未找到,响应体包含具体原因" // @Failure 404 {object} map[string]string "图片未找到,响应体包含具体原因"
// @Router /image/date/{date} [get] // @Router /image/date/{date} [get]
func GetByDate(c *gin.Context) { func GetByDate(c *gin.Context) {
date := c.Param("date") date := c.Param("date")
mkt := c.Query("mkt") mkt := c.Query("mkt")
img, err := image.GetImageByDate(date, mkt) img, err := image.GetImageByDate(date, mkt)
if err == image.ErrFetchStarted {
c.JSON(http.StatusAccepted, gin.H{"message": fmt.Sprintf("On-demand fetch started for region [%s]. Please try again later.", mkt)})
return
}
if err != nil { if err != nil {
sendImageNotFound(c, mkt) sendImageNotFound(c, mkt)
return return
@@ -152,12 +177,17 @@ func GetByDate(c *gin.Context) {
// @Param mkt query string false "地区编码 (如 zh-CN, en-US)" // @Param mkt query string false "地区编码 (如 zh-CN, en-US)"
// @Produce json // @Produce json
// @Success 200 {object} ImageMetaResp // @Success 200 {object} ImageMetaResp
// @Success 202 {object} map[string]string "按需抓取任务已启动"
// @Failure 404 {object} map[string]string "图片未找到,响应体包含具体原因" // @Failure 404 {object} map[string]string "图片未找到,响应体包含具体原因"
// @Router /image/date/{date}/meta [get] // @Router /image/date/{date}/meta [get]
func GetByDateMeta(c *gin.Context) { func GetByDateMeta(c *gin.Context) {
date := c.Param("date") date := c.Param("date")
mkt := c.Query("mkt") mkt := c.Query("mkt")
img, err := image.GetImageByDate(date, mkt) img, err := image.GetImageByDate(date, mkt)
if err == image.ErrFetchStarted {
c.JSON(http.StatusAccepted, gin.H{"message": fmt.Sprintf("On-demand fetch started for region [%s]. Please try again later.", mkt)})
return
}
if err != nil { if err != nil {
sendImageNotFound(c, mkt) sendImageNotFound(c, mkt)
return return

View File

@@ -2,6 +2,7 @@ package image
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"time" "time"
@@ -15,6 +16,8 @@ import (
"go.uber.org/zap" "go.uber.org/zap"
) )
var ErrFetchStarted = errors.New("on-demand fetch started")
func CleanupOldImages(ctx context.Context) error { func CleanupOldImages(ctx context.Context) error {
days := config.GetConfig().Retention.Days days := config.GetConfig().Retention.Days
if days <= 0 { if days <= 0 {
@@ -60,14 +63,13 @@ func GetTodayImage(mkt string) (*model.Image, error) {
} }
err := tx.Preload("Variants").First(&img).Error err := tx.Preload("Variants").First(&img).Error
if err != nil && mkt != "" && config.GetConfig().API.EnableOnDemandFetch && util.IsValidRegion(mkt) { if err != nil && mkt != "" && config.GetConfig().API.EnableOnDemandFetch && util.IsValidRegion(mkt) {
// 如果没找到,尝试按需抓取该地区 // 如果没找到,尝试异步按需抓取该地区
util.Logger.Info("Image not found in DB, attempting on-demand fetch", zap.String("mkt", mkt)) util.Logger.Info("Image not found in DB, starting asynchronous on-demand fetch", zap.String("mkt", mkt))
f := fetcher.NewFetcher() f := fetcher.NewFetcher()
go func() {
_ = f.FetchRegion(context.Background(), mkt) _ = f.FetchRegion(context.Background(), mkt)
}()
// 抓取后重新查询 return nil, ErrFetchStarted
tx = repo.DB.Where("date = ?", today).Where("mkt = ?", mkt)
err = tx.Preload("Variants").First(&img).Error
} }
if err != nil { if err != nil {
@@ -118,14 +120,13 @@ func GetRandomImage(mkt string) (*model.Image, error) {
} }
tx.Count(&count) tx.Count(&count)
if count == 0 && mkt != "" && config.GetConfig().API.EnableOnDemandFetch && util.IsValidRegion(mkt) { if count == 0 && mkt != "" && config.GetConfig().API.EnableOnDemandFetch && util.IsValidRegion(mkt) {
// 如果没找到,尝试按需抓取该地区 // 如果没找到,尝试异步按需抓取该地区
util.Logger.Info("No images found in DB for region, attempting on-demand fetch", zap.String("mkt", mkt)) util.Logger.Info("No images found in DB for region, starting asynchronous on-demand fetch", zap.String("mkt", mkt))
f := fetcher.NewFetcher() f := fetcher.NewFetcher()
go func() {
_ = f.FetchRegion(context.Background(), mkt) _ = f.FetchRegion(context.Background(), mkt)
}()
// 抓取后重新计数 return nil, ErrFetchStarted
tx = repo.DB.Model(&model.Image{}).Where("mkt = ?", mkt)
tx.Count(&count)
} }
if count == 0 { if count == 0 {
@@ -167,14 +168,13 @@ func GetImageByDate(date string, mkt string) (*model.Image, error) {
} }
err := tx.Preload("Variants").First(&img).Error err := tx.Preload("Variants").First(&img).Error
if err != nil && mkt != "" && config.GetConfig().API.EnableOnDemandFetch && util.IsValidRegion(mkt) { if err != nil && mkt != "" && config.GetConfig().API.EnableOnDemandFetch && util.IsValidRegion(mkt) {
// 如果没找到,尝试按需抓取该地区 // 如果没找到,尝试异步按需抓取该地区
util.Logger.Info("Image not found in DB for date, attempting on-demand fetch", zap.String("mkt", mkt), zap.String("date", date)) util.Logger.Info("Image not found in DB for date, starting asynchronous on-demand fetch", zap.String("mkt", mkt), zap.String("date", date))
f := fetcher.NewFetcher() f := fetcher.NewFetcher()
go func() {
_ = f.FetchRegion(context.Background(), mkt) _ = f.FetchRegion(context.Background(), mkt)
}()
// 抓取后重新查询 return nil, ErrFetchStarted
tx = repo.DB.Where("date = ?", date).Where("mkt = ?", mkt)
err = tx.Preload("Variants").First(&img).Error
} }
// 兜底逻辑 // 兜底逻辑

View File

@@ -32,6 +32,4 @@ var AllRegions = []Region{
{Value: "ko-KR", Label: "韩国"}, {Value: "ko-KR", Label: "韩国"},
{Value: "en-IN", Label: "印度"}, {Value: "en-IN", Label: "印度"},
{Value: "ru-RU", Label: "俄罗斯"}, {Value: "ru-RU", Label: "俄罗斯"},
{Value: "zh-HK", Label: "中国香港"},
{Value: "zh-TW", Label: "中国台湾"},
} }