Ma/GoFilm
1
0
Fork 0
mirror of https://github.com/ProudMuBai/GoFilm.git synced 2026-02-15 15:14:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

collect failed manage

Browse Source
This commit is contained in:
mubai
2025-02-25 23:24:24 +08:00
parent b5b0889abe
commit a2f6e017b9
16 changed files with 574 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
server/plugin/SystemInit/SpiderInit.go
View File

@@ -27,14 +27,12 @@ func FilmSourceInit() {
{Id: util.GenerateSalt(), Name: "HD(OK)", Uri: `https://okzyapi.com/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
{Id: util.GenerateSalt(), Name: "HD(HM)", Uri: `https://json.heimuer.xyz/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
{Id: util.GenerateSalt(), Name: "HD(LY)", Uri: `https://360zy.com/api.php/provide/vod/at/json`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
{Id: util.GenerateSalt(), Name: "HD(YZ)", Uri: `https://api.1080zyku.com/inc/apijson.php`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
{Id: util.GenerateSalt(), Name: "HD(kk)", Uri: `https://kuaikan-api.com/api.php/provide/vod/from/kuaikan/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
{Id: util.GenerateSalt(), Name: "HD(SN)", Uri: `https://suoniapi.com/api.php/provide/vod/from/snm3u8/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false, Interval: 2000},
//{Id: util.GenerateSalt(), Name: "HD(LZ)", Uri: `https://cj.lziapi.com/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
{Id: util.GenerateSalt(), Name: "HD(DB)", Uri: `https://caiji.dbzy.tv/api.php/provide/vod/from/dbm3u8/at/josn/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
//{Id: util.GenerateSalt(), Name: "HD(HW)", Uri: `https://cjhwba.com/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false, Interval: 3000},
//{Id: util.GenerateSalt(), Name: "HD(lzBk)", Uri: `https://cj.lzcaiji.com/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
//{Id: util.GenerateSalt(), Name: "HD(fs)", Uri: `https://www.feisuzyapi.com/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
//{Id: util.GenerateSalt(), Name: "HD(bfApp)", Uri: `http://app.bfzyapi.com/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
//Id: util.GenerateSalt(), {Name: "HD(bfBk)", Uri: `http://by.bfzyapi.com/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false,CollectType:system.CollectVideo, State: false},
//{Id: util.GenerateSalt(), Name: "HD(bfBk)", Uri: `http://app.bfzyapi.com/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
}
err := system.SaveCollectSourceList(l)
if err != nil {

2
server/plugin/common/util/Request.go
View File

@@ -26,6 +26,7 @@ type RequestInfo struct {
Params url.Values `json:"param"` // 请求参数
Header http.Header `json:"header"` // 请求头数据
Resp []byte `json:"resp"` // 响应结果数据
Err string `json:"err"` // 错误信息
}
// RefererUrl 记录上次请求的url
@@ -95,6 +96,7 @@ func ApiGet(r *RequestInfo) {
// 处理请求参数
err := Client.Visit(fmt.Sprintf("%s?%s", r.Uri, r.Params.Encode()))
if err != nil {
r.Err = err.Error()
log.Println("获取数据失败: ", err)
}
}

30
server/plugin/spider/Spider.go
View File

@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"log"
"math"
"net/url"
"server/config"
"server/model/collect"
@@ -140,7 +141,7 @@ func collectFilm(s *system.FilmSource, h, pg int) {
list, err := spiderCore.GetFilmDetail(r)
if err != nil || len(list) <= 0 {
// 添加采集失败记录
fr := system.FailureRecord{OriginId: s.Id, Uri: s.Uri, CollectType: system.CollectVideo, PageNumber: pg, Hour: h, Cause: err.Error(), Status: 1}
fr := system.FailureRecord{OriginId: s.Id, OriginName: s.Name, Uri: s.Uri, CollectType: system.CollectVideo, PageNumber: pg, Hour: h, Cause: fmt.Sprintln(err), Status: 1}
system.SaveFailureRecord(fr)
log.Println("GetMovieDetail Error: ", err)
return
@@ -294,6 +295,33 @@ func CollectSingleFilm(ids string) {
}
}
// ======================================================= 采集拓展内容 =======================================================
// SingleRecoverSpider 二次采集
func SingleRecoverSpider(fr *system.FailureRecord) {
// 通过采集时长范围执行不同的采集方式
switch {
case fr.Hour > 168 && fr.Hour < 360:
// 将此记录之后的所有同类采集记录变更为已重试
system.ChangeRecord(fr, 0)
// 如果采集的内容是 7~15 天之内更新的内容,则采集此记录之后的所有更新内容
// 获取采集参数h, 采集时长变更为 原采集时长 + 采集记录距现在的时长
h := fr.Hour + int(math.Ceil(time.Since(fr.CreatedAt).Hours()))
// 对当前所有已启用的站点 更新最新 h 小时的内容
AutoCollect(h)
case fr.Hour < 0, fr.Hour > 4320:
// 将此记录状态修改为已重试
system.ChangeRecord(fr, 0)
// 如果采集的是 最近180天内更新的内容 或全部内容, 则只对当前一条记录进行二次采集
s := system.FindCollectSourceById(fr.OriginId)
collectFilm(s, fr.Hour, fr.PageNumber)
default:
// 其余范围,暂不处理
break
}
}
// ======================================================= 公共方法 =======================================================
// CollectApiTest 测试采集接口是否可用

2
server/plugin/spider/SpiderCore.go
View File

@@ -93,7 +93,7 @@ func (jc *JsonCollect) GetFilmDetail(r util.RequestInfo) (list []system.MovieDet
//details := system.DetailListInfo{}
// 如果返回数据为空则直接结束本次循环
if len(r.Resp) <= 0 {
err = errors.New("response is empty")
err = errors.New(r.Err)
return
}
// 序列化详情数据