diff --git a/server/config/DataConfig.go b/server/config/DataConfig.go index e2906ef..aa8206d 100644 --- a/server/config/DataConfig.go +++ b/server/config/DataConfig.go @@ -3,10 +3,15 @@ package config import "time" /* -定义一些数据库存放的key值 + 定义一些数据库存放的key值, 以及程序运行时的相关参数配置 */ + +// -------------------------System Config----------------------------------- const ( + // ListenerPort web服务监听的端口 + ListenerPort = "3601" + // MAXGoroutine max goroutine, 执行spider中对协程的数量限制 MAXGoroutine = 10 @@ -17,11 +22,16 @@ const ( // CornUpdateAll 每月28执行一次清库更新 CornUpdateAll = "0 0 2 28 * ?" - // SpiderCipher 设置Spider触发指令 + // SpiderCipher 设置Spider触发指令的验证 SpiderCipher = "Life in a different world from zero" - // -------------------------redis key----------------------------------- + // ImgCacheFlag 是否开启将主站影片图片放入本地进行存储 + ImgCacheFlag = false + ImageDir = "./resource/static/images" +) +// -------------------------redis key----------------------------------- +const ( // CategoryTreeKey 分类树 key CategoryTreeKey = "CategoryTree" CategoryTreeExpired = time.Hour * 24 * 90 @@ -36,6 +46,12 @@ const ( // MultipleSiteDetail 多站点影片信息存储key MultipleSiteDetail = "MultipleSource:%s" + // SearchInfoTemp redis暂存检索数据信息 + SearchInfoTemp = "Search:SearchInfoTemp" + + SearchTitle = "Search:Pid%d:Title" + SearchTag = "Search:Pid%d:%s" + // SearchCount Search scan 识别范围 SearchCount = 3000 // SearchKeys Search Key Hash @@ -44,21 +60,16 @@ const ( SearchScoreListKey = "Search:SearchScoreList" SearchTimeListKey = "Search:SearchTimeList" SearchHeatListKey = "Search:SearchHeatList" - // SearchInfoTemp redis暂存检索数据信息 - SearchInfoTemp = "Search:SearchInfoTemp" - - SearchTitle = "Search:Pid%d:Title" - SearchTag = "Search:Pid%d:%s" ) -/*API相关redis key*/ - +// -------------------------Web API相关redis key----------------------------------- const ( + // IndexCacheKey , 首页数据缓存 IndexCacheKey = "IndexCache" ) +// -------------------------Database Connection Params----------------------------------- const ( - // SearchTableName 存放检索信息的数据表名 SearchTableName = "search" diff --git a/server/main.go b/server/main.go index c327652..8d1b96b 100644 --- a/server/main.go +++ b/server/main.go @@ -1,6 +1,8 @@ package main import ( + "fmt" + "server/config" "server/model" "server/plugin/db" "server/plugin/spider" @@ -33,7 +35,7 @@ func start() { spider.RegularUpdateMovie() // 开启路由监听 r := router.SetupRouter() - _ = r.Run(`:3601`) + _ = r.Run(fmt.Sprintf(":%s", config.ListenerPort)) } diff --git a/server/model/Movies.go b/server/model/Movies.go index eed16ce..620b34b 100644 --- a/server/model/Movies.go +++ b/server/model/Movies.go @@ -5,8 +5,10 @@ import ( "fmt" "github.com/redis/go-redis/v9" "hash/fnv" + "path/filepath" "regexp" "server/config" + "server/plugin/common/util" "server/plugin/db" "strconv" "strings" @@ -91,6 +93,23 @@ type MovieDetail struct { // ===================================Redis数据交互======================================================== +// SaveMoviePic 保存影片图片到服务器 +func SaveMoviePic(details ...*MovieDetail) { + for _, d := range details { + // 判断 detail 在redis中是否已经存在 + if db.Rdb.Exists(db.Cxt, fmt.Sprintf(config.MovieDetailKey, d.Cid, d.Id)).Val() == 1 { + // 如果已经存在则直接continue + continue + } + // 将影片信息中的pic图片下载保存到resource/images 文件夹下 + err := util.SaveOnlineFile(d.Picture, config.ImageDir) + // 如果没有异常则将detail的图片路径替换为本地的保存路径 + if err == nil { + d.Picture = fmt.Sprintf("http://127.0.0.1:%s/static/image/%s", config.ListenerPort, filepath.Base(d.Picture)) + } + } +} + // SaveDetails 保存影片详情信息到redis中 格式: MovieDetail:Cid?:Id? func SaveDetails(list []MovieDetail) (err error) { // 遍历list中的信息 diff --git a/server/plugin/common/util/FileDownload.go b/server/plugin/common/util/FileDownload.go new file mode 100644 index 0000000..7671dc9 --- /dev/null +++ b/server/plugin/common/util/FileDownload.go @@ -0,0 +1,39 @@ +package util + +import ( + "bufio" + "os" + "path/filepath" +) + +/* +数据请求保存,文件读写 +*/ + +// SaveOnlineFile 保存网络文件, 提供下载url和保存路径, 返回保存后的文件访问url相对路径 +func SaveOnlineFile(url, dir string) (err error) { + // 请求获取文件内容 + r := &RequestInfo{Uri: url} + ApiGet(r) + // 创建保存文件的目录 + if _, err = os.Stat(dir); os.IsNotExist(err) { + err = os.MkdirAll(dir, os.ModePerm) + if err != nil { + return + } + } + // 通过保存路径和url得到保存的具体的文件全路径 + fileName := filepath.Join(dir, filepath.Base(url)) + file, err := os.Create(fileName) + if err != nil { + return + } + defer file.Close() + //_, _ = file.Write(r.Resp) + + writer := bufio.NewWriter(file) + _, err = writer.Write(r.Resp) + err = writer.Flush() + return + +} diff --git a/server/plugin/spider/SpiderRequest.go b/server/plugin/common/util/Request.go similarity index 98% rename from server/plugin/spider/SpiderRequest.go rename to server/plugin/common/util/Request.go index 6a89ce8..94e19f7 100644 --- a/server/plugin/spider/SpiderRequest.go +++ b/server/plugin/common/util/Request.go @@ -1,4 +1,4 @@ -package spider +package util import ( "fmt" @@ -12,6 +12,10 @@ import ( "time" ) +/* +网络请求, 数据爬取 +*/ + var ( Client = CreateClient() ) @@ -32,7 +36,7 @@ func CreateClient() *colly.Collector { c := colly.NewCollector() // 设置请求使用clash的socks5代理 - //setProxy(c) + setProxy(c) // 设置代理信息 //if proxy, err := proxy.RoundRobinProxySwitcher("127.0.0.1:7890"); err != nil { diff --git a/server/plugin/spider/Spider.go b/server/plugin/spider/Spider.go index 66fddf2..72bcaa6 100644 --- a/server/plugin/spider/Spider.go +++ b/server/plugin/spider/Spider.go @@ -9,6 +9,7 @@ import ( "server/config" "server/model" "server/plugin/common/dp" + "server/plugin/common/util" "time" ) @@ -66,7 +67,7 @@ func StartSpider() { SearchInfoToMdb() model.AddSearchIndex() log.Println("SearchInfoToMdb 影片检索信息保存完毕") - //获取其他站点数据13 + //获取其他站点数据 go MtSiteSpider() log.Println("Spider End , 数据保存执行完成") time.Sleep(time.Second * 10) @@ -75,12 +76,12 @@ func StartSpider() { // CategoryList 获取分类数据 func CategoryList() { // 设置请求参数信息 - r := RequestInfo{Uri: MainSite, Params: url.Values{}} + r := util.RequestInfo{Uri: MainSite, Params: url.Values{}} r.Params.Set(`ac`, "list") r.Params.Set(`pg`, "1") r.Params.Set(`t`, "1") // 执行请求, 获取一次list数据 - ApiGet(&r) + util.ApiGet(&r) // 解析resp数据 movieListInfo := model.MovieListInfo{} if len(r.Resp) <= 0 { @@ -103,7 +104,7 @@ func CategoryList() { // MainSiteSpider 主站点数据处理 func MainSiteSpider() { // 获取分页页数 - pageCount, err := GetPageCount(RequestInfo{Uri: MainSite, Params: url.Values{}}) + pageCount, err := GetPageCount(util.RequestInfo{Uri: MainSite, Params: url.Values{}}) // 主站点分页出错直接终止程序 if err != nil { panic(err) @@ -123,7 +124,7 @@ func MainSiteSpider() { if !ok { break } - list, e := GetMovieDetail(pg, RequestInfo{Uri: MainSite, Params: url.Values{}}) + list, e := GetMovieDetail(pg, util.RequestInfo{Uri: MainSite, Params: url.Values{}}) if e != nil { log.Println("GetMovieDetail Error: ", err) continue @@ -152,7 +153,7 @@ func MtSiteSpider() { // PlayDetailSpider SpiderSimpleInfo 获取单个站点的播放源 func PlayDetailSpider(s Site) { // 获取分页页数 - pageCount, err := GetPageCount(RequestInfo{Uri: s.Uri, Params: url.Values{}}) + pageCount, err := GetPageCount(util.RequestInfo{Uri: s.Uri, Params: url.Values{}}) // 出错直接终止当前站点数据获取 if err != nil { log.Println(err) @@ -174,7 +175,7 @@ func PlayDetailSpider(s Site) { if !ok { break } - list, e := GetMovieDetail(pg, RequestInfo{Uri: s.Uri, Params: url.Values{}}) + list, e := GetMovieDetail(pg, util.RequestInfo{Uri: s.Uri, Params: url.Values{}}) if e != nil || len(list) <= 0 { log.Println("GetMovieDetail Error: ", err) continue @@ -221,7 +222,7 @@ func UpdateMovieDetail() { // UpdateMainDetail 更新主站点的最新影片 func UpdateMainDetail() { // 获取分页页数 - r := RequestInfo{Uri: MainSite, Params: url.Values{}} + r := util.RequestInfo{Uri: MainSite, Params: url.Values{}} r.Params.Set("h", config.UpdateInterval) pageCount, err := GetPageCount(r) if err != nil { @@ -256,7 +257,7 @@ func UpdateMainDetail() { func UpdatePlayDetail() { for _, s := range SiteList { // 获取单个站点的分页数 - r := RequestInfo{Uri: s.Uri, Params: url.Values{}} + r := util.RequestInfo{Uri: s.Uri, Params: url.Values{}} r.Params.Set("h", config.UpdateInterval) pageCount, err := GetPageCount(r) if err != nil { @@ -288,11 +289,11 @@ func StartSpiderRe() { // =========================公共方法============================== // GetPageCount 获取总页数 -func GetPageCount(r RequestInfo) (count int, err error) { +func GetPageCount(r util.RequestInfo) (count int, err error) { // 发送请求获取pageCount r.Params.Set("ac", "detail") r.Params.Set("pg", "2") - ApiGet(&r) + util.ApiGet(&r) // 判断请求结果是否为空, 如果为空直接输出错误并终止 if len(r.Resp) <= 0 { err = errors.New("response is empty") @@ -309,7 +310,7 @@ func GetPageCount(r RequestInfo) (count int, err error) { } // GetMovieDetail 处理详情接口请求返回的数据 -func GetMovieDetail(pageNumber int, r RequestInfo) (list []model.MovieDetail, err error) { +func GetMovieDetail(pageNumber int, r util.RequestInfo) (list []model.MovieDetail, err error) { // 防止json解析异常引发panic defer func() { if e := recover(); e != nil { @@ -319,7 +320,7 @@ func GetMovieDetail(pageNumber int, r RequestInfo) (list []model.MovieDetail, er // 设置分页请求参数 r.Params.Set(`ac`, `detail`) r.Params.Set(`pg`, fmt.Sprint(pageNumber)) - ApiGet(&r) + util.ApiGet(&r) // 影视详情信息 details := model.DetailListInfo{} // 如果返回数据为空则直接结束本次循环 diff --git a/server/router/router.go b/server/router/router.go index 58a2a09..80ff103 100644 --- a/server/router/router.go +++ b/server/router/router.go @@ -4,6 +4,7 @@ import ( "github.com/gin-gonic/gin" "log" "net/http" + "server/config" "server/controller" ) @@ -13,6 +14,9 @@ func SetupRouter() *gin.Engine { // 开启跨域 r.Use(Cors()) + // 静态资源配置 + r.Static("/static/image", config.ImageDir) + r.GET(`/index`, controller.Index) r.GET(`/navCategory`, controller.CategoriesInfo) r.GET(`/filmDetail`, controller.FilmDetail)