From 0b32d81785e982816edb57c3dd47eed3f63bf2dd Mon Sep 17 00:00:00 2001 From: mubai <1609539827@qq.com> Date: Sat, 24 Jun 2023 22:37:22 +0800 Subject: [PATCH] add search tag api --- server/README.md | 46 ++++----- server/config/DataConfig.go | 23 +++-- server/controller/IndexController.go | 10 +- server/logic/IndexLogic.go | 6 ++ server/main.go | 1 + server/model/Categories.go | 12 +++ server/model/Movies.go | 124 ++-------------------- server/model/Search.go | 149 +++++++++++++++++++++++++++ server/plugin/spider/Spider.go | 18 ++-- 9 files changed, 230 insertions(+), 159 deletions(-) diff --git a/server/README.md b/server/README.md index acfdae6..346c694 100644 --- a/server/README.md +++ b/server/README.md @@ -107,29 +107,29 @@ server > search 表 (用于记录影片的相关检索信息, 主要用于影片的 搜索, 分类, 排序 等) -| 字段名称 | 类型 | 字段释义 | -| ------------ | -------- | ---------------- | -| id | bigint | 自增主键 | -| created_at | datetime | 记录创建时间 | -| updated_at | datetime | 记录更新时间 | -| deleted_at | datetime | 逻辑删除字段 | -| mid | bigint | 影片ID | -| cid | bigint | 二级分类ID | -| pid | bigint | 一级分类ID | -| name | varchar | 影片名称 | -| sub_title | varchar | 子标题(影片别名) | -| c_name | varchar | 分类名称 | -| class_tag | varchar | 剧情标签 | -| area | varchar | 地区 | -| language | varchar | 语言 | -| year | bigint | 上映年份 | -| initial | varchar | 首字母 | -| score | double | 豆瓣评分 | -| update_stamp | bigint | 影片更新时间戳 | -| hits | bigint | 热度(播放次数) | -| state | varchar | 状态(正片) | -| remarks | varchar | 更新状态(完结 | -| release_data | bigint | 上映时间戳 | +| 字段名称 | 类型 | 字段释义 | +| ------------ | -------- | ---------------------- | +| id | bigint | 自增主键 | +| created_at | datetime | 记录创建时间 | +| updated_at | datetime | 记录更新时间 | +| deleted_at | datetime | 逻辑删除字段 | +| mid | bigint | 影片ID | +| cid | bigint | 二级分类ID | +| pid | bigint | 一级分类ID | +| name | varchar | 影片名称 | +| sub_title | varchar | 子标题(影片别名) | +| c_name | varchar | 分类名称 | +| class_tag | varchar | 剧情标签 | +| area | varchar | 地区 | +| language | varchar | 语言 | +| year | bigint | 上映年份 | +| initial | varchar | 首字母 | +| score | double | 豆瓣评分 | +| update_stamp | bigint | 影片更新时间戳 | +| hits | bigint | 热度(播放次数) | +| state | varchar | 状态(正片) | +| remarks | varchar | 更新状态(完结 \| xx集) | +| release_data | bigint | 上映时间戳 | diff --git a/server/config/DataConfig.go b/server/config/DataConfig.go index b78ae37..e2906ef 100644 --- a/server/config/DataConfig.go +++ b/server/config/DataConfig.go @@ -10,6 +10,18 @@ const ( // MAXGoroutine max goroutine, 执行spider中对协程的数量限制 MAXGoroutine = 10 + // CornMovieUpdate 影片更新定时任务间隔 + CornMovieUpdate = "0 0/20 * * * ?" + // UpdateInterval 获取最近几小时更新的影片 (h 小时) 默认3小时 + UpdateInterval = "3" + // CornUpdateAll 每月28执行一次清库更新 + CornUpdateAll = "0 0 2 28 * ?" + + // SpiderCipher 设置Spider触发指令 + SpiderCipher = "Life in a different world from zero" + + // -------------------------redis key----------------------------------- + // CategoryTreeKey 分类树 key CategoryTreeKey = "CategoryTree" CategoryTreeExpired = time.Hour * 24 * 90 @@ -35,15 +47,8 @@ const ( // SearchInfoTemp redis暂存检索数据信息 SearchInfoTemp = "Search:SearchInfoTemp" - // CornMovieUpdate 影片更新定时任务间隔 - CornMovieUpdate = "0 0/20 * * * ?" - // UpdateInterval 获取最近几小时更新的影片 (h 小时) 默认3小时 - UpdateInterval = "3" - // CornUpdateAll 每月28执行一次清库更新 - CornUpdateAll = "0 0 2 28 * ?" - - // SpiderCipher 设置Spider触发指令 - SpiderCipher = "Life in a different world from zero" + SearchTitle = "Search:Pid%d:Title" + SearchTag = "Search:Pid%d:%s" ) /*API相关redis key*/ diff --git a/server/controller/IndexController.go b/server/controller/IndexController.go index 5c49a56..f21bc8e 100644 --- a/server/controller/IndexController.go +++ b/server/controller/IndexController.go @@ -140,8 +140,9 @@ func FilmCategory(c *gin.Context) { c.JSON(http.StatusOK, gin.H{ "status": StatusOk, "data": gin.H{ - "list": logic.IL.GetFilmCategory(pid, "pid", &page), - "category": category, + "list": logic.IL.GetFilmCategory(pid, "pid", &page), + "category": category, + "searchTags": logic.IL.SearchTags(pid), }, "page": page, }) @@ -152,8 +153,9 @@ func FilmCategory(c *gin.Context) { c.JSON(http.StatusOK, gin.H{ "status": StatusOk, "data": gin.H{ - "list": logic.IL.GetFilmCategory(cid, "cid", &page), - "category": category, + "list": logic.IL.GetFilmCategory(cid, "cid", &page), + "category": category, + "searchTags": logic.IL.SearchTags(pid), }, "page": page, }) diff --git a/server/logic/IndexLogic.go b/server/logic/IndexLogic.go index 5c22c65..12ea7aa 100644 --- a/server/logic/IndexLogic.go +++ b/server/logic/IndexLogic.go @@ -150,6 +150,12 @@ func (i *IndexLogic) RelateMovie(detail model.MovieDetail, page *model.Page) []m return model.GetRelateMovieBasicInfo(search, page) } +// SearchTags 整合对应分类的搜索tag +func (i *IndexLogic) SearchTags(pid int64) map[string]interface{} { + // 通过pid 获取对应分类的 tags + return model.GetSearchTag(pid) +} + /* 将多个站点的对应影视播放源追加到主站点播放列表中 1. 将主站点影片的name 和 subtitle 进行处理添加到用于匹配对应播放源的map中 diff --git a/server/main.go b/server/main.go index 954f99f..ac6e3f0 100644 --- a/server/main.go +++ b/server/main.go @@ -23,6 +23,7 @@ func init() { } func main() { start() + //spider.MtSiteSpider() } func start() { diff --git a/server/model/Categories.go b/server/model/Categories.go index 87f837f..a94d29c 100644 --- a/server/model/Categories.go +++ b/server/model/Categories.go @@ -41,3 +41,15 @@ func ExistsCategoryTree() bool { } return exists == 1 } + +// GetChildrenTree 根据影片Id获取对应分类的子分类信息 +func GetChildrenTree(id int64) []*CategoryTree { + tree := GetCategoryTree() + for _, t := range tree.Children { + if t.Id == id { + return t.Children + } + } + return nil + +} diff --git a/server/model/Movies.go b/server/model/Movies.go index e75b416..ef8a42e 100644 --- a/server/model/Movies.go +++ b/server/model/Movies.go @@ -89,6 +89,8 @@ type MovieDetail struct { MovieDescriptor `json:"descriptor"` //影片描述信息 } +// ===================================Redis数据交互======================================================== + // SaveDetails 保存影片详情信息到redis中 格式: MovieDetail:Cid?:Id? func SaveDetails(list []MovieDetail) (err error) { // 遍历list中的信息 @@ -100,15 +102,13 @@ func SaveDetails(list []MovieDetail) (err error) { err = db.Rdb.Set(db.Cxt, fmt.Sprintf(config.MovieDetailKey, detail.Cid, detail.Id), data, config.CategoryTreeExpired).Err() // 2. 同步保存简略信息到redis中 SaveMovieBasicInfo(detail) - // 3. 保存Search检索信息到redis, 暂时搁置 - //if err == nil { - // // 转换 detail信息 - // searchInfo := ConvertSearchInfo(detail) - // // 放弃redis进行检索, 多条件处理不方便 - // //err = AddSearchInfo(searchInfo) - // // 只存储用于检索对应影片的关键字信息 - // SearchKeyword(searchInfo) - //} + // 3. 保存 Search tag redis中 + if err == nil { + // 转换 detail信息 + searchInfo := ConvertSearchInfo(detail) + // 只存储用于检索对应影片的关键字信息 + SaveSearchTag(searchInfo) + } } // 保存一份search信息到mysql, 批量存储 @@ -153,7 +153,7 @@ func SaveSitePlayList(siteName string, list []MovieDetail) (err error) { continue } // 如果DbId不为0, 则以dbID作为key进行hash额外存储一次 - if d.DbId > 0 { + if d.DbId != 0 { res[GenerateHashKey(d.DbId)] = string(data) } res[GenerateHashKey(d.Name)] = string(data) @@ -167,108 +167,6 @@ func SaveSitePlayList(siteName string, list []MovieDetail) (err error) { return } -// AddSearchInfo 将影片关键字信息整合后存入search 集合中 -func AddSearchInfo(searchInfo SearchInfo) (err error) { - // 片名 Name 分类 CName 类别标签 classTag 地区 Area 语言 Language 年份 Year 首字母 Initial, 排序 - data, _ := json.Marshal(searchInfo) - // 时间排序 score -->时间戳 DbId 排序 --> 热度, 评分排序 DbScore - err = db.Rdb.ZAdd(db.Cxt, fmt.Sprintf("%s:Pid%d", config.SearchTimeListKey, searchInfo.Pid), redis.Z{Score: float64(searchInfo.UpdateStamp), Member: data}).Err() - err = db.Rdb.ZAdd(db.Cxt, fmt.Sprintf("%s:Pid%d", config.SearchScoreListKey, searchInfo.Pid), redis.Z{Score: searchInfo.Score, Member: data}).Err() - err = db.Rdb.ZAdd(db.Cxt, fmt.Sprintf("%s:Pid%d", config.SearchHeatListKey, searchInfo.Pid), redis.Z{Score: float64(searchInfo.Hits), Member: data}).Err() - // 添加搜索关键字信息 - SearchKeyword(searchInfo) - return -} - -// SearchKeyword 设置search关键字集合 -func SearchKeyword(search SearchInfo) { - // 首先获取redis中的search 关键字信息 - key := fmt.Sprintf("%s:Pid%d", config.SearchKeys, search.Pid) - keyword := db.Rdb.HGetAll(db.Cxt, key).Val() - if keyword["Year"] == "" { - currentYear := time.Now().Year() - year := "" - for i := 0; i < 12; i++ { - // 提供当前年份前推十二年的搜索 - year = fmt.Sprintf("%s,%d", year, currentYear-i) - } - initial := "" - for i := 65; i <= 90; i++ { - initial = fmt.Sprintf("%s,%c", initial, i) - } - keyword = map[string]string{ - //"Name": "", - "Category": "", - "Tag": "", - "Area": "", - "Language": "", - "Year": strings.Trim(year, ","), - "Initial": strings.Trim(initial, ","), - "Sort": "Time,Db,Score", // 默认,一般不修改 - } - } - // 分类标签处理 - if !strings.Contains(keyword["Category"], search.CName) { - keyword["Category"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Category"], search.CName), ",") - } - // 影视内容分类处理 - if strings.Contains(search.ClassTag, "/") { - for _, t := range strings.Split(search.ClassTag, "/") { - if !strings.Contains(keyword["Tag"], t) { - keyword["Tag"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Tag"], t), ",") - } - } - } else if strings.Contains(search.ClassTag, ",") { - for _, t := range strings.Split(search.ClassTag, ",") { - if !strings.Contains(keyword["Tag"], t) { - keyword["Tag"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Tag"], t), ",") - } - } - } else { - if !strings.Contains(keyword["Tag"], search.ClassTag) { - keyword["Tag"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Tag"], search.ClassTag), ",") - } - } - // 如果地区中包含 / 分隔符 则先进行切分处理 - if strings.Contains(search.Area, "/") { - for _, s := range strings.Split(search.Area, "/") { - if !strings.Contains(keyword["Area"], strings.TrimSpace(s)) { - keyword["Area"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Area"], s), ",") - } - } - } else if strings.Contains(search.Area, ",") { - for _, s := range strings.Split(search.Area, ",") { - if !strings.Contains(keyword["Area"], strings.TrimSpace(s)) { - keyword["Area"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Area"], s), ",") - } - } - } else { - if !strings.Contains(keyword["Area"], search.Area) { - keyword["Area"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Area"], search.Area), ",") - } - } - // 语言处理 - if strings.Contains(search.Language, "/") { - for _, l := range strings.Split(search.Language, "/") { - if !strings.Contains(keyword["Language"], l) { - keyword["Language"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Language"], l), ",") - } - } - - } else if strings.Contains(search.Language, ",") { - for _, l := range strings.Split(search.Language, ",") { - if !strings.Contains(keyword["Language"], l) { - keyword["Language"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Language"], l), ",") - } - } - } else { - if !strings.Contains(keyword["Language"], search.Language) { - keyword["Language"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Language"], search.Language), ",") - } - } - _ = db.Rdb.HMSet(db.Cxt, key, keyword).Err() -} - // BatchSaveSearchInfo 批量保存Search信息 func BatchSaveSearchInfo(list []MovieDetail) { var infoList []SearchInfo @@ -362,8 +260,6 @@ func GenerateHashKey[K string | ~int | int64](key K) string { return fmt.Sprint(h.Sum32()) } -// 处理分类方法0 - // ============================采集方案.v1 遗留================================================== // SaveMoves 保存影片分页请求list diff --git a/server/model/Search.go b/server/model/Search.go index 146f600..cc1a520 100644 --- a/server/model/Search.go +++ b/server/model/Search.go @@ -96,6 +96,122 @@ func DelMtPlay(keys []string) { db.Rdb.Del(db.Cxt, keys...) } +/* +SearchKeyword 设置search关键字集合(影片分类检索类型数据) + 类型, 剧情 , 地区, 语言, 年份, 首字母, 排序 + 1. 在影片详情缓存到redis时将影片的相关数据进行记录, 存在相同类型则分值加一 + 2. 通过分值对类型进行排序类型展示到页面 +*/ + +func SaveSearchTag(search SearchInfo) { + // 声明用于存储采集的影片的分类检索信息 + //searchMap := make(map[string][]map[string]int) + + // Redis中的记录形式 Search:SearchKeys:Pid1:Title Hash + // Redis中的记录形式 Search:SearchKeys:Pid1:xxx Hash + + // 获取redis中的searchMap + key := fmt.Sprintf(config.SearchTitle, search.Pid) + searchMap := db.Rdb.HGetAll(db.Cxt, key).Val() + // 是否存储对应分类的map, 如果不存在则缓存一份 + if len(searchMap) == 0 { + searchMap["Category"] = "类型" + searchMap["Plot"] = "剧情" + searchMap["Area"] = "地区" + searchMap["Language"] = "语言" + searchMap["Year"] = "年份" + searchMap["Initial"] = "首字母" + searchMap["Sort"] = "排序" + db.Rdb.HMSet(db.Cxt, key, searchMap) + } + // 对searchMap中的各个类型进行处理 + for k, _ := range searchMap { + tagKey := fmt.Sprintf(config.SearchTag, search.Pid, k) + tagCount := db.Rdb.ZCard(db.Cxt, tagKey).Val() + switch k { + case "Category": + // 获取 Category 数据, 如果不存在则缓存一份 + if tagCount == 0 { + var tags []redis.Z + for _, t := range GetChildrenTree(search.Pid) { + tags = append(tags, redis.Z{Score: float64(-t.Id), Member: t.Name}) + } + db.Rdb.ZAdd(db.Cxt, fmt.Sprintf(config.SearchTag, search.Pid, k), tags...) + } + case "Year": + // 获取 Year 数据, 如果不存在则缓存一份 + if tagCount == 0 { + var tags []redis.Z + currentYear := time.Now().Year() + for i := 0; i < 12; i++ { + tags = append(tags, redis.Z{Score: float64(currentYear - i), Member: currentYear - i}) + } + db.Rdb.ZAdd(db.Cxt, fmt.Sprintf(config.SearchTag, search.Pid, k), tags...) + } + case "Initial": + // 如果不存在 首字母 Tag 数据, 则缓存一份 + if tagCount == 0 { + var tags []redis.Z + for i := 65; i <= 90; i++ { + tags = append(tags, redis.Z{Score: float64(90 - i), Member: fmt.Sprintf("%c", i)}) + } + db.Rdb.ZAdd(db.Cxt, fmt.Sprintf(config.SearchTag, search.Pid, k), tags...) + } + case "Sort": + if tagCount == 0 { + tags := []redis.Z{ + {2, "time"}, + {1, "hits"}, + {0, "score"}, + } + db.Rdb.ZAdd(db.Cxt, fmt.Sprintf(config.SearchTag, search.Pid, k), tags...) + } + case "Plot": + HandleSearchTags(search.ClassTag, tagKey) + case "Area": + HandleSearchTags(search.Area, tagKey) + case "Language": + HandleSearchTags(search.Language, tagKey) + default: + break + } + } + +} + +func HandleSearchTags(preTags string, k string) { + // 先处理字符串中的空白符 然后对处理前的tag字符串进行分割 + preTags = regexp.MustCompile(`[\s\n\r]+`).ReplaceAllString(preTags, "") + f := func(sep string) { + for _, t := range strings.Split(preTags, sep) { + // 获取 tag对应的score + score := db.Rdb.ZScore(db.Cxt, k, t).Val() + // 在原score的基础上+1 重新存入redis中 + + db.Rdb.ZAdd(db.Cxt, k, redis.Z{Score: score + 1, Member: t}) + } + } + switch { + case strings.Contains(preTags, "/"): + f("/") + case strings.Contains(preTags, ","): + f(",") + case strings.Contains(preTags, "，"): + f("，") + case strings.Contains(preTags, "、"): + f("、") + default: + // 获取 tag对应的score + if len(preTags) == 0 || preTags == "其它" { + db.Rdb.ZAdd(db.Cxt, k, redis.Z{Score: 0, Member: preTags}) + } else { + score := db.Rdb.ZScore(db.Cxt, k, preTags).Val() + db.Rdb.ZAdd(db.Cxt, k, redis.Z{Score: score + 1, Member: preTags}) + } + } + +} + // ================================= Spider 数据处理(mysql) ================================= // CreateSearchTable 创建存储检索信息的数据表 @@ -332,6 +448,39 @@ func GetMultiplePlay(siteName, key string) []MovieUrlInfo { return playList } +// GetSearchTag 通过影片分类 Pid 返回对应分类的tag信息 +func GetSearchTag(pid int64) map[string]interface{} { + res := make(map[string]interface{}) + titles := db.Rdb.HGetAll(db.Cxt, fmt.Sprintf(config.SearchTitle, pid)).Val() + for k, v := range titles { + // 通过 k 获取对应的 tag , 并以score进行排序 + tags := db.Rdb.ZRevRange(db.Cxt, fmt.Sprintf(config.SearchTag, pid, k), 0, 10).Val() + res[v] = tags + + // 过滤分类tag + switch k { + case "Category", "Year", "Initial", "Sort": + tags := db.Rdb.ZRevRange(db.Cxt, fmt.Sprintf(config.SearchTag, pid, k), 0, -1).Val() + res[v] = tags + case "Plot": + tags := db.Rdb.ZRevRange(db.Cxt, fmt.Sprintf(config.SearchTag, pid, k), 0, 10).Val() + res[v] = tags + case "Area": + tags := db.Rdb.ZRevRange(db.Cxt, fmt.Sprintf(config.SearchTag, pid, k), 0, 11).Val() + res[v] = tags + case "Language": + tags := db.Rdb.ZRevRange(db.Cxt, fmt.Sprintf(config.SearchTag, pid, k), 0, 6).Val() + res[v] = tags + default: + break + } + + } + return res +} + +// ================================= 接口数据缓存 ================================= + // DataCache API请求 数据缓存 func DataCache(key string, data map[string]interface{}) { val, _ := json.Marshal(data) diff --git a/server/plugin/spider/Spider.go b/server/plugin/spider/Spider.go index 72d70eb..c428b95 100644 --- a/server/plugin/spider/Spider.go +++ b/server/plugin/spider/Spider.go @@ -30,6 +30,7 @@ import ( const ( MainSite = "https://cj.lziapi.com/api.php/provide/vod/" + //MainSite = "https://cj.lzcaiji.com/api.php/provide/vod/" ) type Site struct { @@ -39,17 +40,16 @@ type Site struct { // SiteList 播放源采集站 var SiteList = []Site{ - //{"tk", "https://api.tiankongapi.com/api.php/provide/vod"}, - //{"yh", "https://m3u8.apiyhzy.com/api.php/provide/vod/"}, - //{"zk", "https://api.1080zyku.com/inc/apijson.php"}, 数据格式不规范,不采用 - //{"fs", "https://www.feisuzyapi.com/api.php/provide/vod/"}, + // 备用采集站 //{"su", "https://subocaiji.com/api.php/provide/vod/at/json"}, + //{"bf", "https://bfzyapi.com/api.php/provide/vod/"}, + //{"ff", "https://cj.ffzyapi.com/api.php/provide/vod/"}, + //{"ff", "https://svip.ffzyapi8.com/api.php/provide/vod/"}, //{"lz", "https://cj.lziapi.com/api.php/provide/vod/"}, - //{"ff", "https://cj.ffzyapi.com/api.php/provide/vod/"}, - //{"fs", "https://www.feisuzyapi.com/api.php/provide/vod/"}, - //{"bf", "https://bfzyapi.com/api.php/provide/vod/"}, - {"test", "https://bfzyapi.com/api.php/provide/vod/"}, + {"fs", "https://www.feisuzyapi.com/api.php/provide/vod/"}, + {"bf", "http://by.bfzyapi.com/api.php/provide/vod/"}, + {"kk", "https://kuaikan-api.com/api.php/provide/vod/from/kuaikan"}, } // StartSpider 执行多源spider @@ -290,7 +290,7 @@ func StartSpiderRe() { func GetPageCount(r RequestInfo) (count int, err error) { // 发送请求获取pageCount r.Params.Set("ac", "detail") - r.Params.Set("pg", "1") + r.Params.Set("pg", "2") ApiGet(&r) // 判断请求结果是否为空, 如果为空直接输出错误并终止 if len(r.Resp) <= 0 {