Ma/GoFilm
1
0
Fork 0
mirror of https://github.com/ProudMuBai/GoFilm.git synced 2026-02-04 06:54:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

add search tag api

Browse Source
This commit is contained in:
mubai
2023-06-24 22:37:22 +08:00
parent 0591c6faa2
commit 0b32d81785
9 changed files with 230 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
server/README.md
View File

@@ -107,29 +107,29 @@ server
> search 表 (用于记录影片的相关检索信息, 主要用于影片的 搜索, 分类, 排序 等)
| 字段名称 | 类型 | 字段释义 |
| ------------ | -------- | ---------------- |
| id | bigint | 自增主键 |
| created_at | datetime | 记录创建时间 |
| updated_at | datetime | 记录更新时间 |
| deleted_at | datetime | 逻辑删除字段 |
| mid | bigint | 影片ID |
| cid | bigint | 二级分类ID |
| pid | bigint | 一级分类ID |
| name | varchar | 影片名称 |
| sub_title | varchar | 子标题(影片别名) |
| c_name | varchar | 分类名称 |
| class_tag | varchar | 剧情标签 |
| area | varchar | 地区 |
| language | varchar | 语言 |
| year | bigint | 上映年份 |
| initial | varchar | 首字母 |
| score | double | 豆瓣评分 |
| update_stamp | bigint | 影片更新时间戳 |
| hits | bigint | 热度(播放次数) |
| state | varchar | 状态(正片) |
| remarks | varchar | 更新状态(完结 |
| release_data | bigint | 上映时间戳 |
| 字段名称 | 类型 | 字段释义 |
| ------------ | -------- | ---------------------- |
| id | bigint | 自增主键 |
| created_at | datetime | 记录创建时间 |
| updated_at | datetime | 记录更新时间 |
| deleted_at | datetime | 逻辑删除字段 |
| mid | bigint | 影片ID |
| cid | bigint | 二级分类ID |
| pid | bigint | 一级分类ID |
| name | varchar | 影片名称 |
| sub_title | varchar | 子标题(影片别名) |
| c_name | varchar | 分类名称 |
| class_tag | varchar | 剧情标签 |
| area | varchar | 地区 |
| language | varchar | 语言 |
| year | bigint | 上映年份 |
| initial | varchar | 首字母 |
| score | double | 豆瓣评分 |
| update_stamp | bigint | 影片更新时间戳 |
| hits | bigint | 热度(播放次数) |
| state | varchar | 状态(正片) |
| remarks | varchar | 更新状态(完结 \| xx集) |
| release_data | bigint | 上映时间戳 |

23
server/config/DataConfig.go
View File

@@ -10,6 +10,18 @@ const (
// MAXGoroutine max goroutine, 执行spider中对协程的数量限制
MAXGoroutine = 10
// CornMovieUpdate 影片更新定时任务间隔
CornMovieUpdate = "0 0/20 * * * ?"
// UpdateInterval 获取最近几小时更新的影片 (h 小时) 默认3小时
UpdateInterval = "3"
// CornUpdateAll 每月28执行一次清库更新
CornUpdateAll = "0 0 2 28 * ?"
// SpiderCipher 设置Spider触发指令
SpiderCipher = "Life in a different world from zero"
// -------------------------redis key-----------------------------------
// CategoryTreeKey 分类树 key
CategoryTreeKey = "CategoryTree"
CategoryTreeExpired = time.Hour * 24 * 90
@@ -35,15 +47,8 @@ const (
// SearchInfoTemp redis暂存检索数据信息
SearchInfoTemp = "Search:SearchInfoTemp"
// CornMovieUpdate 影片更新定时任务间隔
CornMovieUpdate = "0 0/20 * * * ?"
// UpdateInterval 获取最近几小时更新的影片 (h 小时) 默认3小时
UpdateInterval = "3"
// CornUpdateAll 每月28执行一次清库更新
CornUpdateAll = "0 0 2 28 * ?"
// SpiderCipher 设置Spider触发指令
SpiderCipher = "Life in a different world from zero"
SearchTitle = "Search:Pid%d:Title"
SearchTag = "Search:Pid%d:%s"
)
/*API相关redis key*/

10
server/controller/IndexController.go
View File

@@ -140,8 +140,9 @@ func FilmCategory(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{
"status": StatusOk,
"data": gin.H{
"list": logic.IL.GetFilmCategory(pid, "pid", &page),
"category": category,
"list": logic.IL.GetFilmCategory(pid, "pid", &page),
"category": category,
"searchTags": logic.IL.SearchTags(pid),
},
"page": page,
})
@@ -152,8 +153,9 @@ func FilmCategory(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{
"status": StatusOk,
"data": gin.H{
"list": logic.IL.GetFilmCategory(cid, "cid", &page),
"category": category,
"list": logic.IL.GetFilmCategory(cid, "cid", &page),
"category": category,
"searchTags": logic.IL.SearchTags(pid),
},
"page": page,
})

6
server/logic/IndexLogic.go
View File

@@ -150,6 +150,12 @@ func (i *IndexLogic) RelateMovie(detail model.MovieDetail, page *model.Page) []m
return model.GetRelateMovieBasicInfo(search, page)
}
// SearchTags 整合对应分类的搜索tag
func (i *IndexLogic) SearchTags(pid int64) map[string]interface{} {
// 通过pid 获取对应分类的 tags
return model.GetSearchTag(pid)
}
/*
将多个站点的对应影视播放源追加到主站点播放列表中
1. 将主站点影片的name 和 subtitle 进行处理添加到用于匹配对应播放源的map中

1
server/main.go
View File

@@ -23,6 +23,7 @@ func init() {
}
func main() {
start()
//spider.MtSiteSpider()
}
func start() {

12
server/model/Categories.go
View File

@@ -41,3 +41,15 @@ func ExistsCategoryTree() bool {
}
return exists == 1
}
// GetChildrenTree 根据影片Id获取对应分类的子分类信息
func GetChildrenTree(id int64) []*CategoryTree {
tree := GetCategoryTree()
for _, t := range tree.Children {
if t.Id == id {
return t.Children
}
}
return nil
}

124
server/model/Movies.go
View File

@@ -89,6 +89,8 @@ type MovieDetail struct {
MovieDescriptor `json:"descriptor"` //影片描述信息
}
// ===================================Redis数据交互========================================================
// SaveDetails 保存影片详情信息到redis中 格式: MovieDetail:Cid?:Id?
func SaveDetails(list []MovieDetail) (err error) {
// 遍历list中的信息
@@ -100,15 +102,13 @@ func SaveDetails(list []MovieDetail) (err error) {
err = db.Rdb.Set(db.Cxt, fmt.Sprintf(config.MovieDetailKey, detail.Cid, detail.Id), data, config.CategoryTreeExpired).Err()
// 2. 同步保存简略信息到redis中
SaveMovieBasicInfo(detail)
// 3. 保存Search检索信息到redis, 暂时搁置
//if err == nil {
// // 转换 detail信息
// searchInfo := ConvertSearchInfo(detail)
// // 放弃redis进行检索, 多条件处理不方便
// //err = AddSearchInfo(searchInfo)
// // 只存储用于检索对应影片的关键字信息
// SearchKeyword(searchInfo)
//}
// 3. 保存 Search tag redis中
if err == nil {
// 转换 detail信息
searchInfo := ConvertSearchInfo(detail)
// 只存储用于检索对应影片的关键字信息
SaveSearchTag(searchInfo)
}
}
// 保存一份search信息到mysql, 批量存储
@@ -153,7 +153,7 @@ func SaveSitePlayList(siteName string, list []MovieDetail) (err error) {
continue
}
// 如果DbId不为0, 则以dbID作为key进行hash额外存储一次
if d.DbId > 0 {
if d.DbId != 0 {
res[GenerateHashKey(d.DbId)] = string(data)
}
res[GenerateHashKey(d.Name)] = string(data)
@@ -167,108 +167,6 @@ func SaveSitePlayList(siteName string, list []MovieDetail) (err error) {
return
}
// AddSearchInfo 将影片关键字信息整合后存入search 集合中
func AddSearchInfo(searchInfo SearchInfo) (err error) {
// 片名 Name 分类 CName 类别标签 classTag 地区 Area 语言 Language 年份 Year 首字母 Initial, 排序
data, _ := json.Marshal(searchInfo)
// 时间排序 score -->时间戳 DbId 排序 --> 热度, 评分排序 DbScore
err = db.Rdb.ZAdd(db.Cxt, fmt.Sprintf("%s:Pid%d", config.SearchTimeListKey, searchInfo.Pid), redis.Z{Score: float64(searchInfo.UpdateStamp), Member: data}).Err()
err = db.Rdb.ZAdd(db.Cxt, fmt.Sprintf("%s:Pid%d", config.SearchScoreListKey, searchInfo.Pid), redis.Z{Score: searchInfo.Score, Member: data}).Err()
err = db.Rdb.ZAdd(db.Cxt, fmt.Sprintf("%s:Pid%d", config.SearchHeatListKey, searchInfo.Pid), redis.Z{Score: float64(searchInfo.Hits), Member: data}).Err()
// 添加搜索关键字信息
SearchKeyword(searchInfo)
return
}
// SearchKeyword 设置search关键字集合
func SearchKeyword(search SearchInfo) {
// 首先获取redis中的search 关键字信息
key := fmt.Sprintf("%s:Pid%d", config.SearchKeys, search.Pid)
keyword := db.Rdb.HGetAll(db.Cxt, key).Val()
if keyword["Year"] == "" {
currentYear := time.Now().Year()
year := ""
for i := 0; i < 12; i++ {
// 提供当前年份前推十二年的搜索
year = fmt.Sprintf("%s,%d", year, currentYear-i)
}
initial := ""
for i := 65; i <= 90; i++ {
initial = fmt.Sprintf("%s,%c", initial, i)
}
keyword = map[string]string{
//"Name": "",
"Category": "",
"Tag": "",
"Area": "",
"Language": "",
"Year": strings.Trim(year, ","),
"Initial": strings.Trim(initial, ","),
"Sort": "Time,Db,Score", // 默认,一般不修改
}
}
// 分类标签处理
if !strings.Contains(keyword["Category"], search.CName) {
keyword["Category"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Category"], search.CName), ",")
}
// 影视内容分类处理
if strings.Contains(search.ClassTag, "/") {
for _, t := range strings.Split(search.ClassTag, "/") {
if !strings.Contains(keyword["Tag"], t) {
keyword["Tag"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Tag"], t), ",")
}
}
} else if strings.Contains(search.ClassTag, ",") {
for _, t := range strings.Split(search.ClassTag, ",") {
if !strings.Contains(keyword["Tag"], t) {
keyword["Tag"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Tag"], t), ",")
}
}
} else {
if !strings.Contains(keyword["Tag"], search.ClassTag) {
keyword["Tag"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Tag"], search.ClassTag), ",")
}
}
// 如果地区中包含 / 分隔符 则先进行切分处理
if strings.Contains(search.Area, "/") {
for _, s := range strings.Split(search.Area, "/") {
if !strings.Contains(keyword["Area"], strings.TrimSpace(s)) {
keyword["Area"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Area"], s), ",")
}
}
} else if strings.Contains(search.Area, ",") {
for _, s := range strings.Split(search.Area, ",") {
if !strings.Contains(keyword["Area"], strings.TrimSpace(s)) {
keyword["Area"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Area"], s), ",")
}
}
} else {
if !strings.Contains(keyword["Area"], search.Area) {
keyword["Area"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Area"], search.Area), ",")
}
}
// 语言处理
if strings.Contains(search.Language, "/") {
for _, l := range strings.Split(search.Language, "/") {
if !strings.Contains(keyword["Language"], l) {
keyword["Language"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Language"], l), ",")
}
}
} else if strings.Contains(search.Language, ",") {
for _, l := range strings.Split(search.Language, ",") {
if !strings.Contains(keyword["Language"], l) {
keyword["Language"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Language"], l), ",")
}
}
} else {
if !strings.Contains(keyword["Language"], search.Language) {
keyword["Language"] = strings.Trim(fmt.Sprintf("%s,%s", keyword["Language"], search.Language), ",")
}
}
_ = db.Rdb.HMSet(db.Cxt, key, keyword).Err()
}
// BatchSaveSearchInfo 批量保存Search信息
func BatchSaveSearchInfo(list []MovieDetail) {
var infoList []SearchInfo
@@ -362,8 +260,6 @@ func GenerateHashKey[K string | ~int | int64](key K) string {
return fmt.Sprint(h.Sum32())
}
// 处理分类方法0
// ============================采集方案.v1 遗留==================================================
// SaveMoves 保存影片分页请求list

149
server/model/Search.go
View File

@@ -96,6 +96,122 @@ func DelMtPlay(keys []string) {
db.Rdb.Del(db.Cxt, keys...)
}
/*
SearchKeyword 设置search关键字集合(影片分类检索类型数据)
类型, 剧情 , 地区, 语言, 年份, 首字母, 排序
1. 在影片详情缓存到redis时将影片的相关数据进行记录, 存在相同类型则分值加一
2. 通过分值对类型进行排序类型展示到页面
*/
func SaveSearchTag(search SearchInfo) {
// 声明用于存储采集的影片的分类检索信息
//searchMap := make(map[string][]map[string]int)
// Redis中的记录形式 Search:SearchKeys:Pid1:Title Hash
// Redis中的记录形式 Search:SearchKeys:Pid1:xxx Hash
// 获取redis中的searchMap
key := fmt.Sprintf(config.SearchTitle, search.Pid)
searchMap := db.Rdb.HGetAll(db.Cxt, key).Val()
// 是否存储对应分类的map, 如果不存在则缓存一份
if len(searchMap) == 0 {
searchMap["Category"] = "类型"
searchMap["Plot"] = "剧情"
searchMap["Area"] = "地区"
searchMap["Language"] = "语言"
searchMap["Year"] = "年份"
searchMap["Initial"] = "首字母"
searchMap["Sort"] = "排序"
db.Rdb.HMSet(db.Cxt, key, searchMap)
}
// 对searchMap中的各个类型进行处理
for k, _ := range searchMap {
tagKey := fmt.Sprintf(config.SearchTag, search.Pid, k)
tagCount := db.Rdb.ZCard(db.Cxt, tagKey).Val()
switch k {
case "Category":
// 获取 Category 数据, 如果不存在则缓存一份
if tagCount == 0 {
var tags []redis.Z
for _, t := range GetChildrenTree(search.Pid) {
tags = append(tags, redis.Z{Score: float64(-t.Id), Member: t.Name})
}
db.Rdb.ZAdd(db.Cxt, fmt.Sprintf(config.SearchTag, search.Pid, k), tags...)
}
case "Year":
// 获取 Year 数据, 如果不存在则缓存一份
if tagCount == 0 {
var tags []redis.Z
currentYear := time.Now().Year()
for i := 0; i < 12; i++ {
tags = append(tags, redis.Z{Score: float64(currentYear - i), Member: currentYear - i})
}
db.Rdb.ZAdd(db.Cxt, fmt.Sprintf(config.SearchTag, search.Pid, k), tags...)
}
case "Initial":
// 如果不存在 首字母 Tag 数据, 则缓存一份
if tagCount == 0 {
var tags []redis.Z
for i := 65; i <= 90; i++ {
tags = append(tags, redis.Z{Score: float64(90 - i), Member: fmt.Sprintf("%c", i)})
}
db.Rdb.ZAdd(db.Cxt, fmt.Sprintf(config.SearchTag, search.Pid, k), tags...)
}
case "Sort":
if tagCount == 0 {
tags := []redis.Z{
{2, "time"},
{1, "hits"},
{0, "score"},
}
db.Rdb.ZAdd(db.Cxt, fmt.Sprintf(config.SearchTag, search.Pid, k), tags...)
}
case "Plot":
HandleSearchTags(search.ClassTag, tagKey)
case "Area":
HandleSearchTags(search.Area, tagKey)
case "Language":
HandleSearchTags(search.Language, tagKey)
default:
break
}
}
}
func HandleSearchTags(preTags string, k string) {
// 先处理字符串中的空白符 然后对处理前的tag字符串进行分割
preTags = regexp.MustCompile(`[\s\n\r]+`).ReplaceAllString(preTags, "")
f := func(sep string) {
for _, t := range strings.Split(preTags, sep) {
// 获取 tag对应的score
score := db.Rdb.ZScore(db.Cxt, k, t).Val()
// 在原score的基础上+1 重新存入redis中
db.Rdb.ZAdd(db.Cxt, k, redis.Z{Score: score + 1, Member: t})
}
}
switch {
case strings.Contains(preTags, "/"):
f("/")
case strings.Contains(preTags, ","):
f(",")
case strings.Contains(preTags, "，"):
f("，")
case strings.Contains(preTags, "、"):
f("、")
default:
// 获取 tag对应的score
if len(preTags) == 0 || preTags == "其它" {
db.Rdb.ZAdd(db.Cxt, k, redis.Z{Score: 0, Member: preTags})
} else {
score := db.Rdb.ZScore(db.Cxt, k, preTags).Val()
db.Rdb.ZAdd(db.Cxt, k, redis.Z{Score: score + 1, Member: preTags})
}
}
}
// ================================= Spider 数据处理(mysql) =================================
// CreateSearchTable 创建存储检索信息的数据表
@@ -332,6 +448,39 @@ func GetMultiplePlay(siteName, key string) []MovieUrlInfo {
return playList
}
// GetSearchTag 通过影片分类 Pid 返回对应分类的tag信息
func GetSearchTag(pid int64) map[string]interface{} {
res := make(map[string]interface{})
titles := db.Rdb.HGetAll(db.Cxt, fmt.Sprintf(config.SearchTitle, pid)).Val()
for k, v := range titles {
// 通过 k 获取对应的 tag , 并以score进行排序
tags := db.Rdb.ZRevRange(db.Cxt, fmt.Sprintf(config.SearchTag, pid, k), 0, 10).Val()
res[v] = tags
// 过滤分类tag
switch k {
case "Category", "Year", "Initial", "Sort":
tags := db.Rdb.ZRevRange(db.Cxt, fmt.Sprintf(config.SearchTag, pid, k), 0, -1).Val()
res[v] = tags
case "Plot":
tags := db.Rdb.ZRevRange(db.Cxt, fmt.Sprintf(config.SearchTag, pid, k), 0, 10).Val()
res[v] = tags
case "Area":
tags := db.Rdb.ZRevRange(db.Cxt, fmt.Sprintf(config.SearchTag, pid, k), 0, 11).Val()
res[v] = tags
case "Language":
tags := db.Rdb.ZRevRange(db.Cxt, fmt.Sprintf(config.SearchTag, pid, k), 0, 6).Val()
res[v] = tags
default:
break
}
}
return res
}
// ================================= 接口数据缓存 =================================
// DataCache API请求 数据缓存
func DataCache(key string, data map[string]interface{}) {
val, _ := json.Marshal(data)

18
server/plugin/spider/Spider.go
View File

@@ -30,6 +30,7 @@ import (
const (
MainSite = "https://cj.lziapi.com/api.php/provide/vod/"
//MainSite = "https://cj.lzcaiji.com/api.php/provide/vod/"
)
type Site struct {
@@ -39,17 +40,16 @@ type Site struct {
// SiteList 播放源采集站
var SiteList = []Site{
//{"tk", "https://api.tiankongapi.com/api.php/provide/vod"},
//{"yh", "https://m3u8.apiyhzy.com/api.php/provide/vod/"},
//{"zk", "https://api.1080zyku.com/inc/apijson.php"}, 数据格式不规范,不采用
//{"fs", "https://www.feisuzyapi.com/api.php/provide/vod/"},
// 备用采集站
//{"su", "https://subocaiji.com/api.php/provide/vod/at/json"},
//{"bf", "https://bfzyapi.com/api.php/provide/vod/"},
//{"ff", "https://cj.ffzyapi.com/api.php/provide/vod/"},
//{"ff", "https://svip.ffzyapi8.com/api.php/provide/vod/"},
//{"lz", "https://cj.lziapi.com/api.php/provide/vod/"},
//{"ff", "https://cj.ffzyapi.com/api.php/provide/vod/"},
//{"fs", "https://www.feisuzyapi.com/api.php/provide/vod/"},
//{"bf", "https://bfzyapi.com/api.php/provide/vod/"},
{"test", "https://bfzyapi.com/api.php/provide/vod/"},
{"fs", "https://www.feisuzyapi.com/api.php/provide/vod/"},
{"bf", "http://by.bfzyapi.com/api.php/provide/vod/"},
{"kk", "https://kuaikan-api.com/api.php/provide/vod/from/kuaikan"},
}
// StartSpider 执行多源spider
@@ -290,7 +290,7 @@ func StartSpiderRe() {
func GetPageCount(r RequestInfo) (count int, err error) {
// 发送请求获取pageCount
r.Params.Set("ac", "detail")
r.Params.Set("pg", "1")
r.Params.Set("pg", "2")
ApiGet(&r)
// 判断请求结果是否为空, 如果为空直接输出错误并终止
if len(r.Resp) <= 0 {