Storage optimization

server/plugin/SystemInit/DatabaseInIt.go

@@ -10,6 +10,10 @@ func TableInIt() {
 	system.InitAdminAccount()
 	// 创建 Search Table
 	system.CreateSearchTable()
+	// 创建 MovieDetails Table
+	system.CreateMovieDetailTable()
+	// 创建 SlaveMovieInfo Table
+	system.CreateSlaveMovieInfoTable()
 	// 创建图片信息管理表
 	system.CreateFileTable()
 	// 创建采集失效记录表

server/plugin/SystemInit/SpiderInit.go

@@ -30,12 +30,6 @@ func FilmSourceInit() {
 		{Id: util.GenerateSalt(), Name: "HD(SN)", Uri: `https://suoniapi.com/api.php/provide/vod/from/snm3u8/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false, Interval: 2000},
 		{Id: util.GenerateSalt(), Name: "HD(DB)", Uri: `https://caiji.dbzy.tv/api.php/provide/vod/from/dbm3u8/at/josn/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
 		{Id: util.GenerateSalt(), Name: "HD(IK)", Uri: `https://ikunzyapi.com/api.php/provide/vod/at/json`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
-		//{Id: util.GenerateSalt(), Name: "WX(T2)", Uri: `https://api.wuxianzy.net/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
-		//{Id: util.GenerateSalt(), Name: "OK(BK)", Uri: `https://api.okzy.org/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
-		//{Id: util.GenerateSalt(), Name: "HD(HW)", Uri: `https://cjhwba.com/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false, Interval: 3000},
-		//{Id: util.GenerateSalt(), Name: "HD(lzBk)", Uri: `https://cj.lzcaiji.com/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
-		//{Id: util.GenerateSalt(), Name: "HD(fs)", Uri: `https://www.feisuzyapi.com/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
-		//{Id: util.GenerateSalt(), Name: "HD(bfBk)", Uri: `http://app.bfzyapi.com/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
 	}
 	err := system.SaveCollectSourceList(l)
 	if err != nil {

server/plugin/common/conver/Collect.go

@@ -3,9 +3,11 @@ package conver
 import (
 	"encoding/xml"
 	"log"
+	"regexp"
 	"server/config"
 	"server/model/collect"
 	"server/model/system"
+	"server/plugin/common/util"
 	"strings"
 )
 
@@ -70,36 +72,42 @@ func ConvertFilmDetails(details []collect.FilmDetail) []system.MovieDetail {
 
 // ConvertFilmDetail 将影片详情数据处理转化为 system.MovieDetail
 func ConvertFilmDetail(detail collect.FilmDetail) system.MovieDetail {
+	/*
+		对需数据进行相应的简化处理
+		1.对常见分割符进行统一化处理
+		2.如果演员和导演名单过长,则进行截断, 最多只保留3个
+	*/
+	detail.VodActor = regexp.MustCompile(`[$&#%]`).ReplaceAllString(detail.VodActor, ",")
+
 	md := system.MovieDetail{
-		Id:       detail.VodID,
+		Mid:      detail.VodID,
 		Cid:      detail.TypeID,
 		Pid:      detail.TypeID1,
 		Name:     detail.VodName,
 		Picture:  detail.VodPic,
 		DownFrom: detail.VodDownFrom,
-		MovieDescriptor: system.MovieDescriptor{
-			SubTitle:    detail.VodSub,
-			CName:       detail.TypeName,
-			EnName:      detail.VodEn,
-			Initial:     detail.VodLetter,
-			ClassTag:    detail.VodClass,
-			Actor:       detail.VodActor,
-			Director:    detail.VodDirector,
-			Writer:      detail.VodWriter,
-			Blurb:       detail.VodBlurb,
-			Remarks:     detail.VodRemarks,
-			ReleaseDate: detail.VodPubDate,
-			Area:        detail.VodArea,
-			Language:    detail.VodLang,
-			Year:        detail.VodYear,
-			State:       detail.VodState,
-			UpdateTime:  detail.VodTime,
-			AddTime:     detail.VodTimeAdd,
-			DbId:        detail.VodDouBanID,
-			DbScore:     detail.VodDouBanScore,
-			Hits:        detail.VodHits,
-			Content:     detail.VodContent,
-		},
+		SubTitle: detail.VodSub,
+		CName:    detail.TypeName,
+		EnName:   detail.VodEn,
+		Initial:  detail.VodLetter,
+		ClassTag: detail.VodClass,
+		Actor:    util.TruncateBySep(detail.VodActor, 3),
+		Director: util.TruncateBySep(detail.VodDirector, 2),
+		Writer:   util.TruncateBySep(detail.VodWriter, 2),
+		//Blurb:       detail.VodBlurb,
+		Blurb:       "", // blurb 和 content 内容重复度过高, 且内存占用过高, 所以舍弃简介字段
+		Remarks:     detail.VodRemarks,
+		ReleaseDate: detail.VodPubDate,
+		Area:        detail.VodArea,
+		Language:    detail.VodLang,
+		Year:        detail.VodYear,
+		State:       detail.VodState,
+		UpdateTime:  detail.VodTime,
+		AddTime:     detail.VodTimeAdd,
+		DbId:        detail.VodDouBanID,
+		DbScore:     detail.VodDouBanScore,
+		Hits:        detail.VodHits,
+		Content:     detail.VodContent,
 	}
 	// 通过分割符切分播放源信息  PlaySeparator $$$
 	md.PlayFrom = strings.Split(detail.VodPlayFrom, detail.VodPlayNote)
@@ -111,8 +119,8 @@ func ConvertFilmDetail(detail collect.FilmDetail) system.MovieDetail {
 }
 
 // GenFilmPlayList 处理影片播放地址数据, 只保留m3u8与mp4格式的链接,生成playList
-func GenFilmPlayList(playUrl, separator string) [][]system.MovieUrlInfo {
-	var res [][]system.MovieUrlInfo
+func GenFilmPlayList(playUrl, separator string) system.MoviePlayList {
+	var res system.MoviePlayList
 	if separator != "" {
 		// 1. 通过分隔符切分播放源地址
 		for _, l := range strings.Split(playUrl, separator) {
@@ -120,7 +128,6 @@ func GenFilmPlayList(playUrl, separator string) [][]system.MovieUrlInfo {
 			if strings.Contains(l, ".m3u8") || strings.Contains(l, ".mp4") {
 				// 2. 将每组播放源对应的播放列表信息存储到列表中
 				res = append(res, ConvertPlayUrl(l))
-
 			}
 		}
 	} else {
@@ -134,8 +141,8 @@ func GenFilmPlayList(playUrl, separator string) [][]system.MovieUrlInfo {
 }
 
 // GenAllFilmPlayList 处理影片播放地址数据, 保留全部播放链接,生成playList
-func GenAllFilmPlayList(playUrl, separator string) [][]system.MovieUrlInfo {
-	var res [][]system.MovieUrlInfo
+func GenAllFilmPlayList(playUrl, separator string) system.MoviePlayList {
+	var res system.MoviePlayList
 	if separator != "" {
 		// 1. 通过分隔符切分播放源地址
 		for _, l := range strings.Split(playUrl, separator) {
@@ -150,18 +157,18 @@ func GenAllFilmPlayList(playUrl, separator string) [][]system.MovieUrlInfo {
 }
 
 // ConvertPlayUrl 将单个playFrom的播放地址字符串处理成列表形式
-func ConvertPlayUrl(playUrl string) []system.MovieUrlInfo {
+func ConvertPlayUrl(playUrl string) []system.PlayItem {
 	// 对每个片源的集数和播放地址进行分割 Episode$Link#Episode$Link
-	var l []system.MovieUrlInfo
+	var l []system.PlayItem
 	for _, p := range strings.Split(playUrl, "#") {
 		// 处理 Episode$Link 形式的播放信息
 		if strings.Contains(p, "$") {
-			l = append(l, system.MovieUrlInfo{
+			l = append(l, system.PlayItem{
 				Episode: strings.Split(p, "$")[0],
 				Link:    strings.Split(p, "$")[1],
 			})
 		} else {
-			l = append(l, system.MovieUrlInfo{
+			l = append(l, system.PlayItem{
 				Episode: "(｀・ω・´)",
 				Link:    p,
 			})

server/plugin/common/util/StringUtil.go

@@ -12,6 +12,7 @@ import (
 	"log"
 	"net/url"
 	"regexp"
+	"strings"
 )
 
 // GenerateUUID 生成UUID
@@ -101,6 +102,7 @@ func ValidURL(s string) bool {
 	return true
 }
 
+// ValidPwd 校验密码
 func ValidPwd(s string) error {
 	if len(s) < 8 || len(s) > 12 {
 		return fmt.Errorf("密码长度不符合规范, 必须为8-10位")
@@ -124,3 +126,69 @@ func ValidPwd(s string) error {
 	}
 	return nil
 }
+
+// TruncateBySep 截断字符串,保留指定数量的结果
+func TruncateBySep(s string, limit int) string {
+	// 如果保留数量小于等于0则返回空值
+	if limit <= 0 {
+		return ""
+	}
+	// 先强制对不同的分割符进行统一替换为 ,
+	s = regexp.MustCompile(`[$&#%]`).ReplaceAllString(s, ",")
+	// 使用 strings.Split 分割字符串
+	// Split 会在分隔符连续出现或出现在首尾时产生空字符串，这通常符合预期
+	parts := strings.Split(s, ",")
+	// 片段数量小于或等于限制，直接返回原字符串
+	// 返回原字符串是为了保留原始的格式（比如末尾是否有分隔符）
+	// 即使不截断也重新 Join 一遍（去除多余的空片段等)
+	return strings.Join(parts[:limit], ",")
+}
+
+// CleanFilmName 清洗影片名称，只保留主体
+func CleanFilmName(name string) string {
+	if name == "" {
+		return ""
+	}
+	// 1. 去除常见的前缀 (方括号、圆括号内的内容) 匹配 [xxx], 【xxx】, (xxx), （xxx）
+	//rePrefix := regexp.MustCompile(`^\s*[\[【\(（][^\]】\)）]*[\]】\)）]\s*`)
+	//for rePrefix.MatchString(name) {
+	//	name = rePrefix.ReplaceAllString(name, "")
+	//}
+	// 2.定义需要清洗的特殊标识关键字集合
+	var noisePatterns = []string{
+		`第 [零一二三四五六七八九十\d]+ 季`, `第 [零一二三四五六七八九十\d]+ 话`, `第 [零一二三四五六七八九十\d]+ 集`,
+		`Season\s*\d+`, `S\d+`, `Ep\d+`, `\d{1,3}\s*(话 | 集)`,
+		`\s+(II|III|IV|V|VI|VII|VIII|IX|X)\s*$`,
+		`剧场版`, `电影版`, `OVA`, `OAD`, `SP`, `特别篇`, `总集篇`, `外传`, `序`, `破`, `急`, `终章`,
+		`\d{3,4}[Pp]`, `HD`, `FHD`, `UHD`, `4K`, `BD`, `BluRay`, `BDRip`, `HEVC`, `H264`, `H265`,
+		`GB`, `MB`, `MP4`, `MKV`, `AVI`, `RMVB`,
+		`字幕组`, `动漫`, `动画`, `新版`, `重制版`, `连载`, `更新`, `全集`, `合集`,
+		`Uncensored`, `NoCen`, `Dubbed`, `Subbed`, `Raw`, `生肉`, `熟肉`,
+	}
+	// 3. 处理拼接完整的正则表达式
+	fullPattern := `(?i)(?:\s+|\.+|_+|-+) (` + strings.Join(noisePatterns, "|") + `).*$`
+	cutRegex := regexp.MustCompile(fullPattern)
+	// 去除满足匹配集的子串
+	name = cutRegex.ReplaceAllString(name, "")
+	// 特殊处理 "之" 字结构 (仅当 "之" 后紧跟噪音词时切除)	之\s*(噪音词)
+	reRegex := regexp.MustCompile(`(?i) 之\s* (` + strings.Join(noisePatterns, "|") + `).*$`)
+	name = reRegex.ReplaceAllString(name, "")
+
+	// 修剪 - 去除末尾残留的符号和空白
+	name = strings.TrimRight(name, " \t\n\r._-])：:")
+
+	return name
+}
+
+// FormatSpecialChar 格式化特殊字符, 统一替换为逗号
+func FormatSpecialChar(src string) string {
+	// 执行替换
+	return strings.Map(func(r rune) rune {
+		switch r {
+		case '#', '/', '$', '&', '%', '^', '*', '-':
+			return ','
+		default:
+			return r
+		}
+	}, src)
+}

server/plugin/db/mysql.go

@@ -1,11 +1,12 @@
 package db
 
 import (
+	"server/config"
+
 	"gorm.io/driver/mysql"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
 	"gorm.io/gorm/schema"
-	"server/config"
 )
 
 var Mdb *gorm.DB
@@ -25,7 +26,8 @@ func InitMysql() (err error) {
 			SingularTable: true, //是否使用 结构体名称作为表名 (关闭自动变复数)
 			//NameReplacer:  strings.NewReplacer("spider_", ""), // 替表名和字段中的 Me 为 空
 		},
-		Logger: logger.Default.LogMode(logger.Info), //设置日志级别为Info
+		Logger: logger.Default.LogMode(logger.Warn), //设置日志级别为Info
+		//Logger: logger.Default.LogMode(logger.Info), //设置日志级别为Info
 	})
 	return
 }