slow query optimization

server/model/system/Movies.go

@@ -192,7 +192,7 @@ func AddSlaveMovieInfoIndex() {
 	// 如果不存在索引则创建对应索引
 	if !db.Mdb.Migrator().HasIndex(&s, "idx_mid") {
 		// 添加索引
-		db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_sid ON %s (sid DESC)", tableName))
+		//db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_sid ON %s (sid DESC)", tableName))
 		db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_mid ON %s (mid DESC)", tableName))
 		db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_dbId ON %s (db_id DESC", tableName))
 	}
@@ -730,7 +730,8 @@ func GetRelateMovieBasicInfo(search SearchInfo, page *Page) []MovieBasicInfo {
 
 	// 优先进行名称相似匹配, 先对影片名称进行精简, 只保留主体用于匹配同系列影片
 	name := util.CleanFilmName(search.Name)
-	sql = fmt.Sprintf(`select mid from %s where (name LIKE "%%%s%%" or sub_title LIKE "%%%[2]s%%") AND cid=%d AND search.deleted_at IS NULL union`, search.TableName(), name, search.Cid)
+	//sql = fmt.Sprintf(`select mid from %s where (name LIKE "%%%s%%" or sub_title LIKE "%%%[2]s%%") AND cid=%d AND search.deleted_at IS NULL union`, search.TableName(), name, search.Cid)
+	sql = fmt.Sprintf(`select mid from %s where MATCH(name, sub_title) AGAINST('%s') AND cid=%d AND search.deleted_at IS NULL union`, search.TableName(), name, search.Cid)
 
 	// 添加其他相似匹配规则 同属二级分类
 	sql = fmt.Sprintf(`%s (select mid from %s where cid=%d AND `, sql, search.TableName(), search.Cid)
@@ -792,7 +793,10 @@ func GetMultiplePlay(mIds []string, dbId int64) []SlaveMovieInfo {
 		}
 		// 如果迭代完s依旧为空,则去mysql中进行匹配
 		if s.Mid == "" {
-			if err := db.Mdb.Model(&SlaveMovieInfo{}).Select("sid, play_list").Where("sid = ? AND (mid IN (?) OR db_id = ?)", c.Id, mIds, dbId).First(&s).Error; err != nil {
+			//if err := db.Mdb.Model(&SlaveMovieInfo{}).Select("sid, play_list").Where("sid = ? AND (mid IN (?) OR db_id = ?)", c.Id, mIds, dbId).First(&s).Error; err != nil {
+			//mq := db.Mdb.Model(&SlaveMovieInfo{}).Select("sid, play_list").Where("sid = ? AND mid IN (?)", c.Id, dbId).Limit(1)
+			//dq := db.Mdb.Model(&SlaveMovieInfo{}).Select("sid, play_list").Where("sid = ? AND db_id = ?", c.Id, dbId).Limit(1)
+			if err := db.Mdb.Raw("(SELECT sid, play_list FROM `slave_infos` WHERE sid = ? AND mid IN (?)) UNION ALL (SELECT sid, play_list FROM `slave_infos` WHERE sid = ? AND db_id = ? ORDER BY `slave_infos`.`id` LIMIT 1) LIMIT 1", c.Id, mIds, c.Id, dbId).First(&s).Error; err != nil {
 				log.Println("GetMultiplePlay Failed: ", err)
 				continue
 			}

server/model/system/Search.go

@@ -74,14 +74,17 @@ func AddSearchIndex() {
 	var s SearchInfo
 	tableName := s.TableName()
 	// 添加索引
-	db.Mdb.Exec(fmt.Sprintf("CREATE UNIQUE INDEX idx_mid ON %s (mid)", tableName))
-	db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_pid ON %s (pid)", tableName))
-	db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_cid ON %s (cid)", tableName))
-	db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_time ON %s (update_stamp DESC)", tableName))
-	db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_hits ON %s (hits DESC)", tableName))
-	db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_score ON %s (score DESC)", tableName))
-	db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_release ON %s (release_stamp DESC)", tableName))
-	db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_year ON %s (year DESC)", tableName))
+	//db.Mdb.Exec(fmt.Sprintf("CREATE UNIQUE INDEX idx_mid ON %s (mid)", tableName)
+	//db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_pid ON %s (pid)", tableName))
+	//db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_cid ON %s (cid)", tableName))
+	//db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_time ON %s (update_stamp DESC)", tableName))
+	//db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_hits ON %s (hits DESC)", tableName))
+	//db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_score ON %s (score DESC)", tableName))
+	//db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_release ON %s (release_stamp DESC)", tableName))
+	//db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_year ON %s (year DESC)", tableName))
+
+	// 一次执行完所有索引的创建
+	db.Mdb.Exec(fmt.Sprintf("ALTER TABLE %s ADD UNIQUE INDEX idx_mid (mid DESC), ADD INDEX idx_pid (pid), ADD INDEX idx_cid (cid), ADD INDEX idx_time (update_stamp DESC), ADD INDEX idx_hits (hits DESC), ADD INDEX idx_score (score DESC), ADD INDEX idx_release (release_stamp DESC), ADD INDEX idx_year (year DESC), ADD FULLTEXT INDEX idx_names (name, sub_title) WITH PARSER ngram", tableName))
 
 }
 
@@ -289,7 +292,7 @@ func BatchSaveOrUpdate(ml []MovieDetail) error {
 			if err != nil {
 				log.Println("Save Search Info error: ", err)
 			}
-			break
+			continue
 		}
 		// 如果不存在对应信息则保存一份tag
 		BatchHandleSearchTag(s)

server/plugin/SystemInit/SpiderInit.go

@@ -25,6 +25,7 @@ func FilmSourceInit() {
 		{Id: util.GenerateSalt(), Name: "HD(BF)", Uri: `https://bfzyapi.com/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false, Interval: 2500},
 		{Id: util.GenerateSalt(), Name: "HD(FF)", Uri: `http://cj.ffzyapi.com/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
 		{Id: util.GenerateSalt(), Name: "HD(OK)", Uri: `https://api.okzyw.net/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
+		{Id: util.GenerateSalt(), Name: "HD(MD)", Uri: `https://www.mdzyapi.com/api.php/provide/vod/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
 		{Id: util.GenerateSalt(), Name: "HD(LY)", Uri: `https://360zy.com/api.php/provide/vod/at/json`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},
 		{Id: util.GenerateSalt(), Name: "HD(SN)", Uri: `https://suoniapi.com/api.php/provide/vod/from/snm3u8/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false, Interval: 2000},
 		{Id: util.GenerateSalt(), Name: "HD(DB)", Uri: `https://caiji.dbzy.tv/api.php/provide/vod/from/dbm3u8/at/josn/`, ResultModel: system.JsonResult, Grade: system.SlaveCollect, SyncPictures: false, CollectType: system.CollectVideo, State: false},

server/plugin/common/util/StringUtil.go

@@ -156,10 +156,13 @@ func CleanFilmName(name string) string {
 	//for rePrefix.MatchString(name) {
 	//	name = rePrefix.ReplaceAllString(name, "")
 	//}
+
+	// 去除名称末尾的空格以及后续内容
+	name = regexp.MustCompile(`\s+\S*$`).ReplaceAllString(name, "")
 	// 2.定义需要清洗的特殊标识关键字集合
 	var noisePatterns = []string{
-		`第 [零一二三四五六七八九十\d]+ 季`, `第 [零一二三四五六七八九十\d]+ 话`, `第 [零一二三四五六七八九十\d]+ 集`,
-		`Season\s*\d+`, `S\d+`, `Ep\d+`, `\d{1,3}\s*(话 | 集)`,
+		`第[零一二三四五六七八九十\d]+季`, `第[零一二三四五六七八九十\d]+话`, `第[零一二三四五六七八九十\d]+集`,
+		`Season\s*\d+`, `S\d+`, `Ep\d+`, `\d{1,3}\s*(话|集)`,
 		`\s+(II|III|IV|V|VI|VII|VIII|IX|X)\s*$`,
 		`剧场版`, `电影版`, `OVA`, `OAD`, `SP`, `特别篇`, `总集篇`, `外传`, `序`, `破`, `急`, `终章`,
 		`\d{3,4}[Pp]`, `HD`, `FHD`, `UHD`, `4K`, `BD`, `BluRay`, `BDRip`, `HEVC`, `H264`, `H265`,
@@ -168,7 +171,7 @@ func CleanFilmName(name string) string {
 		`Uncensored`, `NoCen`, `Dubbed`, `Subbed`, `Raw`, `生肉`, `熟肉`,
 	}
 	// 3. 处理拼接完整的正则表达式
-	fullPattern := `(?i)(?:\s+|\.+|_+|-+) (` + strings.Join(noisePatterns, "|") + `).*$`
+	fullPattern := `(?i)(?:\s*|\.+|_+|-+)*(` + strings.Join(noisePatterns, "|") + `).*$`
 	cutRegex := regexp.MustCompile(fullPattern)
 	// 去除满足匹配集的子串
 	name = cutRegex.ReplaceAllString(name, "")

server/plugin/db/mysql.go

@@ -26,9 +26,9 @@ func InitMysql() (err error) {
 			SingularTable: true, //是否使用 结构体名称作为表名 (关闭自动变复数)
 			//NameReplacer:  strings.NewReplacer("spider_", ""), // 替表名和字段中的 Me 为 空
 		},
-		//Logger: logger.Default.LogMode(logger.Warn), //设置日志级别为Info
-		//Logger: logger.Default.LogMode(logger.Info), //设置日志级别为Info
-		Logger: logger.Default.LogMode(logger.Error), //设置日志级别为Info
+		//Logger: logger.Default.LogMode(logger.Warn), //设置日志级别为 Warn
+		Logger: logger.Default.LogMode(logger.Info), //设置日志级别为Info
+		//Logger: logger.Default.LogMode(logger.Error), //设置日志级别为 Error
 	})
 	return
 }