sql optimize

2026-05-01 07:27:29 +08:00 · 2026-04-23 22:42:05 +08:00
parent 3fd906a73e
commit 7a578dd642
14 changed files with 200 additions and 97 deletions
--- a/server/model/system/Movies.go
+++ b/server/model/system/Movies.go
@@ -193,8 +193,9 @@ func AddSlaveMovieInfoIndex() {
 	if !db.Mdb.Migrator().HasIndex(&s, "idx_mid") {
 		// 添加索引
 		//db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_sid ON %s (sid DESC)", tableName))
-		db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_mid ON %s (mid DESC)", tableName))
-		db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_dbId ON %s (db_id DESC", tableName))
+		//db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_mid ON %s (mid DESC)", tableName))
+		//db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_dbId ON %s (db_id DESC, sid DESC)", tableName))
+		db.Mdb.Exec(fmt.Sprintf("ALTER TABLE %s ADD INDEX idx_mid (mid DESC),ADD INDEX idx_ds (db_id DESC, sid DESC)", tableName))
 	}
 }

@@ -478,7 +479,7 @@ func SyncMovieDetail(sid string, grade SourceGrade, mode int) {
 	case MasterCollect:
 		// 循环扫描detail信息, 存储完成后进行删除
 		for {
-			vs, nextCursor, err := db.Rdb.HScan(db.Cxt, config.MovieDetailKey, cursor, "", config.FilmScanSize).Result()
+			vs, nextCursor, err := db.Rdb.HScan(db.Cxt, config.MovieDetailKey, cursor, "*", config.FilmScanSize).Result()
 			if err != nil {
 				log.Println("ScanMovieDetail Failed: ", err)
 			}
@@ -581,7 +582,7 @@ func GetDetailByMid(mid int64) MovieDetail {
 				log.Println("Find BasicInfo Failed: ", err)
 				return m
 			}
-			//// 执行本地图片匹配
+			// 执行本地图片匹配
 			ReplaceDetailPic(&m)
 			return m
 		}
@@ -734,20 +735,25 @@ func GetRelateMovieBasicInfo(search SearchInfo, page *Page) []MovieBasicInfo {
 	sql = fmt.Sprintf(`select mid from %s where MATCH(name, sub_title) AGAINST('%s') AND cid=%d AND search.deleted_at IS NULL union`, search.TableName(), name, search.Cid)

 	// 添加其他相似匹配规则 同属二级分类
-	sql = fmt.Sprintf(`%s (select mid from %s where cid=%d AND `, sql, search.TableName(), search.Cid)
+	sql = fmt.Sprintf(`%s (select mid from %s where cid=%d AND`, sql, search.TableName(), search.Cid)
 	// 根据剧情标签查找相似影片, classTag 使用的分隔符为 , | /首先去除 classTag 中包含的所有空格
 	search.ClassTag = strings.ReplaceAll(search.ClassTag, " ", "")
 	// 如果 classTag 中包含分割符则进行拆分匹配
-	cl := strings.Split(util.FormatSpecialChar(search.ClassTag), ",")
-	if len(cl) > 0 {
-		s := "("
-		for _, c := range cl {
-			s = fmt.Sprintf(`%s class_tag like "%%%s%%" OR`, s, c)
-		}
-		sql = fmt.Sprintf("%s %s)", sql, strings.TrimSuffix(s, "OR"))
-	} else {
-		sql = fmt.Sprintf(`%s class_tag like "%%%s%%"`, sql, search.ClassTag)
-	}
+	//cl := strings.Split(util.FormatSpecialChar(search.ClassTag), ",")
+	// 将
+	search.ClassTag = strings.ReplaceAll(util.FormatSpecialChar(search.ClassTag), ",", " ")
+	//if len(cl) > 0 {
+	//	s := "("
+	//	for _, c := range cl {
+	//		//s = fmt.Sprintf(`%s class_tag like "%%%s%%" OR`, s, c)
+	//		s = fmt.Sprintf(`%s class_tag like "%%%s%%" OR`, s, c)
+	//	}
+	//	sql = fmt.Sprintf("%s %s)", sql, strings.TrimSuffix(s, "OR"))
+	//	sql = fmt.Sprintf("%s %s)", sql, strings.TrimSuffix(s, "OR"))
+	//} else {
+	//	sql = fmt.Sprintf(`%s class_tag like "%%%s%%"`, sql, search.ClassTag)
+	//}
+	sql = fmt.Sprintf(`%s MATCH(class_tag) AGAINST('%s')`, sql, search.ClassTag)
 	// 除名称外的相似影片使用随机排序
 	//sql = fmt.Sprintf("%s ORDER BY RAND() limit %d,%d)", sql, page.Current, page.PageSize)
 	sql = fmt.Sprintf("%s AND search.deleted_at IS NULL limit %d,%d)", sql, page.Current, page.PageSize)
@@ -775,6 +781,7 @@ func GetMultiplePlay(mIds []string, dbId int64) []SlaveMovieInfo {
 			l = append(l, s)
 			continue
 		}
+		// 如果匹配失败则使用name生成的mIds获取数据
 		for _, mid := range mIds {
 			// 初始化临时变量 SlaveMovieInfo
 			if s = GetSlaveDetailInCache(c.Id, mid); s.Mid != "" {
@@ -791,13 +798,25 @@ func GetMultiplePlay(mIds []string, dbId int64) []SlaveMovieInfo {
 			//l = append(l, s)
 			//break
 		}
-		// 如果迭代完s依旧为空,则去mysql中进行匹配
+
+		// Redis中没有匹配到对应数据, 则去slave_info表中获取数据
+		//如果 dbID 不为0 则优先使用sid 和 dbId 去mysql中锁定对应数据
 		if s.Mid == "" {
-			//if err := db.Mdb.Model(&SlaveMovieInfo{}).Select("sid, play_list").Where("sid = ? AND (mid IN (?) OR db_id = ?)", c.Id, mIds, dbId).First(&s).Error; err != nil {
-			//mq := db.Mdb.Model(&SlaveMovieInfo{}).Select("sid, play_list").Where("sid = ? AND mid IN (?)", c.Id, dbId).Limit(1)
-			//dq := db.Mdb.Model(&SlaveMovieInfo{}).Select("sid, play_list").Where("sid = ? AND db_id = ?", c.Id, dbId).Limit(1)
-			if err := db.Mdb.Raw("(SELECT sid, play_list FROM `slave_infos` WHERE sid = ? AND mid IN (?)) UNION ALL (SELECT sid, play_list FROM `slave_infos` WHERE sid = ? AND db_id = ? ORDER BY `slave_infos`.`id` LIMIT 1) LIMIT 1", c.Id, mIds, c.Id, dbId).First(&s).Error; err != nil {
-				log.Println("GetMultiplePlay Failed: ", err)
+			//if err := db.Mdb.Select("sid, play_list").Where("sid = ? AND db_id = ?", c.Id, dbId).Last(&s).Error; err != nil {
+			//	log.Println("GetMultiplePlay Failed: ", err)
+			//} else {
+			//	continue
+			//}
+			if err := db.Mdb.Select("sid, play_list").Where("sid = ? AND db_id = ?", c.Id, dbId).Last(&s).Error; err == nil {
+				l = append(l, s)
+				continue
+			}
+		}
+		// 如果db_id依旧获取失败, 则使用mIds进行最后的获取
+		if s.Mid == "" {
+			//if err := db.Mdb.Raw("(SELECT sid, play_list FROM `slave_infos` WHERE sid = ? AND mid IN (?)) UNION ALL (SELECT sid, play_list FROM `slave_infos` WHERE sid = ? AND db_id = ? ORDER BY `slave_infos`.`id` LIMIT 1) LIMIT 1", c.Id, mIds, c.Id, dbId).First(&s).Error; err != nil {
+			if err := db.Mdb.Select("sid, play_list").Where("sid = ? AND mid IN (?)", c.Id, mIds).Last(&s).Error; err != nil {
+				//log.Println("GetMultiplePlay Failed: ", err)
 				continue
 			}
 			l = append(l, s)
--- a/server/model/system/Search.go
+++ b/server/model/system/Search.go
@@ -84,7 +84,11 @@ func AddSearchIndex() {
 	//db.Mdb.Exec(fmt.Sprintf("CREATE INDEX idx_year ON %s (year DESC)", tableName))

 	// 一次执行完所有索引的创建
-	db.Mdb.Exec(fmt.Sprintf("ALTER TABLE %s ADD UNIQUE INDEX idx_mid (mid DESC), ADD INDEX idx_pid (pid), ADD INDEX idx_cid (cid), ADD INDEX idx_time (update_stamp DESC), ADD INDEX idx_hits (hits DESC), ADD INDEX idx_score (score DESC), ADD INDEX idx_release (release_stamp DESC), ADD INDEX idx_year (year DESC), ADD FULLTEXT INDEX idx_names (name, sub_title) WITH PARSER ngram", tableName))
+	db.Mdb.Exec(fmt.Sprintf("ALTER TABLE %s ADD UNIQUE INDEX idx_mid (mid DESC), ADD INDEX idx_pid (pid), ADD INDEX idx_cid (cid), ADD INDEX idx_time (update_stamp DESC),"+
+		" ADD INDEX idx_hits (hits DESC), ADD INDEX idx_score (score DESC), ADD INDEX idx_release (release_stamp DESC), ADD INDEX idx_year (year DESC), "+
+		"ADD FULLTEXT INDEX idx_names (name, sub_title) WITH PARSER ngram", tableName))
+	// 全文索引无法同时创建需要分别创建
+	db.Mdb.Exec(fmt.Sprintf("ALTER TABLE %s ADD FULLTEXT INDEX idx_tags (class_tag) WITH PARSER ngram", tableName))

 }

@@ -374,12 +378,14 @@ func SearchFilmKeyword(keyword string, page *Page) []int64 {
 	var ids []int64
 	// 1. 先统计搜索满足条件的数据量
 	var count int64
-	db.Mdb.Model(&SearchInfo{}).Where("name LIKE ?", fmt.Sprint(`%`, keyword, `%`)).Or("sub_title LIKE ?", fmt.Sprint(`%`, keyword, `%`)).Count(&count)
+	//db.Mdb.Model(&SearchInfo{}).Where("name LIKE ?", fmt.Sprint(`%`, keyword, `%`)).Or("sub_title LIKE ?", fmt.Sprint(`%`, keyword, `%`)).Count(&count)
+	// 搜索时使用精确匹配,防止默认的分词
+	db.Mdb.Model(&SearchInfo{}).Where("MATCH(name,sub_title) AGAINST(? IN BOOLEAN MODE)", fmt.Sprintf(`"%s"`, keyword)).Count(&count)
 	page.Total = int(count)
 	page.PageCount = int((page.Total + page.PageSize - 1) / page.PageSize)
 	// 2. 获取满足条件的数据
 	db.Mdb.Model(&SearchInfo{}).Limit(page.PageSize).Offset((page.Current-1)*page.PageSize).Select("mid").
-		Where("name LIKE ?", fmt.Sprintf(`%%%s%%`, keyword)).Or("sub_title LIKE ?", fmt.Sprintf(`%%%s%%`, keyword)).Order("year DESC, update_stamp DESC").Find(&ids)
+		Where("MATCH(name,sub_title) AGAINST(? IN BOOLEAN MODE)", fmt.Sprintf(`"%s"`, keyword)).Order("year DESC, update_stamp DESC").Find(&ids)

 	return ids
 }
@@ -458,7 +464,11 @@ func HandleTagStr(title string, tags ...string) []map[string]string {
 // GetSearchInfosByTags 查询满足searchTag条件的影片分页数据
 func GetSearchInfosByTags(st SearchTagsVO, page *Page) []int64 {
 	// 准备查询语句的条件
-	qw := db.Mdb.Model(&SearchInfo{})
+	var s SearchInfo
+	qw := db.Mdb.Table(s.TableName()).Select(fmt.Sprintf("%s.mid", s.TableName()))
+	// 生成左连接的子查询
+	//qweFlag := false
+	//qwe := db.Mdb.Table(s.TableName()).Select("mid")
 	// 通过searchTags的非空属性值, 拼接对应的查询条件
 	t := reflect.TypeOf(st)
 	v := reflect.ValueOf(st)
@@ -477,6 +487,7 @@ func GetSearchInfosByTags(st SearchTagsVO, page *Page) []int64 {
 			switch k {
 			case "pid", "cid", "year":
 				qw = qw.Where(fmt.Sprintf("%s = ?", k), value)
+				//qwe = qwe.Where(fmt.Sprintf("%s = ?", k), value)
 			case "area", "language":
 				if strings.EqualFold(value.(string), "其它") {
 					qw = qw.Where(fmt.Sprintf("%s NOT IN ?", k), ts)
@@ -488,9 +499,15 @@ func GetSearchInfosByTags(st SearchTagsVO, page *Page) []int64 {
 					for _, t := range ts {
 						qw = qw.Where("class_tag NOT LIKE ?", fmt.Sprintf("%%%v%%", t))
 					}
+					// 通过cl整合需要排除的tag
+					//cl := strings.Join(ts, " ")
+					//qweFlag = true
+					//qwe = qwe.Where("MATCH(class_tag) AGAINST(? IN BOOLEAN MODE)", fmt.Sprintf("%s", cl))
+					//db.Mdb.Table(fmt.Sprintf("%s as s1", s.TableName()))
 					break
 				}
-				qw = qw.Where("class_tag LIKE ?", fmt.Sprintf("%%%v%%", value))
+				qw = qw.Where("MATCH(class_tag) AGAINST(? IN BOOLEAN MODE)", fmt.Sprintf(`"%v"`, value))
+				//qw = qw.Where("class_tag LIKE ?", fmt.Sprintf("%%%v%%", value))
 			case "sort":
 				if strings.EqualFold(value.(string), "release_stamp") {
 					qw.Order(fmt.Sprintf("year DESC ,%v DESC", value))
@@ -502,12 +519,15 @@ func GetSearchInfosByTags(st SearchTagsVO, page *Page) []int64 {
 			}
 		}
 	}
+	//if qweFlag {
+	//	qw = qw.Joins("LEFT JOIN (?) as s2 ON search.mid = s2.mid", qwe).Where("s2.mid IS NULL")
+	//}

 	// 返回分页参数
 	GetPage(qw, page)
 	// 查询具体的searchInfo 分页数据
 	var ids []int64
-	if err := qw.Select("mid").Limit(page.PageSize).Offset((page.Current - 1) * page.PageSize).Find(&ids).Error; err != nil {
+	if err := qw.Limit(page.PageSize).Offset((page.Current - 1) * page.PageSize).Find(&ids).Error; err != nil {
 		log.Println(err)
 		return nil
 	}
@@ -547,7 +567,7 @@ func GetSearchPage(s SearchVo) []SearchInfo {
 	query := db.Mdb.Model(&SearchInfo{})
 	// 如果参数不为空则追加对应查询条件
 	if s.Name != "" {
-		query = query.Where("name LIKE ?", fmt.Sprintf("%%%s%%", s.Name))
+		query = query.Where("MATCH(name, sub_title) AGAINST(? IN BOOLEAN MODE)", fmt.Sprintf(`"%s"`, s.Name))
 	}
 	// 分类ID为负数则默认不追加该条件
 	if s.Cid > 0 {
@@ -556,7 +576,7 @@ func GetSearchPage(s SearchVo) []SearchInfo {
 		query = query.Where("pid = ?", s.Pid)
 	}
 	if s.Plot != "" {
-		query = query.Where("class_tag LIKE ?", fmt.Sprintf("%%%s%%", s.Plot))
+		query = query.Where("MATCH(class_tag) AGAINST(?)", s.Plot)
 	}
 	if s.Area != "" {
 		query = query.Where("area = ?", s.Area)
@@ -690,7 +710,7 @@ func FindFilmIds(params map[string]string, page *Page) ([]int64, error) {
 				query = query.Where("cid = ?", cid)
 			}
 		case "wd":
-			query = query.Where("name like ?", fmt.Sprintf("%%%s%%", v))
+			query = query.Where("MATCH(name, sub_title) AGAINST(?)", v)
 		case "h":
 			if h, err := strconv.ParseInt(v, 10, 64); err == nil {
 				query = query.Where("update_stamp >= ?", time.Now().Unix()-h*3600)