collect optimization

server/config/DataConfig.go

@@ -15,6 +15,12 @@ const (
 	// MAXGoroutine max goroutine, 执行spider中对协程的数量限制
 	MAXGoroutine = 10
 
+	// FilmSaveCacheThreshold 采集数据保存时直接保存 or 缓存到 redis ( h < 168, 缓存到redis)
+	FilmSaveCacheThreshold = 168
+
+	// FilmScanSize 同步影片数据时每次扫描的数据量
+	FilmScanSize = 50
+
 	FilmPictureUploadDir = "./static/upload/gallery"
 	FilmPictureUrlPath   = "/upload/pic/poster/"
 	FilmPictureAccess    = "/api/upload/pic/poster/"
@@ -29,12 +35,14 @@ const (
 	MovieListInfoKey = "MovieList:Cid%d"
 
 	// MovieDetailKey movie detail影视详情信息 可以
-	MovieDetailKey = "MovieDetail:Cid%d:Id%d"
+	MovieDetailKey = "MovieDetail:Master"
 	// MovieBasicInfoKey 影片基本信息, 简略版本
 	MovieBasicInfoKey = "MovieBasicInfo:Cid%d:Id%d"
 
-	// MultipleSiteDetail 多站点影片信息存储key
-	MultipleSiteDetail = "MultipleSource:%s"
+	// MultipleSiteDetailKey 多站点影片信息存储key
+	MultipleSiteDetailKey = "MovieDetail:Slave:%s"
+
+	MovieDetailTemp = "Temp:MovieDetail:%s"
 
 	// SearchInfoTemp redis暂存检索数据信息
 	SearchInfoTemp = "Search:SearchInfoTemp"
@@ -45,7 +53,7 @@ const (
 	SearchTag = "Search:Pid%d:%s"
 
 	// VirtualPictureKey 待同步图片临时存储 key
-	VirtualPictureKey = "VirtualPicture"
+	VirtualPictureKey = "Temp:VirtualPicture"
 	// MaxScanCount redis Scan 操作每次扫描的数据量, 每次最多扫描300条数据
 	MaxScanCount = 300
 )

server/config/SecurityConfig.go

@@ -1,20 +1,44 @@
 package config
 
-const PrivateKey = `-----BEGIN RSA PRIVATE KEY-----
-MIIBPAIBAAJBANNnshoUaT2gFNrihmFdmC1cBCs1XLFc5Fn3MfNOR3aOGDO0ohXl
-bku6Ir/qITN/yeH5pY34WEcETet3YhESpE8CAwEAAQJBAI7Ekdfg/u26RTtJDd2F
-WrcPVFVl1TKGfERxl08sB0D9HLvUSBfAEg/UpfWSQ57aSJ9b0gVKmDhgF8FymuUV
-v2kCIQDzXXSZ/oeKmqObwad0Fa82IFof3LeZdpbrjyz3w45JDQIhAN5hdmuW+y2w
-UgSy0o4zGFsEG/RBZsvVnSSfkdR47dPLAiEA2XbPNLQu5fnc7NeVDLQ7xsAOCJ6w
-KR/BKGjeI9/JCxkCIQCjMkU0ec2FXxMhzZXFs2uZR6+4FdL5nZ9ABDaCBekK9wIg
-XEfd11qabi9jPrbsOVNZCTk51B7Ug0ZwGyn0BA8Jlo0=
------END RSA PRIVATE KEY-----
+const PrivateKey = `-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCWE8ZtWREHrTpd
+WRu0X2pcDWsW6C2bB268+2vFWNhmLd9P3DOeCYyNDk2HfyZHDzUovhuxsePA+R77
+y+Vl5UvZivci6mS4gqQCSof158j76z3rkChTawlBAs3cKU6GuWCeHq02Z4ajXXQn
+3ekNATVj/grnAC+RaDguHu2ZSXnt4kQvtX7IErnCWmwDw6BXLN2EVMN/+QZBk34y
+woc0OqM/aXr1e4NZJ6hUj8ji+s1ViqGisWM3KRSPX6Z1g3tLvjjwpQ9+BQ2jztvF
+QC25Ysjf/AuITlnjEizW/fTepxXxzjj8GTCnTtonn+dqz4WUoh8lhhf2lvRURIiq
+quv4HcVDAgMBAAECggEANERp2H5rBHHrwEP+9pg+ehXARKeS7omtowqrM31oTFnQ
+isclgPZRaXTgQn7hbWEaAfkHuyC5seTxSf9ugsjTXE3eGV5puqrhBN1BMb7KTBZX
+J9m6eabu/6HC4VjPfPNF93u2lpqClRXAlPvfpQRVPGBnGqGBb5JM8p8Z38drvBGH
+ZehjrkDpcnTESCVqm2wywXXiwwdR7kyl1EZO9lNUwxGutP4IY1owaSfB/yXv9XPW
+q0aQBiDdwMV/Csez8eUKjr1sYEpoIb1WnTigL03Wvaml5MMA5Lrq7E2ejvFamXeq
+GBenoJlJri+4rpLWZUz2QyaMfNitF0aHxeHnLSFzVQKBgQDFpLaHA73UuvStU3lp
+/Ug8kAycoSUjMorY7CHN2qiqqVE5fgEI4tPIa+9/+iMR7ENBT92+TdwBFnm5C+Jp
+zb1fWIkmNcvgCCOB3voVTsBaFVlX1x4uDu0XREQ37tJW/WlcASh9xdjS8VMenc2x
+5L2RC5BZHBHcYsCbEUBO82N6zQKBgQDCY6zMXWxyLxgZbVyXr4BjoP13rVvH2lnr
+yUgb9akE25WO2BRNQeFzlJ3Cbjf+Co5jnTnAXyHW8vcC8EwSIWMgER5tjkrQ829d
+KxZFdfqvfbqDTTiQjTBX60UZPnTQmXZ4vtM/pm0uY9Os4A6s2Z4zsBU14+8Uemwc
+QNdyg9ZgTwKBgAy6wBrQDeDuraWgM7i03HsyF+3anya2MaUpCEBW7fgX8ahPSkfF
+PDizYviOJsYXvnktaK3CHAJh7TMS5eFEpAbJEARLALeMLqVFtDu9F7x1lYyHyWTU
+B0cxWWd01U1PAwMu++ml3YIwktbs+PuVLPRzlzd2DFxya89Q/CBZorWdAoGATrYf
+nSzjIaUyI/GlgYI+AZmJqPEYbedba984BKJHi0EC4TFbnyDjEnMGIlmQ8JwDphBe
+9ydOQ4RkQttcYicExVJJOOFaKBu6Z0ZwITYxiOfnIw2cruOMxLzuYNdqY6EQWLok
+Pey83wDrEGA1cQY5SgKtxAGEqI0dw5753lPF9X8CgYEAgrOVmtQz9AvT5fHjehmt
+yHtEh62NMU6IolwGXOMbhaJNFawKwPuSD5o9pyHiGJewuVVc+xxdaxyJyYp1jbAg
+ip9NFXC9bHAanEhguWcjuCo7UzOgAJAWZLtpi1ehtfydj2aoGEpg3G+oA/A9M2sj
+pbOGmHayWJekUoZktxIYVxo=
+-----END PRIVATE KEY-----
 `
 
-const PublicKey = `-----BEGIN RSA PUBLIC KEY-----
-MEgCQQDTZ7IaFGk9oBTa4oZhXZgtXAQrNVyxXORZ9zHzTkd2jhgztKIV5W5LuiK/
-6iEzf8nh+aWN+FhHBE3rd2IREqRPAgMBAAE=
------END RSA PUBLIC KEY-----
+const PublicKey = `-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAlhPGbVkRB606XVkbtF9q
+XA1rFugtmwduvPtrxVjYZi3fT9wzngmMjQ5Nh38mRw81KL4bsbHjwPke+8vlZeVL
+2Yr3IupkuIKkAkqH9efI++s965AoU2sJQQLN3ClOhrlgnh6tNmeGo110J93pDQE1
+Y/4K5wAvkWg4Lh7tmUl57eJEL7V+yBK5wlpsA8OgVyzdhFTDf/kGQZN+MsKHNDqj
+P2l69XuDWSeoVI/I4vrNVYqhorFjNykUj1+mdYN7S7448KUPfgUNo87bxUAtuWLI
+3/wLiE5Z4xIs1v303qcV8c44/Bkwp07aJ5/nas+FlKIfJYYX9pb0VESIqqrr+B3F
+QwIDAQAB
+-----END PUBLIC KEY-----
 `
 
 const (