Ma/origin
1
0
Fork 0
mirror of https://github.com/duanhf2012/origin.git synced 2026-02-03 22:45:13 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

优化消息队列

Browse Source
This commit is contained in:
origin
2022-12-29 14:54:54 +08:00
parent 66770f07a5
commit 3833884777
3 changed files with 6 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
sysservice/messagequeueservice/CustomerSubscriber.go
View File

@@ -163,9 +163,9 @@ func (cs *CustomerSubscriber) subscribe() bool {
cs.publishToCustomer(topicData)
return true
}
//从持久化数据中来找
topicData = cs.subscriber.dataPersist.FindTopicData(cs.topic, cs.StartIndex, int64(cs.oneBatchQuantity))
topicData = cs.subscriber.dataPersist.FindTopicData(cs.topic, cs.StartIndex, int64(cs.oneBatchQuantity),cs.topicCache[:0])
return cs.publishToCustomer(topicData)
}

2
sysservice/messagequeueservice/MessageQueueService.go
View File

@@ -16,7 +16,7 @@ type QueueDataPersist interface {
OnReceiveTopicData(topic string, topicData []TopicData) //当收到推送过来的数据时
OnPushTopicDataToCustomer(topic string, topicData []TopicData) //当推送数据到Customer时回调
PersistTopicData(topic string, topicData []TopicData, retryCount int) ([]TopicData, []TopicData, bool) //持久化数据,失败则返回false，上层会重复尝试，直到成功，建议在函数中加入次数，超过次数则返回true
FindTopicData(topic string, startIndex uint64, limit int64) []TopicData //查找数据,参数bool代表数据库查找是否成功
FindTopicData(topic string, startIndex uint64, limit int64, topicBuff []TopicData) []TopicData //查找数据,参数bool代表数据库查找是否成功
LoadCustomerIndex(topic string, customerId string) (uint64, bool) //false时代表获取失败，一般是读取错误，会进行重试。如果不存在时，返回(0,true)
GetIndex(topicData *TopicData) uint64 //通过topic数据获取进度索引号
PersistIndex(topic string, customerId string, index uint64) //持久化进度索引号

17
sysservice/messagequeueservice/MongoPersist.go
View File

@@ -20,8 +20,6 @@ type MongoPersist struct {
url string //连接url
dbName string //数据库名称
retryCount int //落地数据库重试次数
topic []TopicData //用于临时缓存
}
const CustomerCollectName = "SysCustomer"
@@ -85,14 +83,6 @@ func (mp *MongoPersist) ReadCfg() error {
return nil
}
func (mp *MongoPersist) getTopicBuff(limit int) []TopicData {
if cap(mp.topic) < limit {
mp.topic = make([]TopicData, limit)
}
return mp.topic[:0]
}
func (mp *MongoPersist) OnExit() {
}
@@ -184,7 +174,7 @@ func (mp *MongoPersist) PersistTopicData(topic string, topicData []TopicData, re
}
// FindTopicData 查找数据
func (mp *MongoPersist) findTopicData(topic string, startIndex uint64, limit int64) ([]TopicData, bool) {
func (mp *MongoPersist) findTopicData(topic string, startIndex uint64, limit int64,topicBuff []TopicData) ([]TopicData, bool) {
s := mp.mongo.TakeSession()
@@ -226,7 +216,6 @@ func (mp *MongoPersist) findTopicData(topic string, startIndex uint64, limit int
}
//序列化返回
topicBuff := mp.getTopicBuff(int(limit))
for i := 0; i < len(res); i++ {
rawData, errM := bson.Marshal(res[i])
if errM != nil {
@@ -261,7 +250,7 @@ func (mp *MongoPersist) getCollectCount(topic string,today string) (int64 ,error
}
// FindTopicData 查找数据
func (mp *MongoPersist) FindTopicData(topic string, startIndex uint64, limit int64) []TopicData {
func (mp *MongoPersist) FindTopicData(topic string, startIndex uint64, limit int64,topicBuff []TopicData) []TopicData {
//某表找不到，一直往前找，找到当前置为止
for days := 1; days <= MaxDays; days++ {
//是否可以跳天
@@ -285,7 +274,7 @@ func (mp *MongoPersist) FindTopicData(topic string, startIndex uint64, limit int
}
//从startIndex开始一直往后查
topicData, isSucc := mp.findTopicData(topic, startIndex, limit)
topicData, isSucc := mp.findTopicData(topic, startIndex, limit,topicBuff)
//有数据或者数据库出错时返回，返回后，会进行下一轮的查询遍历
if len(topicData) > 0 || isSucc == false {
return topicData