优化消息队列服务持久化

concurrent/concurrent.go

@@ -54,16 +54,6 @@ func (c *Concurrent) AsyncDoByQueue(queueId int64, fn func() bool, cb func(err e
 		return
 	}
 
-	if len(c.tasks) > cap(c.tasks) {
-		log.SError("tasks channel is full")
-		if cb != nil {
-			c.pushAsyncDoCallbackEvent(func(err error) {
-				cb(errors.New("tasks channel is full"))
-			})
-		}
-		return
-	}
-
 	if fn == nil {
 		c.pushAsyncDoCallbackEvent(cb)
 		return
@@ -75,6 +65,14 @@ func (c *Concurrent) AsyncDoByQueue(queueId int64, fn func() bool, cb func(err e
 
 	select {
 	case c.tasks <- task{queueId, fn, cb}:
+	default:
+		log.SError("tasks channel is full")
+		if cb != nil {
+			c.pushAsyncDoCallbackEvent(func(err error) {
+				cb(errors.New("tasks channel is full"))
+			})
+		}
+		return
 	}
 }
 

concurrent/dispatch.go

@@ -12,7 +12,7 @@ import (
 	"github.com/duanhf2012/origin/util/queue"
 )
 
-var idleTimeout = 2 * time.Second
+var idleTimeout = int64(2 * time.Second)
 const maxTaskQueueSessionId = 10000
 
 type dispatch struct {
@@ -47,7 +47,7 @@ func (d *dispatch) open(minGoroutineNum int32, maxGoroutineNum int32, tasks chan
 
 func (d *dispatch) run() {
 	defer d.waitDispatch.Done()
-	timeout := time.NewTimer(idleTimeout)
+	timeout := time.NewTimer(time.Duration(atomic.LoadInt64(&idleTimeout)))
 
 	for {
 		select {
@@ -65,9 +65,9 @@ func (d *dispatch) run() {
 			case <-timeout.C:
 				d.processTimer()
 				if atomic.LoadInt32(&d.minConcurrentNum) == -1 && len(d.tasks) == 0 {
-					idleTimeout = time.Millisecond * 10
+					atomic.StoreInt64(&idleTimeout,int64(time.Millisecond * 10))
 				}
-				timeout.Reset(idleTimeout)
+				timeout.Reset(time.Duration(atomic.LoadInt64(&idleTimeout)))
 			}
 		}
 
@@ -80,7 +80,7 @@ func (d *dispatch) run() {
 }
 
 func (d *dispatch) processTimer() {
-	if d.idle == true && d.workerNum > d.minConcurrentNum {
+	if d.idle == true && d.workerNum > atomic.LoadInt32(&d.minConcurrentNum) {
 		d.processIdle()
 	}
 

network/tcp_conn.go

@@ -1,11 +1,12 @@
 package network
 
 import (
+	"errors"
 	"github.com/duanhf2012/origin/log"
 	"net"
 	"sync"
+	"sync/atomic"
 	"time"
-	"errors"
 )
 
 type ConnSet map[net.Conn]struct{}
@@ -14,7 +15,7 @@ type TCPConn struct {
 	sync.Mutex
 	conn      net.Conn
 	writeChan chan []byte
-	closeFlag bool
+	closeFlag int32
 	msgParser *MsgParser
 }
 
@@ -49,7 +50,7 @@ func newTCPConn(conn net.Conn, pendingWriteNum int, msgParser *MsgParser,writeDe
 		conn.Close()
 		tcpConn.Lock()
 		freeChannel(tcpConn)
-		tcpConn.closeFlag = true
+		atomic.StoreInt32(&tcpConn.closeFlag,1)
 		tcpConn.Unlock()
 	}()
 
@@ -60,9 +61,9 @@ func (tcpConn *TCPConn) doDestroy() {
 	tcpConn.conn.(*net.TCPConn).SetLinger(0)
 	tcpConn.conn.Close()
 
-	if !tcpConn.closeFlag {
+	if atomic.LoadInt32(&tcpConn.closeFlag)==0 {
 		close(tcpConn.writeChan)
-		tcpConn.closeFlag = true
+		atomic.StoreInt32(&tcpConn.closeFlag,1)
 	}
 }
 
@@ -76,12 +77,12 @@ func (tcpConn *TCPConn) Destroy() {
 func (tcpConn *TCPConn) Close() {
 	tcpConn.Lock()
 	defer tcpConn.Unlock()
-	if tcpConn.closeFlag {
+	if atomic.LoadInt32(&tcpConn.closeFlag)==1 {
 		return
 	}
 
 	tcpConn.doWrite(nil)
-	tcpConn.closeFlag = true
+	atomic.StoreInt32(&tcpConn.closeFlag,1)
 }
 
 func (tcpConn *TCPConn) GetRemoteIp() string {
@@ -104,7 +105,7 @@ func (tcpConn *TCPConn) doWrite(b []byte) error{
 func (tcpConn *TCPConn) Write(b []byte) error{
 	tcpConn.Lock()
 	defer tcpConn.Unlock()
-	if tcpConn.closeFlag || b == nil {
+	if atomic.LoadInt32(&tcpConn.closeFlag)==1 || b == nil {
 		tcpConn.ReleaseReadMsg(b)
 		return errors.New("conn is close")
 	}
@@ -133,14 +134,14 @@ func (tcpConn *TCPConn) ReleaseReadMsg(byteBuff []byte){
 }
 
 func (tcpConn *TCPConn) WriteMsg(args ...[]byte) error {
-	if tcpConn.closeFlag == true {
+	if atomic.LoadInt32(&tcpConn.closeFlag) == 1 {
 		return errors.New("conn is close")
 	}
 	return tcpConn.msgParser.Write(tcpConn, args...)
 }
 
 func (tcpConn *TCPConn) WriteRawMsg(args []byte) error {
-	if tcpConn.closeFlag == true {
+	if atomic.LoadInt32(&tcpConn.closeFlag) == 1 {
 		return errors.New("conn is close")
 	}
 
@@ -149,7 +150,7 @@ func (tcpConn *TCPConn) WriteRawMsg(args []byte) error {
 
 
 func (tcpConn *TCPConn) IsConnected() bool {
-	return tcpConn.closeFlag == false
+	return atomic.LoadInt32(&tcpConn.closeFlag) == 0
 }
 
 func (tcpConn *TCPConn) SetReadDeadline(d time.Duration)  {

node/node.go

@@ -295,9 +295,9 @@ func GetService(serviceName string) service.IService {
 	return service.GetService(serviceName)
 }
 
-func SetConfigDir(configDir string) {
-	configDir = configDir
-	cluster.SetConfigDir(configDir)
+func SetConfigDir(cfgDir string) {
+	configDir = cfgDir
+	cluster.SetConfigDir(cfgDir)
 }
 
 func GetConfigDir() string {

profiler/profiler.go

@@ -193,9 +193,11 @@ func Report() {
 
 		record = prof.record
 		prof.record = list.New()
+		callNum := prof.callNum
+		totalCostTime := prof.totalCostTime
 		prof.stackLocker.RUnlock()
 
-		DefaultReportFunction(name,prof.callNum,prof.totalCostTime,record)
+		DefaultReportFunction(name,callNum,totalCostTime,record)
 	}
 }
 

rpc/lclient.go

@@ -44,7 +44,8 @@ func (lc *LClient) Go(rpcHandler IRpcHandler,noReply bool, serviceMethod string,
 		sErr := errors.New("Call serviceMethod " + serviceMethod + " is error!")
 		log.SError(sErr.Error())
 		call := MakeCall()
-		call.Err = sErr
+		call.DoError(sErr)
+
 		return call
 	}
 
@@ -53,12 +54,13 @@ func (lc *LClient) Go(rpcHandler IRpcHandler,noReply bool, serviceMethod string,
 		//调用自己rpcHandler处理器
 		err := pLocalRpcServer.myselfRpcHandlerGo(lc.selfClient,serviceName, serviceMethod, args, requestHandlerNull,reply)
 		call := MakeCall()
+
 		if err != nil {
-			call.Err = err
+			call.DoError(err)
 			return call
 		}
 
-		call.done<-call
+		call.DoOK()
 		return call
 	}
 

rpc/rclient.go

@@ -44,8 +44,9 @@ func (rc *RClient) Go(rpcHandler IRpcHandler,noReply bool, serviceMethod string,
 	_, processor := GetProcessorType(args)
 	InParam, err := processor.Marshal(args)
 	if err != nil {
+		log.SError(err.Error())
 		call := MakeCall()
-		call.Err = err
+		call.DoError(err)
 		return call
 	}
 
@@ -65,14 +66,17 @@ func (rc *RClient) RawGo(rpcHandler IRpcHandler,processor IRpcProcessor, noReply
 
 	if err != nil {
 		call.Seq = 0
-		call.Err = err
+		log.SError(err.Error())
+		call.DoError(err)
 		return call
 	}
 
 	conn := rc.GetConn()
 	if conn == nil || conn.IsConnected()==false {
 		call.Seq = 0
-		call.Err = errors.New(serviceMethod + "  was called failed,rpc client is disconnect")
+		sErr := errors.New(serviceMethod + "  was called failed,rpc client is disconnect")
+		log.SError(sErr.Error())
+		call.DoError(sErr)
 		return call
 	}
 
@@ -83,8 +87,11 @@ func (rc *RClient) RawGo(rpcHandler IRpcHandler,processor IRpcProcessor, noReply
 	err = conn.WriteMsg([]byte{uint8(processor.GetProcessorType())}, bytes)
 	if err != nil {
 		rc.selfClient.RemovePending(call.Seq)
+
+		log.SError(err.Error())
+
 		call.Seq = 0
-		call.Err = err
+		call.DoError(err)
 	}
 
 	return call

rpc/rpc.go

@@ -102,6 +102,15 @@ func (rpcResponse *RpcResponse) Clear() *RpcResponse{
 	return rpcResponse
 }
 
+func (call *Call) DoError(err error){
+	call.Err = err
+	call.done <- call
+}
+
+func (call *Call) DoOK(){
+	call.done <- call
+}
+
 func (call *Call) Clear() *Call{
 	call.Seq = 0
 	call.ServiceMethod = ""

rpc/rpchandler.go

@@ -291,14 +291,16 @@ func (handler *RpcHandler) HandlerRpcRequest(request *RpcRequest) {
 		request.requestHandle(nil, RpcError(rErr))
 		return
 	}
+
+	requestHanle := request.requestHandle
 	returnValues := v.method.Func.Call(paramList)
 	errInter := returnValues[0].Interface()
 	if errInter != nil {
 		err = errInter.(error)
 	}
 
-	if request.requestHandle != nil && v.hasResponder == false {
-		request.requestHandle(oParam.Interface(), ConvertError(err))
+	if v.hasResponder == false && requestHanle != nil  {
+		requestHanle(oParam.Interface(), ConvertError(err))
 	}
 }
 
@@ -459,11 +461,6 @@ func (handler *RpcHandler) callRpc(nodeId int, serviceMethod string, args interf
 
 	pClient := pClientList[0]
 	pCall := pClient.Go(handler.rpcHandler,false, serviceMethod, args, reply)
-	if pCall.Err != nil {
-		err = pCall.Err
-		ReleaseCall(pCall)
-		return err
-	}
 
 	err = pCall.Done().Err
 	pClient.RemovePending(pCall.Seq)

rpc/server.go

@@ -256,10 +256,10 @@ func (server *Server) selfNodeRpcHandlerGo(processor IRpcProcessor, client *Clie
 
 	rpcHandler := server.rpcHandleFinder.FindRpcHandler(handlerName)
 	if rpcHandler == nil {
+		err := errors.New("service method " + serviceMethod + " not config!")
+		log.SError(err.Error())
 		pCall.Seq = 0
-		pCall.Err = errors.New("service method " + serviceMethod + " not config!")
-		pCall.done <- pCall
-		log.SError(pCall.Err.Error())
+		pCall.DoError(err)
 
 		return pCall
 	}
@@ -273,10 +273,10 @@ func (server *Server) selfNodeRpcHandlerGo(processor IRpcProcessor, client *Clie
 		var err error
 		iParam,err = processor.Clone(args)
 		if err != nil {
+			sErr := errors.New("RpcHandler " + handlerName + "."+serviceMethod+" deep copy inParam is error:" + err.Error())
+			log.SError(sErr.Error())
 			pCall.Seq = 0
-			pCall.Err = errors.New("RpcHandler " + handlerName + "."+serviceMethod+" deep copy inParam is error:" + err.Error())
-			pCall.done <- pCall
-			log.SError(pCall.Err.Error())
+			pCall.DoError(sErr)
 
 			return pCall
 		}
@@ -289,9 +289,10 @@ func (server *Server) selfNodeRpcHandlerGo(processor IRpcProcessor, client *Clie
 		var err error
 		req.inParam, err = rpcHandler.UnmarshalInParam(processor, serviceMethod, rpcMethodId, rawArgs)
 		if err != nil {
+			log.SError(err.Error())
+			pCall.Seq = 0
+			pCall.DoError(err)
 			ReleaseRpcRequest(req)
-			pCall.Err = err
-			pCall.done <- pCall
 			return pCall
 		}
 	}
@@ -321,20 +322,22 @@ func (server *Server) selfNodeRpcHandlerGo(processor IRpcProcessor, client *Clie
 
 				return
 			}
+
 			if len(Err) == 0 {
 				v.Err = nil
+				v.DoOK()
 			} else {
-				v.Err = Err
+				log.SError(Err.Error())
+				v.DoError(Err)
 			}
-			v.done <- v
 		}
 	}
 
 	err := rpcHandler.PushRpcRequest(req)
 	if err != nil {
+		log.SError(err.Error())
+		pCall.DoError(err)
 		ReleaseRpcRequest(req)
-		pCall.Err = err
-		pCall.done <- pCall
 	}
 
 	return pCall

sysservice/messagequeueservice/MongoPersist.go

@@ -1,18 +1,49 @@
 package messagequeueservice
 
 import (
+	"errors"
 	"fmt"
 	"github.com/duanhf2012/origin/log"
 	"github.com/duanhf2012/origin/service"
 	"github.com/duanhf2012/origin/sysmodule/mongodbmodule"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo/options"
-	"sunserver/common/util"
 	"time"
 )
 
 const MaxDays = 180
 
+type DataType interface {
+	int | uint | int64 | uint64 | float32 | float64 | int32 | uint32 | int16 | uint16
+}
+
+func convertToNumber[DType DataType](val interface{}) (error, DType) {
+	switch val.(type) {
+	case int64:
+		return nil, DType(val.(int64))
+	case int:
+		return nil, DType(val.(int))
+	case uint:
+		return nil, DType(val.(uint))
+	case uint64:
+		return nil, DType(val.(uint64))
+	case float32:
+		return nil, DType(val.(float32))
+	case float64:
+		return nil, DType(val.(float64))
+	case int32:
+		return nil, DType(val.(int32))
+	case uint32:
+		return nil, DType(val.(uint32))
+	case int16:
+		return nil, DType(val.(int16))
+	case uint16:
+		return nil, DType(val.(uint16))
+	}
+
+	return errors.New("unsupported type"), 0
+}
+
 type MongoPersist struct {
 	service.Module
 	mongo mongodbmodule.MongoModule
@@ -363,7 +394,7 @@ func (mp *MongoPersist) GetIndex(topicData *TopicData) uint64 {
 
 	for _, e := range document {
 		if e.Key == "_id" {
-			errC, seq := util.ConvertToNumber[uint64](e.Value)
+			errC, seq := convertToNumber[uint64](e.Value)
 			if errC != nil {
 				log.Error("value is error:%s,%+v, ", errC.Error(), e.Value)
 			}

sysservice/rankservice/MongodbPersist.go

@@ -6,9 +6,9 @@ import (
 	"github.com/duanhf2012/origin/rpc"
 	"github.com/duanhf2012/origin/service"
 	"github.com/duanhf2012/origin/sysmodule/mongodbmodule"
-	"github.com/duanhf2012/origin/util/coroutine"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo/options"
+	"runtime"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -71,7 +71,9 @@ func (mp *MongoPersist) OnInit() error {
 	}
 
 	//开启协程
-	coroutine.GoRecover(mp.persistCoroutine,-1)
+	mp.waitGroup.Add(1)
+	go mp.persistCoroutine()
+
 	return nil
 }
 
@@ -260,7 +262,6 @@ func (mp *MongoPersist) JugeTimeoutSave() bool{
 }
 
 func (mp *MongoPersist)  persistCoroutine(){
-	mp.waitGroup.Add(1)
 	defer mp.waitGroup.Done()
 	for atomic.LoadInt32(&mp.stop)==0 || mp.hasPersistData(){
 		//间隔时间sleep
@@ -291,6 +292,15 @@ func (mp *MongoPersist) hasPersistData() bool{
 }
 
 func (mp *MongoPersist) saveToDB(){
+	defer func() {
+		if r := recover(); r != nil {
+			buf := make([]byte, 4096)
+			l := runtime.Stack(buf, false)
+			errString := fmt.Sprint(r)
+			log.SError(" Core dump info[", errString, "]\n", string(buf[:l]))
+		}
+	}()
+
 	//1.copy数据
 	mp.Lock()
 	mapRemoveRankData := mp.mapRemoveRankData

sysservice/rankservice/RankSkip.go

@@ -356,12 +356,12 @@ func (rs *RankSkip) GetRankNodeDataByRank(rank uint64) (*RankData, uint64) {
 // GetRankKeyPrevToLimit 获取key前count名的数据
 func (rs *RankSkip) GetRankKeyPrevToLimit(findKey, count uint64, result *rpc.RankDataList) error {
 	if rs.GetRankLen() <= 0 {
-		return fmt.Errorf("rank[", rs.rankId, "] no data")
+		return fmt.Errorf("rank[%d] no data", rs.rankId)
 	}
 
 	findData, ok := rs.mapRankData[findKey]
 	if ok == false {
-		return fmt.Errorf("rank[", rs.rankId, "] no data")
+		return fmt.Errorf("rank[%d] no data", rs.rankId)
 	}
 
 	_, rankPos := rs.skipList.GetWithPosition(findData)
@@ -385,12 +385,12 @@ func (rs *RankSkip) GetRankKeyPrevToLimit(findKey, count uint64, result *rpc.Ran
 // GetRankKeyPrevToLimit 获取key前count名的数据
 func (rs *RankSkip) GetRankKeyNextToLimit(findKey, count uint64, result *rpc.RankDataList) error {
 	if rs.GetRankLen() <= 0 {
-		return fmt.Errorf("rank[", rs.rankId, "] no data")
+		return fmt.Errorf("rank[%d] no data", rs.rankId)
 	}
 
 	findData, ok := rs.mapRankData[findKey]
 	if ok == false {
-		return fmt.Errorf("rank[", rs.rankId, "] no data")
+		return fmt.Errorf("rank[%d] no data", rs.rankId)
 	}
 
 	_, rankPos := rs.skipList.GetWithPosition(findData)