优化etcd发现服务，支持记录附加信息rpc

cluster/discovery.go

@@ -62,7 +62,7 @@ func (cls *Cluster) setupConfigDiscovery(localNodeId string, setupServiceFun Set
 	return nil
 }
 
-func (cls *Cluster) GetOriginDiscoveryNodeList() []NodeInfo {
+func (cls *Cluster) GetOriginDiscovery() *OriginDiscovery {
 	return cls.discoveryInfo.Origin
 }
 

cluster/etcddiscovery.go

@@ -15,6 +15,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"path"
 	"runtime"
 	"strings"
 	"sync/atomic"
@@ -38,7 +39,7 @@ type EtcdDiscoveryService struct {
 	byteLocalNodeInfo string
 	mapClient map[*clientv3.Client]*etcdClientInfo
 	isClose int32
-
+	bRetire bool
 	mapDiscoveryNodeId map[string]map[string]struct{} //map[networkName]map[nodeId]
 }
 
@@ -100,7 +101,8 @@ func (ed *EtcdDiscoveryService) OnInit() error {
 		}
 
 		ctx,_:=context.WithTimeout(context.Background(),time.Second*3)
-		_,err = client.Put(ctx,testKey,"")
+		_,err = client.Leases(ctx)
+		//_,err = client.Put(ctx,testKey,"")
 		if err != nil {
 			log.Error("etcd discovery init fail",log.Any("endpoint",etcdDiscoveryCfg.EtcdList[i].Endpoints),log.ErrorAttr("err",err))
 			return err
@@ -186,9 +188,37 @@ func (ed *EtcdDiscoveryService) tryWatch(client *clientv3.Client,etcdClient *etc
 	})
 }
 
+func (ed *EtcdDiscoveryService) tryLaterRetire() {
+	ed.AfterFunc(time.Second, func(*timer.Timer) {
+		if ed.retire() != nil {
+			ed.tryLaterRetire()
+		}
+	})
+}
+
+func (ed *EtcdDiscoveryService) retire() error{
+	//从etcd中更新
+	for c,ec := range ed.mapClient {
+		for _, watchKey := range ec.watchKeys {
+			// 注册服务节点到 etcd
+			_, err := c.Put(context.Background(), ed.getRegisterKey(watchKey), ed.byteLocalNodeInfo, clientv3.WithLease(ec.leaseID))
+			if err != nil {
+				log.Error("etcd Put fail", log.ErrorAttr("err", err))
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 func (ed *EtcdDiscoveryService) OnRetire(){
-	atomic.StoreInt32(&ed.isClose,1)
-	ed.close()
+	ed.bRetire = true
+	ed.marshalNodeInfo()
+
+	if ed.retire()!= nil {
+		ed.tryLaterRetire()
+	}
 }
 
 func (ed *EtcdDiscoveryService) OnRelease(){
@@ -212,7 +242,7 @@ func (ed *EtcdDiscoveryService) marshalNodeInfo() error{
 	var nodeInfo rpc.NodeInfo
 	nodeInfo.NodeId = nInfo.NodeId
 	nodeInfo.ListenAddr = nInfo.ListenAddr
-	nodeInfo.Retire = nInfo.Retire
+	nodeInfo.Retire = ed.bRetire
 	nodeInfo.PublicServiceList = nInfo.PublicServiceList
 	nodeInfo.MaxRpcParamLen = nInfo.MaxRpcParamLen
 
@@ -344,7 +374,7 @@ func (ed *EtcdDiscoveryService) delNode(fullKey string) string{
 		return ""
 	}
 
-	 ed.funDelNode(nodeId,true)
+	 ed.funDelNode(nodeId,false)
 	return nodeId
 }
 
@@ -415,7 +445,7 @@ func (ed *EtcdDiscoveryService) OnEventGets(watchKey string,Kvs []*mvccpb.KeyVal
 	mapLastNodeId := ed.mapDiscoveryNodeId[watchKey] // 根据watchKey获取对应的节点ID集合
 	for nodeId := range mapLastNodeId { // 遍历所有节点ID
 	    if _,ok := mapNode[nodeId];ok == false && nodeId != ed.localNodeId { // 检查节点是否不存在于mapNode且不是本地节点
-	        ed.funDelNode(nodeId,true) // 调用函数删除该节点
+	        ed.funDelNode(nodeId,false) // 调用函数删除该节点
 			delete(ed.mapDiscoveryNodeId[watchKey],nodeId)
 			log.Debug(">>etcd OnEventGets Delete",log.String("watchKey",watchKey),log.String("nodeId",nodeId))
 	    }
@@ -441,3 +471,56 @@ func (ed *EtcdDiscoveryService) addNodeId(watchKey string,nodeId string) {
 
 	ed.mapDiscoveryNodeId[watchKey][nodeId] = struct{}{}
 }
+
+func (ed *EtcdDiscoveryService) OnNodeDisconnect(nodeId string) {
+	//将Discard结点清理
+	cluster.DiscardNode(nodeId)
+}
+
+func (ed *EtcdDiscoveryService) RPC_ServiceRecord(etcdServiceRecord *service.EtcdServiceRecordEvent,empty *service.Empty) error{
+		var client *clientv3.Client
+
+		//写入到etcd中
+		for c, info := range ed.mapClient{
+			for _,watchKey := range info.watchKeys {
+				if ed.getNetworkNameByWatchKey(watchKey) == etcdServiceRecord.NetworkName {
+					client = c
+					break
+				}
+			}
+		}
+
+		if client == nil {
+			log.Error("etcd record fail,cannot find network name",log.String("networkName",etcdServiceRecord.NetworkName))
+			return errors.New("annot find network name")
+		}
+
+		var lg *clientv3.LeaseGrantResponse
+		var err error
+
+		if etcdServiceRecord.TTLSecond > 0 {
+			ctx,_:=context.WithTimeout(context.Background(),time.Second*3)
+			lg, err = client.Grant(ctx, etcdServiceRecord.TTLSecond)
+			if err != nil {
+				log.Error("etcd record fail,cannot grant lease",log.ErrorAttr("err",err))
+				return errors.New("cannot grant lease")
+			}
+		}
+
+		if lg != nil {
+			ctx,_:=context.WithTimeout(context.Background(),time.Second*3)
+			_, err = client.Put(ctx, path.Join(originDir,etcdServiceRecord.RecordKey),etcdServiceRecord.RecordInfo, clientv3.WithLease(lg.ID))
+			if err != nil {
+				log.Error("etcd record fail,cannot put record",log.ErrorAttr("err",err))
+			}
+			return errors.New("cannot put record")
+		}
+
+		_,err = client.Put(context.Background(), path.Join(originDir,etcdServiceRecord.RecordKey),etcdServiceRecord.RecordInfo)
+		if err != nil {
+			log.Error("etcd record fail,cannot put record",log.ErrorAttr("err",err))
+			return errors.New("cannot put record")
+		}
+
+		return nil
+}

cluster/origindiscovery.go

@@ -5,9 +5,9 @@ import (
 	"github.com/duanhf2012/origin/v2/log"
 	"github.com/duanhf2012/origin/v2/rpc"
 	"github.com/duanhf2012/origin/v2/service"
-	"time"
 	"github.com/duanhf2012/origin/v2/util/timer"
 	"google.golang.org/protobuf/proto"
+	"time"
 )
 
 const OriginDiscoveryMasterName = "DiscoveryMaster"
@@ -16,6 +16,71 @@ const RegServiceDiscover = OriginDiscoveryMasterName + ".RPC_RegServiceDiscover"
 const SubServiceDiscover = OriginDiscoveryClientName + ".RPC_SubServiceDiscover"
 const AddSubServiceDiscover = OriginDiscoveryMasterName + ".RPC_AddSubServiceDiscover"
 const NodeRetireRpcMethod = OriginDiscoveryMasterName+".RPC_NodeRetire"
+//
+//type nodeTTL struct {
+//	nodeId string
+//	refreshTime time.Time
+//}
+//
+//type nodeSetTTL struct {
+//	l *list.List
+//	mapElement map[string]*list.Element
+//	ttl time.Duration
+//}
+//
+//func (ns *nodeSetTTL) init(ttl time.Duration) {
+//	ns.ttl = ttl
+//	ns.mapElement = make(map[string]*list.Element,32)
+//	ns.l = list.New()
+//}
+//
+//func (ns *nodeSetTTL) removeNode(nodeId string) {
+//	ele,ok:=ns.mapElement[nodeId]
+//	if ok == false {
+//		return
+//	}
+//
+//	ns.l.Remove(ele)
+//	delete(ns.mapElement,nodeId)
+//}
+//
+//func (ns *nodeSetTTL) addAndRefreshNode(nodeId string){
+//	ele,ok:=ns.mapElement[nodeId]
+//	if ok == false {
+//		ele = ns.l.PushBack(nodeId)
+//		ele.Value = &nodeTTL{nodeId,time.Now()}
+//		ns.mapElement[nodeId] = ele
+//		return
+//	}
+//
+//	ele.Value.(*nodeTTL).refreshTime =  time.Now()
+//	ns.l.MoveToBack(ele)
+//}
+//
+//func (ns *nodeSetTTL) checkTTL(cb func(nodeIdList []string)){
+//	nodeIdList := []string{}
+//	for{
+//		f := ns.l.Front()
+//		if f == nil {
+//			break
+//		}
+//
+//		nt := f.Value.(*nodeTTL)
+//		if time.Now().Sub(nt.refreshTime) > ns.ttl {
+//			nodeIdList = append(nodeIdList,nt.nodeId)
+//		}else{
+//			break
+//		}
+//
+//		//删除结点
+//		ns.l.Remove(f)
+//		delete(ns.mapElement,nt.nodeId)
+//	}
+//
+//	if len(nodeIdList) >0 {
+//		cb(nodeIdList)
+//	}
+//}
 
 type OriginDiscoveryMaster struct {
 	service.Service
@@ -48,7 +113,6 @@ func init() {
 	clientService.SetName(OriginDiscoveryClientName)
 }
 
-
 func (ds *OriginDiscoveryMaster) isRegNode(nodeId string) bool {
 	_, ok := ds.mapNodeInfo[nodeId]
 	return ok
@@ -247,12 +311,14 @@ func (dc *OriginDiscoveryClient) OnStart() {
 }
 
 func (dc *OriginDiscoveryClient) addDiscoveryMaster() {
-	discoveryNodeList := cluster.GetOriginDiscoveryNodeList()
-	for i := 0; i < len(discoveryNodeList); i++ {
-		if discoveryNodeList[i].NodeId == cluster.GetLocalNodeInfo().NodeId {
+	discoveryNodeList := cluster.GetOriginDiscovery()
+
+	for i := 0; i < len(discoveryNodeList.MasterNodeList); i++ {
+		if discoveryNodeList.MasterNodeList[i].NodeId == cluster.GetLocalNodeInfo().NodeId {
 			continue
 		}
-		dc.funSetNode(&discoveryNodeList[i])
+		dc.funSetNode(&discoveryNodeList.MasterNodeList[i])
+
 	}
 }
 
@@ -355,8 +421,8 @@ func (dc *OriginDiscoveryClient) OnNodeConnected(nodeId string) {
 func (dc *OriginDiscoveryClient) OnRetire(){
 	dc.bRetire = true
 
-	masterNodeList := cluster.GetOriginDiscoveryNodeList()
-	for i:=0;i<len(masterNodeList);i++{
+	masterNodeList := cluster.GetOriginDiscovery()
+	for i:=0;i<len(masterNodeList.MasterNodeList);i++{
 		var nodeRetireReq rpc.NodeRetireReq
 
 		nodeRetireReq.NodeInfo = &rpc.NodeInfo{}
@@ -367,13 +433,19 @@ func (dc *OriginDiscoveryClient) OnRetire(){
 		nodeRetireReq.NodeInfo.Retire = dc.bRetire
 		nodeRetireReq.NodeInfo.Private = cluster.localNodeInfo.Private
 
-		err := dc.GoNode(masterNodeList[i].NodeId,NodeRetireRpcMethod,&nodeRetireReq)
+		err := dc.GoNode(masterNodeList.MasterNodeList[i].NodeId,NodeRetireRpcMethod,&nodeRetireReq)
 		if err!= nil {
 			log.Error("call "+NodeRetireRpcMethod+" is fail",log.ErrorAttr("err",err))
 		}
 	}
 }
 
+func (dc *OriginDiscoveryClient) tryRegServiceDiscover(nodeId string){
+	dc.AfterFunc(time.Second*3, func(timer *timer.Timer) {
+		dc.regServiceDiscover(nodeId)
+	})
+}
+
 func (dc *OriginDiscoveryClient) regServiceDiscover(nodeId string){
 	nodeInfo := cluster.getOriginMasterDiscoveryNodeInfo(nodeId)
 	if nodeInfo == nil {
@@ -393,15 +465,14 @@ func (dc *OriginDiscoveryClient) regServiceDiscover(nodeId string){
 	err := dc.AsyncCallNode(nodeId, RegServiceDiscover, &req, func(res *rpc.Empty, err error) {
 		if err != nil {
 			log.Error("call "+RegServiceDiscover+" is fail :"+ err.Error())
-			dc.AfterFunc(time.Second*3, func(timer *timer.Timer) {
-				dc.regServiceDiscover(nodeId)
-			})
-
+			dc.tryRegServiceDiscover(nodeId)
 			return
 		}
 	})
+
 	if err != nil {
 		log.Error("call "+ RegServiceDiscover+" is fail :"+ err.Error())
+		dc.tryRegServiceDiscover(nodeId)
 	}
 }
 
@@ -454,7 +525,7 @@ func (cls *Cluster) checkOriginDiscovery(localNodeId string) (bool, bool) {
 	var hasMaster bool   //是否配置Master服务
 
 	//遍历所有结点
-	for _, nodeInfo := range cls.discoveryInfo.Origin {
+	for _, nodeInfo := range cls.discoveryInfo.Origin.MasterNodeList {
 		if nodeInfo.NodeId == localNodeId {
 			localMaster = true
 		}
@@ -485,9 +556,9 @@ func (cls *Cluster) IsOriginMasterDiscoveryNode(nodeId string) bool {
 }
 
 func (cls *Cluster) getOriginMasterDiscoveryNodeInfo(nodeId string) *NodeInfo {
-	for i := 0; i < len(cls.discoveryInfo.Origin); i++ {
-		if cls.discoveryInfo.Origin[i].NodeId == nodeId {
-			return &cls.discoveryInfo.Origin[i]
+	for i := 0; i < len(cls.discoveryInfo.Origin.MasterNodeList); i++ {
+		if cls.discoveryInfo.Origin.MasterNodeList[i].NodeId == nodeId {
+			return &cls.discoveryInfo.Origin.MasterNodeList[i]
 		}
 	}
 

cluster/parsecfg.go

@@ -25,6 +25,11 @@ type EtcdDiscovery struct {
 	EtcdList []EtcdList
 }
 
+type OriginDiscovery struct {
+	TTLSecond int64
+	MasterNodeList []NodeInfo
+}
+
 type DiscoveryType int
 
 const (
@@ -36,7 +41,7 @@ const (
 type DiscoveryInfo struct {
 	discoveryType DiscoveryType
 	Etcd          *EtcdDiscovery  //etcd
-	Origin        []NodeInfo      //orign
+	Origin        *OriginDiscovery      //orign
 }
 
 type NodeInfoList struct {
@@ -101,8 +106,8 @@ func (d *DiscoveryInfo) setEtcd(etcd *EtcdDiscovery) error{
 	return nil
 }
 
-func (d *DiscoveryInfo) setOrigin(nodeInfos []NodeInfo) error{
-	if nodeInfos == nil {
+func (d *DiscoveryInfo) setOrigin(originDiscovery *OriginDiscovery) error{
+	if originDiscovery== nil || len(originDiscovery.MasterNodeList)==0 {
 		return nil
 	}
 
@@ -112,7 +117,7 @@ func (d *DiscoveryInfo) setOrigin(nodeInfos []NodeInfo) error{
 
 	mapListenAddr := make(map[string]struct{})
 	mapNodeId := make(map[string]struct{})
-	for _, n := range nodeInfos {
+	for _, n := range originDiscovery.MasterNodeList {
 		if _, ok := mapListenAddr[n.ListenAddr]; ok == true {
 			return fmt.Errorf("discovery config Origin.ListenAddr %s is repeat", n.ListenAddr)
 		}
@@ -124,7 +129,7 @@ func (d *DiscoveryInfo) setOrigin(nodeInfos []NodeInfo) error{
 		mapNodeId[n.NodeId] = struct{}{}
 	}
 
-	d.Origin = nodeInfos
+	d.Origin = originDiscovery
 	d.discoveryType = OriginType
 	return nil
 }