优化服务发现

cluster/etcddiscovery.go

@@ -6,6 +6,12 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"os"
+	"path"
+	"strings"
+	"sync/atomic"
+	"time"
+
 	"github.com/duanhf2012/origin/v2/event"
 	"github.com/duanhf2012/origin/v2/log"
 	"github.com/duanhf2012/origin/v2/rpc"
@@ -14,16 +20,12 @@ import (
 	"go.etcd.io/etcd/api/v3/mvccpb"
 	"go.etcd.io/etcd/client/v3"
 	"google.golang.org/protobuf/proto"
-	"os"
-	"path"
-	"strings"
-	"sync/atomic"
-	"time"
 )
 
 const originDir = "/origin"
 
 type etcdClientInfo struct {
+	isLocalNetwork bool
 	watchKeys      []string
 	leaseID        clientv3.LeaseID
 	keepAliveChan  <-chan *clientv3.LeaseKeepAliveResponse
@@ -93,6 +95,7 @@ func (ed *EtcdDiscoveryService) OnInit() error {
 		return errors.New("etcd discovery config is nil.")
 	}
 
+	var hasLocalNetwork bool
 	for i := 0; i < len(etcdDiscoveryCfg.EtcdList); i++ {
 		var client *clientv3.Client
 		var tlsConfig *tls.Config
@@ -141,13 +144,25 @@ func (ed *EtcdDiscoveryService) OnInit() error {
 		}
 
 		ec := &etcdClientInfo{}
-		for _, networkName := range etcdDiscoveryCfg.EtcdList[i].NetworkName {
+
+		if etcdDiscoveryCfg.EtcdList[i].LocalNetworkName != "" {
+			hasLocalNetwork = true
+			ec.isLocalNetwork = true
+			ec.watchKeys = append(ec.watchKeys, fmt.Sprintf("%s/%s", originDir, etcdDiscoveryCfg.EtcdList[i].LocalNetworkName))
+		} else {
+			ec.isLocalNetwork = false
+			for _, networkName := range etcdDiscoveryCfg.EtcdList[i].NeighborNetworkName {
 				ec.watchKeys = append(ec.watchKeys, fmt.Sprintf("%s/%s", originDir, networkName))
 			}
+		}
 
 		ed.mapClient[client] = ec
 	}
 
+	if !hasLocalNetwork {
+		return errors.New("etcd discovery init fail,cannot find local network")
+	}
+
 	return nil
 }
 
@@ -167,15 +182,19 @@ func (ed *EtcdDiscoveryService) registerServiceByClient(client *clientv3.Client,
 	}
 
 	etcdClient.leaseID = resp.ID
-	for _, watchKey := range etcdClient.watchKeys {
+
-		// 注册服务节点到 etcd
+	// 注册服务节点到 etcd,LocalNetwork时才会注册，且etcdClient.watchKeys必然>0
-		_, err = client.Put(context.Background(), ed.getRegisterKey(watchKey), ed.byteLocalNodeInfo, clientv3.WithLease(resp.ID))
+	if len(etcdClient.watchKeys) != 1 {
+		log.Error("LocalNetwork watchkey is error")
+		return
+	}
+
+	_, err = client.Put(context.Background(), ed.getRegisterKey(etcdClient.watchKeys[0]), ed.byteLocalNodeInfo, clientv3.WithLease(resp.ID))
 	if err != nil {
 		log.Error("etcd Put fail", log.ErrorField("err", err))
 		ed.tryRegisterService(client, etcdClient)
 		return
 	}
-	}
 
 	etcdClient.keepAliveChan, err = client.KeepAlive(context.Background(), etcdClient.leaseID)
 	if err != nil {
@@ -204,6 +223,10 @@ func (ed *EtcdDiscoveryService) tryRegisterService(client *clientv3.Client, etcd
 		return
 	}
 
+	if !etcdClient.isLocalNetwork {
+		return
+	}
+
 	ed.AfterFunc(time.Second*3, func(t *timer.Timer) {
 		ed.registerServiceByClient(client, etcdClient)
 	})
@@ -229,15 +252,20 @@ func (ed *EtcdDiscoveryService) tryLaterRetire() {
 func (ed *EtcdDiscoveryService) retire() error {
 	//从etcd中更新
 	for c, ec := range ed.mapClient {
-		for _, watchKey := range ec.watchKeys {
+		if !ec.isLocalNetwork {
-			// 注册服务节点到 etcd
+			continue
-			_, err := c.Put(context.Background(), ed.getRegisterKey(watchKey), ed.byteLocalNodeInfo, clientv3.WithLease(ec.leaseID))
+		}
+
+		if len(ec.watchKeys)!=1 {
+			log.Error("LocalNetwork watchkey is error")
+			continue
+		}
+		_, err := c.Put(context.Background(), ed.getRegisterKey(ec.watchKeys[0]), ed.byteLocalNodeInfo, clientv3.WithLease(ec.leaseID))
 		if err != nil {
 			log.Error("etcd Put fail", log.ErrorField("err", err))
 			return err
 		}
 	}
-	}
 
 	return nil
 }
@@ -503,13 +531,18 @@ func (ed *EtcdDiscoveryService) RPC_ServiceRecord(etcdServiceRecord *service.Etc
 
 	//写入到etcd中
 	for c, info := range ed.mapClient {
-		for _, watchKey := range info.watchKeys {
+		if !info.isLocalNetwork {
-			if ed.getNetworkNameByWatchKey(watchKey) == etcdServiceRecord.NetworkName {
+			continue
+		}
+
+		if len(info.watchKeys)!=1 {
+					log.Error("")
+		}
+		if ed.getNetworkNameByWatchKey(info.watchKeys[0]) == etcdServiceRecord.NetworkName {
 			client = c
 			break
 		}
 	}
-	}
 
 	if client == nil {
 		log.Error("etcd record fail,cannot find network name", log.String("networkName", etcdServiceRecord.NetworkName))

cluster/origindiscovery.go

@@ -508,11 +508,17 @@ func (dc *OriginDiscoveryClient) regServiceDiscover(nodeId string) {
 	if nodeId == cluster.GetLocalNodeInfo().NodeId {
 		return
 	}
+
 	nodeInfo := cluster.getOriginMasterDiscoveryNodeInfo(nodeId)
 	if nodeInfo == nil {
 		return
 	}
 
+	// 只能往本地Master结点上注册
+	if !cluster.IsLocalMasterNodeId(nodeId) {
+		return
+	}
+
 	var req rpc.RegServiceDiscoverReq
 	req.NodeInfo = &rpc.NodeInfo{}
 	req.NodeInfo.NodeId = cluster.localNodeInfo.NodeId
@@ -617,12 +623,21 @@ func (cls *Cluster) IsOriginMasterDiscoveryNode(nodeId string) bool {
 	return cls.getOriginMasterDiscoveryNodeInfo(nodeId) != nil
 }
 
+func (cls *Cluster) IsLocalMasterNodeId(nodeId string) bool {
+	if cls.discoveryInfo.Origin == nil {
+		return false
+	}
+
+	return cls.discoveryInfo.Origin.LocalMasterNodeId == nodeId
+}
 func (cls *Cluster) getOriginMasterDiscoveryNodeInfo(nodeId string) *NodeInfo {
 	if cls.discoveryInfo.Origin == nil {
 		return nil
 	}
 
 	for i := 0; i < len(cls.discoveryInfo.Origin.MasterNodeList); i++ {
+
+
 		if cls.discoveryInfo.Origin.MasterNodeList[i].NodeId == nodeId  {
 			return &cls.discoveryInfo.Origin.MasterNodeList[i]
 		}

cluster/parsecfg.go

@@ -16,9 +16,9 @@ import (
 
 var json = jsoniter.ConfigCompatibleWithStandardLibrary
 
-
 type EtcdList struct {
-	NetworkName []string
+	LocalNetworkName    string // 如果配置，则为本地网络,必需配置一个本地网络
+	NeighborNetworkName []string
 	Endpoints           []string
 	UserName            string
 	Password            string
@@ -30,12 +30,12 @@ type EtcdList struct {
 type EtcdDiscovery struct {
 	DialTimeoutMillisecond time.Duration
 	TTLSecond              int64
-
 	EtcdList               []EtcdList
 }
 
 type OriginDiscovery struct {
 	TTLSecond         int64
+	LocalMasterNodeId string
 	MasterNodeList    []NodeInfo
 }
 
@@ -131,21 +131,20 @@ func (d *DiscoveryInfo) setEtcd(etcd *EtcdDiscovery) error {
 		return fmt.Errorf("repeat configuration of Discovery")
 	}
 
-	//Endpoints不允许重复
-	mapAddr := make(map[string]struct{})
 	for _, n := range etcd.EtcdList {
-		for _, endPoint := range n.Endpoints {
-			if _, ok := mapAddr[endPoint]; ok == true {
-				return fmt.Errorf("etcd discovery config Etcd.EtcdList.Endpoints %+v is repeat", endPoint)
-			}
-			mapAddr[endPoint] = struct{}{}
-		}
-
 		//networkName不允许重复
 		mapNetworkName := make(map[string]struct{})
-		for _, netName := range n.NetworkName {
+		if n.LocalNetworkName != "" {
+			if _, ok := mapNetworkName[n.LocalNetworkName]; ok == true {
+				return fmt.Errorf("etcd discovery config Etcd.EtcdList.LocalNetworkName %+v is repeat", n.LocalNetworkName)
+			}
+			mapNetworkName[n.LocalNetworkName] = struct{}{}
+			continue
+		}
+
+		for _, netName := range n.NeighborNetworkName {
 			if _, ok := mapNetworkName[netName]; ok == true {
-				return fmt.Errorf("etcd discovery config Etcd.EtcdList.NetworkName %+v is repeat", n.NetworkName)
+				return fmt.Errorf("etcd discovery config Etcd.EtcdList.NetworkName %+v is repeat", netName)
 			}
 
 			mapNetworkName[netName] = struct{}{}