Ma/origin
1
0
Fork 0
mirror of https://github.com/duanhf2012/origin.git synced 2026-04-14 13:07:30 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

提交origin2.0版本

Browse Source
This commit is contained in:
duanhf2012
2020-03-28 09:57:16 +08:00
parent 0d98f77d07
commit 84fb8ab36d
111 changed files with 3657 additions and 8382 deletions
Download Patch File Download Diff File Expand all files Collapse all files

786
cluster/cluster.go
View File

@@ -1,723 +1,135 @@
package cluster
import (
"errors"
"fmt"
"math/rand"
"net"
"os"
"sort"
"github.com/duanhf2012/originnet/log"
"github.com/duanhf2012/originnet/rpc"
"github.com/duanhf2012/originnet/service"
"strings"
"time"
"github.com/duanhf2012/origin/rpc"
"github.com/duanhf2012/origin/service"
"github.com/duanhf2012/origin/sysmodule"
)
type RpcClient struct {
nodeid int
pclient *rpc.Client
serverAddr string
var configdir = "./config/"
type SubNet struct {
NodeList []NodeInfo
}
func (slf *RpcClient) IsConnected() bool {
return (slf.pclient != nil) && (slf.pclient.IsClosed() == false)
type NodeInfo struct {
NodeId int
ListenAddr string
NodeName string
ServiceList []string
}
type CCluster struct {
port int
cfg *ClusterConfig
nodeclient map[int]*RpcClient
reader net.Conn
writer net.Conn
LocalRpcClient *rpc.Client
innerLocalServiceList map[string]bool
type NodeRpcInfo struct {
nodeinfo NodeInfo
client *rpc.Client
}
func (slf *CCluster) ReadNodeInfo(nodeid int) error {
mapNodeData, err := ReadAllNodeConfig("./config/nodeconfig.json")
var cluster Cluster
type Cluster struct {
localsubnet SubNet //本子网
mapSubNetInfo map[string] SubNet //子网名称，子网信息
mapSubNetNodeInfo map[string]map[int]NodeInfo //map[子网名称]map[NodeId]NodeInfo
localSubNetMapNode map[int]NodeInfo //本子网内 map[NodeId]NodeInfo
localSubNetMapService map[string][]NodeInfo //本子网内所有ServiceName对应的结点列表
localNodeMapService map[string]interface{} //本Node支持的服务
localNodeInfo NodeInfo
localNodeServiceCfg map[string]interface{} //map[servicename]数据
mapRpc map[int] NodeRpcInfo//nodeid
rpcServer rpc.Server
}
func SetConfigDir(cfgdir string){
configdir = cfgdir
}
func (slf *Cluster) Init(currentNodeId int) error{
//1.初始化配置
err := slf.InitCfg(currentNodeId)
if err != nil {
return err
}
slf.cfg, err = ReadCfg("./config/cluster.json", nodeid, mapNodeData)
if err != nil {
return err
slf.rpcServer.Init(slf)
//2.建议rpc连接
slf.mapRpc = map[int] NodeRpcInfo{}
for _,nodeinfo := range slf.localSubNetMapNode {
rpcinfo := NodeRpcInfo{}
rpcinfo.nodeinfo = nodeinfo
rpcinfo.client = &rpc.Client{}
if nodeinfo.NodeId == currentNodeId {
rpcinfo.client.Connect("localhost")
//rpcinfo.client.Connect(nodeinfo.ListenAddr)
}else{
rpcinfo.client.Connect(nodeinfo.ListenAddr)
}
slf.mapRpc[nodeinfo.NodeId] = rpcinfo
}
return nil
}
func (slf *CCluster) GetClusterClient(id int) *rpc.Client {
if id == GetNodeId() {
return slf.LocalRpcClient
func (slf *Cluster) FindRpcHandler(servicename string) rpc.IRpcHandler {
pService := service.GetService(servicename)
if pService == nil {
return nil
}
v, ok := slf.nodeclient[id]
return pService.GetRpcHandler()
}
func (slf *Cluster) Start() {
slf.rpcServer.Start(slf.localNodeInfo.ListenAddr)
}
func GetCluster() *Cluster{
return &cluster
}
func (slf *Cluster) GetRpcClient(nodeid int) *rpc.Client {
c,ok := slf.mapRpc[nodeid]
if ok == false {
return nil
}
return v.pclient
return c.client
}
func (slf *CCluster) GetBindUrl() string {
return slf.cfg.currentNode.ServerAddr
}
func GetRpcClient(serviceMethod string) ([]*rpc.Client,error) {
serviceAndMethod := strings.Split(serviceMethod,".")
if len(serviceAndMethod)!=2 {
return nil,fmt.Errorf("servicemethod param %s is error!",serviceMethod)
}
func (slf *CCluster) AcceptRpc(tpcListen *net.TCPListener) error {
for {
conn, err := tpcListen.Accept()
if err != nil {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "tpcListen.Accept error:%v", err)
return err
//1.找到对应的rpcnodeid
var rpcClientList []*rpc.Client
nodeidList := GetCluster().GetNodeIdByService(serviceAndMethod[0])
if len(nodeidList) ==0 {
return rpcClientList,fmt.Errorf("Cannot Find %s nodeid",serviceMethod)
}
for _,nodeid:= range nodeidList {
pClient := GetCluster().GetRpcClient(nodeid)
if pClient==nil {
log.Error("Cannot connect node id %d",nodeid)
continue
}
go rpc.ServeConn(conn)
rpcClientList = append(rpcClientList,pClient)
}
return nil
return rpcClientList,nil
}
func (slf *CCluster) ListenService() error {
bindStr := slf.GetBindUrl()
parts := strings.Split(bindStr, ":")
if len(parts) < 2 {
service.GetLogger().Printf(sysmodule.LEVER_FATAL, "ListenService address %s is error.", bindStr)
os.Exit(1)
}
bindStr = "0.0.0.0:" + parts[1]
//
tcpaddr, err := net.ResolveTCPAddr("tcp4", bindStr)
if err != nil {
service.GetLogger().Printf(sysmodule.LEVER_FATAL, "ResolveTCPAddr error:%v", err)
os.Exit(1)
return err
}
tcplisten, err2 := net.ListenTCP("tcp", tcpaddr)
if err2 != nil {
service.GetLogger().Printf(sysmodule.LEVER_FATAL, "ListenTCP error:%v", err2)
os.Exit(1)
return err2
}
go slf.AcceptRpc(tcplisten)
slf.ReSetLocalRpcClient()
return nil
}
func (slf *CCluster) ReSetLocalRpcClient() {
slf.reader, slf.writer = net.Pipe()
go rpc.ServeConn(slf.reader)
slf.LocalRpcClient = rpc.NewClient(slf.writer,true)
}
type CPing struct {
TimeStamp int64
}
type CPong struct {
TimeStamp int64
}
func (slf *CPing) Ping(ping *CPing, pong *CPong) error {
pong.TimeStamp = ping.TimeStamp
return nil
}
func (slf *CCluster) GetClusterMode() string {
return slf.cfg.GetClusterMode()
}
func (slf *CCluster) ConnService() error {
ping := CPing{0}
pong := CPong{0}
rpc.RegisterName("CPing", "", &ping)
//连接集群服务器
for _, nodeList := range slf.cfg.mapClusterNodeService {
for _, node := range nodeList {
if node.NodeID == slf.cfg.currentNode.NodeID {
continue
}
slf.nodeclient[node.NodeID] = &RpcClient{node.NodeID, nil, node.ServerAddr}
}
}
//判断集群模式
for {
for _, rpcClient := range slf.nodeclient {
//连接状态发送ping
if rpcClient.IsConnected() == true {
ping.TimeStamp = 0
err := rpcClient.pclient.Call("CPing.Ping", &ping, &pong)
if err != nil {
rpcClient.pclient.Close()
rpcClient.pclient = nil
continue
}
continue
}
//非连接状态重新连接
if rpcClient.pclient != nil {
rpcClient.pclient.Close()
rpcClient.pclient = nil
}
client, err := rpc.Dial("tcp", rpcClient.serverAddr)
if err != nil {
service.GetLogger().Printf(sysmodule.LEVER_WARN, "Connect nodeid:%d,address:%s fail", rpcClient.nodeid, rpcClient.serverAddr)
continue
}
service.GetLogger().Printf(sysmodule.LEVER_INFO, "Connect nodeid:%d,address:%s succ", rpcClient.nodeid, rpcClient.serverAddr)
v, _ := slf.nodeclient[rpcClient.nodeid]
v.pclient = client
}
if slf.LocalRpcClient.IsClosed() {
slf.ReSetLocalRpcClient()
}
time.Sleep(time.Second * 4)
}
return nil
}
func (slf *CCluster) Init(currentNodeid int) error {
slf.nodeclient = make(map[int]*RpcClient)
return slf.ReadNodeInfo(currentNodeid)
}
func (slf *CCluster) Start() error {
service.InstanceServiceMgr().FetchService(slf.OnFetchService)
//监听服务
slf.ListenService()
//集群
go slf.ConnService()
return nil
}
//servicename.methodname
//_servicename.methodname
func (slf *CCluster) Call(NodeServiceMethod string, args interface{}, reply interface{}) error {
var callServiceName string
var serviceName string
nodeidList := slf.GetNodeList(NodeServiceMethod, &callServiceName, &serviceName)
if len(nodeidList) > 1 || len(nodeidList) < 1 {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.Call(%s) find nodes count %d is error.", NodeServiceMethod, len(nodeidList))
return fmt.Errorf("CCluster.Call(%s) find nodes count %d is error.", NodeServiceMethod, len(nodeidList))
}
nodeid := nodeidList[0]
if nodeid == GetNodeId() {
//判断服务是否已经完成初始化
iService := service.InstanceServiceMgr().FindService(serviceName)
if iService == nil {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.Call(%s): NodeId %d cannot find service.", NodeServiceMethod, nodeid)
return fmt.Errorf("CCluster.Call(%s): NodeId %d cannot find service..", NodeServiceMethod, nodeid)
}
if iService.IsInit() == false {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.Call(%s): NodeId %d is not init.", NodeServiceMethod, nodeid)
return fmt.Errorf("CCluster.Call(%s): NodeId %d is not init.", NodeServiceMethod, nodeid)
}
return slf.LocalRpcClient.Call(callServiceName, args, reply)
} else {
pclient := slf.GetClusterClient(nodeid)
if pclient == nil {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.Call(%s): NodeId %d is not find.", NodeServiceMethod, nodeid)
return fmt.Errorf("CCluster.Call(%s): NodeId %d is not find.", NodeServiceMethod, nodeid)
}
err := pclient.Call(callServiceName, args, reply)
if err != nil {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.Call(%s) is fail:%v.", callServiceName, err)
}
return err
}
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.Call(%s) fail.", NodeServiceMethod)
return fmt.Errorf("CCluster.Call(%s) fail.", NodeServiceMethod)
}
func (slf *CCluster) GetNodeList(NodeServiceMethod string, rpcServerMethod *string, rpcServiceName *string) []int {
var nodename string
var servicename string
var methodname string
var nodeidList []int
parts := strings.Split(NodeServiceMethod, ".")
if len(parts) == 2 {
servicename = parts[0]
methodname = parts[1]
} else if len(parts) == 3 {
nodename = parts[0]
servicename = parts[1]
methodname = parts[2]
} else {
return nodeidList
}
if nodename == "" {
nodeidList = make([]int, 0)
if servicename[:1] == "_" {
servicename = servicename[1:]
nodeidList = append(nodeidList, GetNodeId())
} else {
nodeidList = slf.cfg.GetIdByService(servicename, "")
}
} else {
nodeidList = slf.GetIdByNodeService(nodename, servicename)
}
if rpcServiceName != nil {
*rpcServiceName = servicename
}
if rpcServerMethod != nil {
*rpcServerMethod = servicename + "." + methodname
}
return nodeidList
}
//GetNodeIdByServiceName 根据服务名查找nodeid servicename服务名 bOnline是否需要查找在线服务
func (slf *CCluster) GetNodeIdByServiceName(servicename string, bOnline bool) []int {
nodeIDList := slf.cfg.GetIdByService(servicename, "")
if bOnline {
ret := make([]int, 0, len(nodeIDList))
for _, nodeid := range nodeIDList {
if slf.CheckNodeIsConnectedByID(nodeid) {
ret = append(ret, nodeid)
}
}
return ret
}
return nodeIDList
}
//根据Service获取负载均衡信息
//负载均衡的策略是从配置获取所有配置了该服务的NodeId 并按NodeId排序 每个node负责处理数组index所在的那一部分
func (slf *CCluster) GetBalancingInfo(currentNodeId int, servicename string, inSubNet bool) (*BalancingInfo, error) {
subNetName := ""
if inSubNet {
if node, ok := slf.cfg.mapIdNode[currentNodeId]; ok {
subNetName = node.SubNetName
} else {
return nil, fmt.Errorf("[cluster.GetBalancingInfo] cannot find node %d", currentNodeId)
}
}
lst := slf.cfg.GetIdByService(servicename, subNetName)
// if len(lst) <= 0 {
// return nil, fmt.Errorf("[cluster.GetBalancingInfo] cannot find service %s in any node", servicename)
// }
sort.Ints(lst)
ret := &BalancingInfo{
NodeId: currentNodeId,
ServiceName: servicename,
TotalNum: len(lst),
MyIndex: -1,
NodeList: lst,
}
if _, ok := slf.cfg.mapIdNode[currentNodeId]; ok {
for i, v := range lst {
if v == currentNodeId {
ret.MyIndex = i
break
}
}
}
return ret, nil
}
func (slf *CCluster) CheckNodeIsConnectedByID(nodeid int) bool {
if nodeid == GetNodeId() {
return true
}
pclient := slf.GetRpcClientByNodeId(nodeid)
if pclient == nil {
return false
}
return pclient.IsConnected()
}
func (slf *CCluster) GetRpcClientByNodeId(nodeid int) *RpcClient {
pclient, ok := slf.nodeclient[nodeid]
if ok == false {
return nil
}
return pclient
}
func (slf *CCluster) Go(bCast bool, NodeServiceMethod string, args interface{}, queueModle bool) error {
return slf.goImpl(bCast, NodeServiceMethod, args, queueModle, true)
}
func (slf *CCluster) goImpl(bCast bool, NodeServiceMethod string, args interface{}, queueModle bool, log bool) error {
var callServiceName string
var serviceName string
nodeidList := slf.GetNodeList(NodeServiceMethod, &callServiceName, &serviceName)
if len(nodeidList) < 1 {
err := fmt.Errorf("CCluster.Go(%s) not find nodes.", NodeServiceMethod)
if log {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, err.Error())
}
return err
}
if bCast == false && len(nodeidList) > 1 {
return fmt.Errorf("CCluster.Go(%s) find more nodes", NodeServiceMethod)
}
for _, nodeid := range nodeidList {
if nodeid == GetNodeId() {
iService := service.InstanceServiceMgr().FindService(serviceName)
if iService == nil {
return fmt.Errorf("CCluster.Go(%s) cannot find service %s", NodeServiceMethod, serviceName)
}
if iService.IsInit() == false {
err := fmt.Errorf("CCluster.Call(%s): NodeId %d is not init.", NodeServiceMethod, nodeid)
if log {
service.GetLogger().Printf(sysmodule.LEVER_WARN, err.Error())
}
return err
}
replyCall := slf.LocalRpcClient.Go(callServiceName, args, nil, nil, queueModle)
if replyCall.Error != nil {
err := fmt.Errorf("CCluster.Go(%s) fail:%v.", NodeServiceMethod, replyCall.Error)
if log {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, err.Error())
} else {
return err
}
}
} else {
pclient := slf.GetClusterClient(nodeid)
if pclient == nil {
err := fmt.Errorf("CCluster.Go(%s) NodeId %d not find client", NodeServiceMethod, nodeid)
if log {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, err.Error())
}
return err
}
replyCall := pclient.Go(callServiceName, args, nil, nil, queueModle)
if replyCall.Error != nil {
err := fmt.Errorf("CCluster.Go(%s) fail:%v.", NodeServiceMethod, replyCall.Error)
if log {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, err.Error())
}
return err
}
}
}
return nil
}
func (slf *CCluster) CallNode(nodeid int, servicemethod string, args interface{}, reply interface{}) error {
pclient := slf.GetClusterClient(nodeid)
if pclient == nil {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.CallNode(%d,%s) NodeId not find client", nodeid, servicemethod)
return fmt.Errorf("Call: NodeId %d is not find.", nodeid)
}
err := pclient.Call(servicemethod, args, reply)
if err != nil {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.CallNode(%d,%s) fail:%v", nodeid, servicemethod, err)
}
return err
}
func (slf *CCluster) GoNode(nodeid int, args interface{}, servicemethod string, queueModle bool) error {
pclient := slf.GetClusterClient(nodeid)
if pclient == nil {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.GoNode(%d,%s) NodeId not find client", nodeid, servicemethod)
return fmt.Errorf("CCluster.GoNode(%d,%s) NodeId not find client", nodeid, servicemethod)
}
replyCall := pclient.Go(servicemethod, args, nil, nil, queueModle)
if replyCall.Error != nil {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.GoNode(%d,%s) fail:%v", nodeid, servicemethod, replyCall.Error)
}
return replyCall.Error
}
func (ws *CCluster) OnFetchService(iservice service.IService) error {
rpc.RegisterName(iservice.GetServiceName(), "RPC_", iservice)
return nil
}
func (slf *CCluster) CallRandomService(NodeServiceMethod string, args interface{}, reply interface{}) error {
var servicename string
parts := strings.Split(NodeServiceMethod, ".")
if len(parts) == 2 {
servicename = parts[0]
} else {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.CallRandomService(%s) method err", NodeServiceMethod)
return fmt.Errorf("CCluster.GoNode(%s) NodeId method err", NodeServiceMethod)
}
nodeList := slf.GetNodeIdByServiceName(servicename, true)
if len(nodeList) < 1 {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.CallRandomService(%s) no node is online", NodeServiceMethod)
return fmt.Errorf("CCluster.GoNode(%s) no node is online", NodeServiceMethod)
}
nodeIndex := rand.Intn(len(nodeList))
nodeID := nodeList[nodeIndex]
return slf.CallNode(nodeID, NodeServiceMethod, args, reply)
}
func (slf *CCluster) GoRandomService(NodeServiceMethod string, args interface{}, queueModle bool) error {
var servicename string
parts := strings.Split(NodeServiceMethod, ".")
if len(parts) == 2 {
servicename = parts[0]
} else {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.CallRandomService(%s) method err", NodeServiceMethod)
return fmt.Errorf("CCluster.GoNode(%s) NodeId method err", NodeServiceMethod)
}
nodeList := slf.GetNodeIdByServiceName(servicename, true)
if len(nodeList) < 1 {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.CallRandomService(%s) no node is online", NodeServiceMethod)
return fmt.Errorf("CCluster.GoNode(%s) no node is online", NodeServiceMethod)
}
nodeIndex := rand.Intn(len(nodeList))
nodeID := nodeList[nodeIndex]
return slf.GoNode(nodeID, args, NodeServiceMethod, queueModle)
}
//向远程服务器调用
//Node.servicename.methodname
//servicename.methodname
func Call(NodeServiceMethod string, args interface{}, reply interface{}) error {
return InstanceClusterMgr().Call(NodeServiceMethod, args, reply)
}
func (slf *CCluster) CallEx(NodeServiceMethod string, args interface{}, reply interface{}) *RpcCallResult {
return slf.rawcall(NodeServiceMethod, args, reply, false)
}
type RpcCallResult struct {
chanRet chan *rpc.Call
err error
rets *rpc.Call
}
func (slf *RpcCallResult) Make() {
slf.chanRet = make(chan *rpc.Call, 1)
slf.rets = nil
}
func (slf *RpcCallResult) WaitReturn(waittm time.Duration) error {
if slf.chanRet == nil {
return errors.New("cannot make rpccallresult")
}
if waittm <= 0 {
select {
case ret := <-slf.chanRet:
return ret.Error
}
} else {
//
select {
case ret := <-slf.chanRet:
return ret.Error
case <-time.After(waittm):
return errors.New("is time out")
}
}
return errors.New("unknow error.")
}
func (slf *CCluster) rawcall(NodeServiceMethod string, args interface{}, reply interface{}, queueModle bool) *RpcCallResult {
var rpcRet RpcCallResult
rpcRet.Make()
var callServiceName string
var serviceName string
nodeidList := slf.GetNodeList(NodeServiceMethod, &callServiceName, &serviceName)
if len(nodeidList) > 1 || len(nodeidList) < 1 {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.Call(%s) find nodes count %d is error.", NodeServiceMethod, len(nodeidList))
rpcRet.err = fmt.Errorf("CCluster.Call(%s) find nodes count %d is error.", NodeServiceMethod, len(nodeidList))
return &rpcRet
}
nodeid := nodeidList[0]
if nodeid == GetNodeId() {
//判断服务是否已经完成初始化
iService := service.InstanceServiceMgr().FindService(serviceName)
if iService == nil {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.Call(%s): NodeId %d cannot find service.", NodeServiceMethod, nodeid)
rpcRet.err = fmt.Errorf("CCluster.Call(%s): NodeId %d cannot find service..", NodeServiceMethod, nodeid)
return &rpcRet
}
if iService.IsInit() == false {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.Call(%s): NodeId %d is not init.", NodeServiceMethod, nodeid)
rpcRet.err = fmt.Errorf("CCluster.Call(%s): NodeId %d is not init.", NodeServiceMethod, nodeid)
return &rpcRet
}
rpcRet.rets = slf.LocalRpcClient.Go(callServiceName, args, reply, rpcRet.chanRet, queueModle)
return &rpcRet
}
pclient := slf.GetClusterClient(nodeid)
if pclient == nil {
service.GetLogger().Printf(sysmodule.LEVER_ERROR, "CCluster.Call(%s): NodeId %d is not find.", NodeServiceMethod, nodeid)
rpcRet.err = fmt.Errorf("CCluster.Call(%s): NodeId %d is not find.", NodeServiceMethod, nodeid)
return &rpcRet
}
rpcRet.rets = pclient.Go(callServiceName, args, reply, rpcRet.chanRet, queueModle)
return &rpcRet
}
func CallEx(NodeServiceMethod string, args interface{}, reply interface{}) *RpcCallResult {
return InstanceClusterMgr().rawcall(NodeServiceMethod, args, reply, false)
}
func CallNode(NodeId int, servicemethod string, args interface{}, reply interface{}) error {
return InstanceClusterMgr().CallNode(NodeId, servicemethod, args, reply)
}
func GoNode(NodeId int, servicemethod string, args interface{}) error {
return InstanceClusterMgr().GoNode(NodeId, args, servicemethod, false)
}
func Go(NodeServiceMethod string, args interface{}) error {
return InstanceClusterMgr().Go(false, NodeServiceMethod, args, false)
}
func CastGo(NodeServiceMethod string, args interface{}) error {
return InstanceClusterMgr().Go(true, NodeServiceMethod, args, false)
}
func GoNodeQueue(NodeId int, servicemethod string, args interface{}) error {
return InstanceClusterMgr().GoNode(NodeId, args, servicemethod, true)
}
func GoQueue(NodeServiceMethod string, args interface{}) error {
return InstanceClusterMgr().Go(false, NodeServiceMethod, args, true)
}
//在GoQueue的基础上增加是否写日志参数
func GoQueueEx(NodeServiceMethod string, args interface{}, log bool) error {
return InstanceClusterMgr().goImpl(false, NodeServiceMethod, args, true, log)
}
func CastGoQueue(NodeServiceMethod string, args interface{}) error {
return InstanceClusterMgr().Go(true, NodeServiceMethod, args, true)
}
//GetNodeIdByServiceName 根据服务名查找nodeid serviceName服务名 bOnline是否需要查找在线服务
func GetNodeIdByServiceName(serviceName string, bOnline bool) []int {
return InstanceClusterMgr().GetNodeIdByServiceName(serviceName, bOnline)
}
//获取服务的负载均衡信息
//负载均衡的策略是从配置获取所有配置了该服务的NodeId 并按NodeId排序 每个node负责处理数组index所在的那一部分
func GetBalancingInfo(currentNodeId int, servicename string, inSubNet bool) (*BalancingInfo, error) {
return InstanceClusterMgr().GetBalancingInfo(currentNodeId, servicename, inSubNet)
}
//随机选择在线的node发送
func CallRandomService(NodeServiceMethod string, args interface{}, reply interface{}) error {
return InstanceClusterMgr().CallRandomService(NodeServiceMethod, args, reply)
}
func GoRandomService(NodeServiceMethod string, args interface{}) error {
return InstanceClusterMgr().GoRandomService(NodeServiceMethod, args, false)
}
func GoRandomServiceQueue(NodeServiceMethod string, args interface{}) error {
return InstanceClusterMgr().GoRandomService(NodeServiceMethod, args, true)
}
var _self *CCluster
func InstanceClusterMgr() *CCluster {
if _self == nil {
_self = new(CCluster)
_self.innerLocalServiceList = make(map[string]bool)
return _self
}
return _self
}
func (slf *CCluster) GetIdByNodeService(NodeName string, serviceName string) []int {
return slf.cfg.GetIdByNodeService(NodeName, serviceName)
}
func (slf *CCluster) HasLocalService(serviceName string) bool {
_, ok := slf.innerLocalServiceList[serviceName]
return slf.cfg.HasLocalService(serviceName) || ok
}
func (slf *CCluster) HasInit(serviceName string) bool {
return slf.cfg.HasLocalService(serviceName)
}
func GetNodeId() int {
return _self.cfg.currentNode.NodeID
}
func (slf *CCluster) AddLocalService(iservice service.IService) {
servicename := fmt.Sprintf("%T", iservice)
parts := strings.Split(servicename, ".")
if len(parts) != 2 {
service.GetLogger().Printf(service.LEVER_ERROR, "BaseService.Init: service name is error: %q", servicename)
}
servicename = parts[1]
slf.innerLocalServiceList[servicename] = true
}
func GetNodeName(nodeid int) string {
return _self.cfg.GetNodeNameByNodeId(nodeid)
}
func DynamicCall(address string, serviceMethod string, args interface{}, reply interface{}) error {
rpcClient, err := rpc.DialTimeOut("tcp", address, time.Second*1)
if err != nil {
return err
}
defer rpcClient.Close()
err = rpcClient.Call(serviceMethod, args, reply)
if err != nil {
return err
}
return nil
}
func GetRpcServer() *rpc.Server{
return &cluster.rpcServer
}

341
cluster/config.go
View File

@@ -1,341 +0,0 @@
package cluster
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"strings"
)
//负载均衡信息
type BalancingInfo struct {
NodeId int //我的nodeId
ServiceName string //负载均衡的ServiceName
TotalNum int //总共有多少个协同Node
MyIndex int //负责的index [0, TotalNum)
NodeList []int //所有协同的node列表 按NodeId升序排列
}
//判断hash后的Id是否命中我的NodeId
func (slf *BalancingInfo) Hit(hashId int) bool {
if hashId >= 0 && slf.TotalNum > 0 && slf.MyIndex >= 0 {
return hashId%slf.TotalNum == slf.MyIndex
}
return false
}
//判断命中的NodeId，-1表示无法取得
func (slf *BalancingInfo) GetHitNodeId(hashId int) int {
if hashId >= 0 && slf.TotalNum > 0 {
if idx := hashId % slf.TotalNum; idx >= 0 && idx < len(slf.NodeList) {
return slf.NodeList[idx]
}
}
return -1
}
type CNodeCfg struct {
NodeID int
NodeName string
ServiceList []string
ClusterNode []string
}
type CNode struct {
SubNetName string
NodeID int
NodeName string //SubNetName.NodeName
ServerAddr string
ServiceList map[string]bool
}
type SubNetNodeInfo struct {
SubNetMode string
SubNetName string
PublicServiceList []string
NodeList []CNodeCfg //配置列表
//mapClusterNodeService map[string][]CNode //map[nodename] []CNode
//mapClusterServiceNode map[string][]CNode //map[servicename] []CNode
}
type ClusterConfig struct {
SubNet []SubNetNodeInfo
//CurrentSubNetIdx int
mapIdNode map[int]CNode
currentNode CNode //当前node
mapClusterNodeService map[string][]CNode //map[nodename] []CNode
mapClusterServiceNode map[string][]CNode //map[servicename] []CNode
}
func GenNodeName(subnetName string, nodename string) string {
parts := strings.Split(nodename, ".")
if len(parts) < 2 {
return subnetName + "." + nodename
}
return nodename
}
func AddCluster(clusterNodeNameList *[]string, nodename string) bool {
for _, n := range *clusterNodeNameList {
if n == nodename {
return false
}
}
*clusterNodeNameList = append(*clusterNodeNameList, nodename)
return true
}
func ReadCfg(path string, nodeid int, mapNodeData map[int]NodeData) (*ClusterConfig, error) {
clsCfg := &ClusterConfig{}
clsCfg.mapIdNode = map[int]CNode{}
clsCfg.mapClusterNodeService = make(map[string][]CNode, 1)
clsCfg.mapClusterServiceNode = make(map[string][]CNode, 1)
//1.加载解析配置
d, err := ioutil.ReadFile(path)
if err != nil {
fmt.Printf("Read File %s Error!", path)
return nil, err
}
err = json.Unmarshal(d, clsCfg)
if err != nil {
fmt.Printf("Read File %s ,%s Error!", path, err)
return nil, err
}
//存储所有的nodeid对应cnode信息
var clusterNodeNameList []string
for _, c := range clsCfg.SubNet {
for _, v := range c.NodeList {
mapservice := make(map[string]bool, 1)
for _, s := range v.ServiceList {
mapservice[s] = true
}
nodeData, ok := mapNodeData[v.NodeID]
if ok == false {
return nil, errors.New(fmt.Sprintf("Cannot find node id %d in nodeconfig.json file!", v.NodeID))
}
node := CNode{c.SubNetName, v.NodeID, c.SubNetName + "." + v.NodeName, nodeData.NodeAddr, mapservice}
clsCfg.mapIdNode[v.NodeID] = node
if v.NodeID == nodeid {
clsCfg.currentNode = node
for _, servicename := range c.PublicServiceList {
clsCfg.currentNode.ServiceList[servicename] = true
}
for _, nodename := range v.ClusterNode {
AddCluster(&clusterNodeNameList, GenNodeName(c.SubNetName, nodename))
}
AddCluster(&clusterNodeNameList, GenNodeName(c.SubNetName, v.NodeName))
}
}
}
if clsCfg.currentNode.NodeID == 0 {
return nil, errors.New(fmt.Sprintf("Cannot find NodeId %d in cluster.json!", nodeid))
}
//如果集群是FULL模式
if strings.ToUpper(clsCfg.GetClusterMode()) == "FULL" {
for _, subnet := range clsCfg.SubNet {
if subnet.SubNetName == clsCfg.currentNode.SubNetName {
for _, nodes := range subnet.NodeList {
AddCluster(&clusterNodeNameList, GenNodeName(subnet.SubNetName, nodes.NodeName))
}
}
}
}
for _, clusternodename := range clusterNodeNameList {
for _, c := range clsCfg.SubNet {
for _, nodecfg := range c.NodeList {
if clusternodename != c.SubNetName+"."+nodecfg.NodeName {
continue
}
n, ok := clsCfg.mapIdNode[nodecfg.NodeID]
if ok == false {
return nil, errors.New(fmt.Sprintf("Cannot find NodeId %d in cluster.json!", nodecfg.NodeID))
}
clsCfg.mapClusterNodeService[nodecfg.NodeName] = append(clsCfg.mapClusterNodeService[nodecfg.NodeName], n)
for _, sname := range nodecfg.ServiceList {
clsCfg.mapClusterServiceNode[sname] = append(clsCfg.mapClusterServiceNode[sname], n)
}
}
}
}
return clsCfg, nil
}
func (slf *ClusterConfig) GetIdByService(serviceName, subNetName string) []int {
var nodeidlist = []int{}
nodeList, ok := slf.mapClusterServiceNode[serviceName]
if ok == true {
nodeidlist = make([]int, 0, len(nodeList))
for _, v := range nodeList {
if subNetName == "" || subNetName == v.SubNetName {
nodeidlist = append(nodeidlist, v.NodeID)
}
}
}
return nodeidlist
}
func (slf *ClusterConfig) GetIdByNodeService(NodeName string, serviceName string) []int {
var nodeidlist []int
nodeidlist = make([]int, 0)
if NodeName == slf.currentNode.NodeName {
nodeidlist = append(nodeidlist, slf.currentNode.NodeID)
}
v, ok := slf.mapClusterNodeService[NodeName]
if ok == false {
return nodeidlist
}
for _, n := range v {
_, ok = n.ServiceList[serviceName]
if ok == true {
nodeidlist = append(nodeidlist, n.NodeID)
}
}
return nodeidlist
}
func (slf *ClusterConfig) HasLocalService(serviceName string) bool {
_, ok := slf.currentNode.ServiceList[serviceName]
return ok == true
}
func IsExistsNode(nodelist []CNode, pNode *CNode) bool {
for _, node := range nodelist {
if node.NodeID == pNode.NodeID {
return true
}
}
return false
}
func (slf *ClusterConfig) GetNodeNameByNodeId(nodeid int) string {
node, ok := slf.mapIdNode[nodeid]
if ok == false {
return ""
}
return node.NodeName
}
func (slf *ClusterConfig) GetClusterMode() string {
//SubNet []SubNetNodeInfo
for _, subnet := range slf.SubNet {
if subnet.SubNetName == slf.currentNode.SubNetName {
return subnet.SubNetMode
}
}
return ""
}
type CACfg struct {
CertFile string
KeyFile string
}
//NodeConfig ...
type NodeData struct {
NodeID int
LogLevel uint
HttpPort uint16
WSPort uint16
ListenPort uint16
NodeAddr string
CAFile []CACfg
Environment string
IsListenLog int
IsSendErrorMail int
}
type NodeConfig struct {
Public NodeData
NodeList []NodeData
}
//ReadNodeConfig ...
func ReadAllNodeConfig(path string) (map[int]NodeData, error) {
c := &NodeConfig{}
d, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
err = json.Unmarshal(d, c)
if err != nil {
return nil, err
}
var mapNodeData map[int]NodeData
mapNodeData = map[int]NodeData{}
//data = c.Public
for _, v := range c.NodeList {
mapNodeData[v.NodeID] = v
}
return mapNodeData, nil
}
func ReadNodeConfig(path string, nodeid int) (*NodeData, error) {
c := &NodeConfig{}
d, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
err = json.Unmarshal(d, c)
if err != nil {
return nil, err
}
var data NodeData
data = c.Public
for _, v := range c.NodeList {
if v.NodeID == nodeid {
data = v
if v.Environment == "" && c.Public.Environment != "" {
data.Environment = c.Public.Environment
}
if len(v.CAFile) == 0 && len(c.Public.CAFile) != 0 {
data.CAFile = c.Public.CAFile
}
if v.HttpPort == 0 && c.Public.HttpPort != 0 {
data.HttpPort = c.Public.HttpPort
}
if v.WSPort == 0 && c.Public.WSPort != 0 {
data.WSPort = c.Public.WSPort
}
if v.LogLevel == 0 && c.Public.LogLevel != 0 {
data.LogLevel = c.Public.LogLevel
}
if v.IsListenLog == 0 && c.Public.IsListenLog != 0 {
data.IsListenLog = c.Public.IsListenLog
}
break
}
}
return &data, nil
}

192
cluster/parsecfg.go Normal file
View File

@@ -0,0 +1,192 @@
package cluster
import (
"encoding/json"
"fmt"
"io/ioutil"
"strings"
)
func (slf *Cluster) ReadClusterConfig(filepath string) (*SubNet,error) {
c := &SubNet{}
d, err := ioutil.ReadFile(filepath)
if err != nil {
return nil, err
}
err = json.Unmarshal(d, c)
if err != nil {
return nil, err
}
return c,nil
}
func (slf *Cluster) ReadServiceConfig(filepath string) (map[string]interface{},error) {
c := map[string]interface{}{}
d, err := ioutil.ReadFile(filepath)
if err != nil {
return nil, err
}
err = json.Unmarshal(d, &c)
if err != nil {
return nil, err
}
return c,nil
}
func (slf *Cluster) ReadAllSubNetConfig() error {
clusterCfgPath :=strings.TrimRight(configdir,"/") +"/cluster"
fileInfoList,err := ioutil.ReadDir(clusterCfgPath)
if err != nil {
return err
}
slf.mapSubNetInfo =map[string] SubNet{}
for _,f := range fileInfoList{
if f.IsDir() == true {
subnetinfo,err:=slf.ReadClusterConfig(strings.TrimRight(strings.TrimRight(clusterCfgPath,"/"),"\\")+"/"+f.Name()+"/"+"cluster.json")
if err != nil {
return err
}
slf.mapSubNetInfo[f.Name()] = *subnetinfo
}
}
return nil
}
func (slf *Cluster) ReadLocalSubNetServiceConfig(subnet string) error {
clusterCfgPath :=strings.TrimRight(configdir,"/") +"/cluster"
fileInfoList,err := ioutil.ReadDir(clusterCfgPath)
if err != nil {
return err
}
slf.mapSubNetInfo =map[string] SubNet{}
for _,f := range fileInfoList{
if f.IsDir() == true && f.Name()==subnet{ //同一子网
localNodeServiceCfg,err:=slf.ReadServiceConfig(strings.TrimRight(strings.TrimRight(clusterCfgPath,"/"),"\\")+"/"+f.Name()+"/"+"service.json")
if err != nil {
return err
}
slf.localNodeServiceCfg =localNodeServiceCfg
}
}
return nil
}
func (slf *Cluster) InitCfg(currentNodeId int) error{
//mapSubNetInfo := map[string] SubNet{} //子网名称，子网信息
mapSubNetNodeInfo := map[string]map[int]NodeInfo{} //map[子网名称]map[NodeId]NodeInfo
localSubNetMapNode := map[int]NodeInfo{} //本子网内 map[NodeId]NodeInfo
localSubNetMapService := map[string][]NodeInfo{} //本子网内所有ServiceName对应的结点列表
localNodeMapService := map[string]interface{}{} //本Node支持的服务
localNodeInfo := NodeInfo{}
err := slf.ReadAllSubNetConfig()
//分析配置
var localSubnetName string
for subnetName,subnetInfo := range slf.mapSubNetInfo {
for _,nodeinfo := range subnetInfo.NodeList {
//装载slf.mapNodeInfo
_,ok := mapSubNetNodeInfo[subnetName]
if ok == false {
mapnodeInfo := make(map[int]NodeInfo,1)
mapnodeInfo[nodeinfo.NodeId] = nodeinfo
mapSubNetNodeInfo[subnetName] = mapnodeInfo
}else{
mapSubNetNodeInfo[subnetName][nodeinfo.NodeId] = nodeinfo
}
//判断本进程的子网
if nodeinfo.NodeId == currentNodeId {
localSubnetName = subnetName
}
}
}
//装载
subnet,ok := slf.mapSubNetInfo[localSubnetName]
if ok == false {
return fmt.Errorf("NodeId %d not in any subnet",currentNodeId)
}
for _,nodeinfo := range subnet.NodeList {
localSubNetMapNode[nodeinfo.NodeId] = nodeinfo
//装载本Node进程所有的服务
if nodeinfo.NodeId == currentNodeId {
for _,s := range nodeinfo.ServiceList {
servicename := s
if strings.Index(s,"_") == 0 {
servicename = s[1:]
}
localNodeMapService[servicename] = nil
}
localNodeInfo = nodeinfo
}
for _,s := range nodeinfo.ServiceList {
//以_打头的，表示只在本机进程，不对整个子网开发
if strings.Index(s,"_") == 0 {
continue
}
if _,ok := localSubNetMapService[s];ok== true{
localSubNetMapService[s] = []NodeInfo{}
}
localSubNetMapService[s] = append(localSubNetMapService[s],nodeinfo)
}
}
if localNodeInfo.NodeId == 0 {
return fmt.Errorf("Canoot find NodeId %d not in any config file.",currentNodeId)
}
slf.mapSubNetNodeInfo=mapSubNetNodeInfo
slf.localSubNetMapNode=localSubNetMapNode
slf.localSubNetMapService = localSubNetMapService
slf.localNodeMapService = localNodeMapService
slf.localsubnet = subnet
slf.localNodeInfo =localNodeInfo
//读取服务
return err
}
func (slf *Cluster) IsConfigService(servicename string) bool {
_,ok := slf.localNodeMapService[servicename]
return ok
}
func (slf *Cluster) GetNodeIdByService(servicename string) []int{
var nodelist []int
nodeInfoList,ok := slf.localSubNetMapService[servicename]
if ok == true {
for _,node := range nodeInfoList {
nodelist = append(nodelist,node.NodeId)
}
}
return nodelist
}
func (slf *Cluster) GetServiceCfg(servicename string) interface{}{
v,ok := slf.localNodeServiceCfg[servicename]
if ok == false{
return nil
}
return v
}