Agents auto-retry connect to heartbeat server alternative addresses

config/agent.json

@@ -10,7 +10,9 @@
     },
     "heartbeat": {
         "enabled": true,
-        "addr": "%%HBS_RPC%%",
+        "addrs": [
+            "%%HBS_RPC%%"
+        ],
         "interval": 60,
         "timeout": 1000
     },

modules/agent/cfg.example.json

@@ -10,7 +10,9 @@
     },
     "heartbeat": {
         "enabled": true,
-        "addr": "127.0.0.1:6030",
+        "addrs": [
+            "127.0.0.1:6030"
+        ],
         "interval": 60,
         "timeout": 1000
     },

modules/agent/cron/builtin.go

@@ -15,16 +15,17 @@
 package cron
 
 import (
-	"github.com/open-falcon/falcon-plus/common/model"
-	"github.com/open-falcon/falcon-plus/modules/agent/g"
 	"log"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/open-falcon/falcon-plus/common/model"
+	"github.com/open-falcon/falcon-plus/modules/agent/g"
 )
 
 func SyncBuiltinMetrics() {
-	if g.Config().Heartbeat.Enabled && g.Config().Heartbeat.Addr != "" {
+	if g.Config().Heartbeat.Enabled && len(g.Config().Heartbeat.Addrs) > 0 {
 		go syncBuiltinMetrics()
 	}
 }

modules/agent/cron/ips.go

@@ -15,14 +15,15 @@
 package cron
 
 import (
-	"github.com/open-falcon/falcon-plus/common/model"
-	"github.com/open-falcon/falcon-plus/modules/agent/g"
 	"log"
 	"time"
+
+	"github.com/open-falcon/falcon-plus/common/model"
+	"github.com/open-falcon/falcon-plus/modules/agent/g"
 )
 
 func SyncTrustableIps() {
-	if g.Config().Heartbeat.Enabled && g.Config().Heartbeat.Addr != "" {
+	if g.Config().Heartbeat.Enabled && len(g.Config().Heartbeat.Addrs) != 0 {
 		go syncTrustableIps()
 	}
 }

modules/agent/cron/plugin.go

@@ -15,12 +15,13 @@
 package cron
 
 import (
-	"github.com/open-falcon/falcon-plus/common/model"
-	"github.com/open-falcon/falcon-plus/modules/agent/g"
-	"github.com/open-falcon/falcon-plus/modules/agent/plugins"
 	"log"
 	"strings"
 	"time"
+
+	"github.com/open-falcon/falcon-plus/common/model"
+	"github.com/open-falcon/falcon-plus/modules/agent/g"
+	"github.com/open-falcon/falcon-plus/modules/agent/plugins"
 )
 
 func SyncMinePlugins() {
@@ -32,7 +33,7 @@ func SyncMinePlugins() {
 		return
 	}
 
-	if g.Config().Heartbeat.Addr == "" {
+	if len(g.Config().Heartbeat.Addrs) == 0 {
 		return
 	}
 

modules/agent/cron/reporter.go

@@ -16,14 +16,15 @@ package cron
 
 import (
 	"fmt"
-	"github.com/open-falcon/falcon-plus/common/model"
-	"github.com/open-falcon/falcon-plus/modules/agent/g"
 	"log"
 	"time"
+
+	"github.com/open-falcon/falcon-plus/common/model"
+	"github.com/open-falcon/falcon-plus/modules/agent/g"
 )
 
 func ReportAgentStatus() {
-	if g.Config().Heartbeat.Enabled && g.Config().Heartbeat.Addr != "" {
+	if g.Config().Heartbeat.Enabled && len(g.Config().Heartbeat.Addrs) != 0 {
 		go reportAgentStatus(time.Duration(g.Config().Heartbeat.Interval) * time.Second)
 	}
 }

modules/agent/g/cfg.go

@@ -31,10 +31,10 @@ type PluginConfig struct {
 }
 
 type HeartbeatConfig struct {
-	Enabled  bool   `json:"enabled"`
-	Addr     string `json:"addr"`
-	Interval int    `json:"interval"`
-	Timeout  int    `json:"timeout"`
+	Enabled  bool     `json:"enabled"`
+	Addrs    []string `json:"addrs"`
+	Interval int      `json:"interval"`
+	Timeout  int      `json:"timeout"`
 }
 
 type TransferConfig struct {

modules/agent/g/rpc.go

@@ -26,9 +26,9 @@ import (
 
 type SingleConnRpcClient struct {
 	sync.Mutex
-	rpcClient *rpc.Client
-	RpcServer string
-	Timeout   time.Duration
+	rpcClient  *rpc.Client
+	RpcServers []string
+	Timeout    time.Duration
 }
 
 func (this *SingleConnRpcClient) close() {
@@ -44,25 +44,29 @@ func (this *SingleConnRpcClient) serverConn() error {
 	}
 
 	var err error
-	var retry int = 1
+	var retry int
 
-	for {
-		if this.rpcClient != nil {
-			return nil
-		}
+	for _, addr := range this.RpcServers {
+		retry = 1
 
-		this.rpcClient, err = net.JsonRpcClient("tcp", this.RpcServer, this.Timeout)
+	RETRY:
+		this.rpcClient, err = net.JsonRpcClient("tcp", addr, this.Timeout)
 		if err != nil {
-			log.Printf("dial %s fail: %v", this.RpcServer, err)
+			log.Println("net.JsonRpcClient failed", err)
 			if retry > 3 {
-				return err
+				continue
 			}
+
 			time.Sleep(time.Duration(math.Pow(2.0, float64(retry))) * time.Second)
 			retry++
-			continue
+			goto RETRY
 		}
-		return err
+		log.Println("connected RPC server", addr)
+
+		return nil
 	}
+
+	return errors.New("connect to RPC servers failed")
 }
 
 func (this *SingleConnRpcClient) Call(method string, args interface{}, reply interface{}) error {
@@ -85,9 +89,9 @@ func (this *SingleConnRpcClient) Call(method string, args interface{}, reply int
 
 	select {
 	case <-time.After(timeout):
-		log.Printf("[WARN] rpc call timeout %v => %v", this.rpcClient, this.RpcServer)
+		log.Printf("[WARN] rpc call timeout %v => %v", this.rpcClient, this.RpcServers)
 		this.close()
-		return errors.New(this.RpcServer + " rpc call timeout")
+		return errors.New("rpc call timeout")
 	case err := <-done:
 		if err != nil {
 			this.close()

modules/agent/g/transfer.go

@@ -45,9 +45,12 @@ func SendMetrics(metrics []*model.MetricValue, resp *model.TransferResponse) {
 }
 
 func initTransferClient(addr string) *SingleConnRpcClient {
+	addrs := []string{
+		addr,
+	}
 	var c *SingleConnRpcClient = &SingleConnRpcClient{
-		RpcServer: addr,
-		Timeout:   time.Duration(Config().Transfer.Timeout) * time.Millisecond,
+		RpcServers: addrs,
+		Timeout:    time.Duration(Config().Transfer.Timeout) * time.Millisecond,
 	}
 	TransferClientsLock.Lock()
 	defer TransferClientsLock.Unlock()

modules/agent/g/var.go

@@ -16,14 +16,16 @@ package g
 
 import (
 	"bytes"
-	"github.com/open-falcon/falcon-plus/common/model"
-	"github.com/toolkits/slice"
+	"fmt"
 	"log"
 	"net"
 	"os"
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/open-falcon/falcon-plus/common/model"
+	"github.com/toolkits/slice"
 )
 
 var Root string
@@ -40,12 +42,15 @@ var LocalIp string
 
 func InitLocalIp() {
 	if Config().Heartbeat.Enabled {
-		conn, err := net.DialTimeout("tcp", Config().Heartbeat.Addr, time.Second*10)
-		if err != nil {
-			log.Println("get local addr failed !")
-		} else {
-			LocalIp = strings.Split(conn.LocalAddr().String(), ":")[0]
-			conn.Close()
+		for _, addr := range Config().Heartbeat.Addrs {
+			conn, err := net.DialTimeout("tcp", addr, time.Second*10)
+			if err != nil {
+				log.Println(fmt.Sprintf("connect to heartbeat server %s failed", addr))
+			} else {
+				defer conn.Close()
+				LocalIp = strings.Split(conn.LocalAddr().String(), ":")[0]
+				break
+			}
 		}
 	} else {
 		log.Println("hearbeat is not enabled, can't get localip")
@@ -57,12 +62,13 @@ var (
 )
 
 func InitRpcClients() {
-	if Config().Heartbeat.Enabled {
+	if Config().Heartbeat.Enabled && len(Config().Heartbeat.Addrs) != 0 {
 		HbsClient = &SingleConnRpcClient{
-			RpcServer: Config().Heartbeat.Addr,
-			Timeout:   time.Duration(Config().Heartbeat.Timeout) * time.Millisecond,
+			RpcServers: Config().Heartbeat.Addrs,
+			Timeout:    time.Duration(Config().Heartbeat.Timeout) * time.Millisecond,
 		}
 	}
+
 }
 
 func SendToTransfer(metrics []*model.MetricValue) {