Merge pull request #45 from liuxuzxx/develop

merge develop to main

README

@@ -74,4 +74,11 @@ Raft是一个共识的算法！
 #### Leader节点宕机了
 #### 有Follower节点断网了
 #### 日志复制缓慢
-#### 新节点加入
+#### 新节点加入
+#### log过大
+```
+log信息的持久化，刚开始的时候选用的是JSON，但是出现的问题是：文件体积过大，解析速度等待验证！
+100w信息，大概需要56MB的空间！当使用JSON进行序列化的时候.
+使用pb之后，大小确实减少了不少，请求50w次的数据量是：10MB空间，对比之前的28MB/50w，约等于1/3的空间
+
+```

go.mod

@@ -5,11 +5,13 @@ go 1.15
 require (
 	github.com/Joker/hpp v1.0.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.9
+	github.com/golang/protobuf v1.4.2
 	github.com/iris-contrib/middleware/cors v0.0.0-20201115103636-07e8bced147f
 	github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
 	github.com/kataras/iris/v12 v12.2.0-alpha.0.20201209053710-4431294a2ee9
 	github.com/mattn/go-colorable v0.1.8 // indirect
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/spf13/viper v1.3.2
 	github.com/yudai/pp v2.0.1+incompatible // indirect
+	google.golang.org/protobuf v1.25.0
 )

pb/db_entry.proto

@@ -0,0 +1,9 @@
+syntax = "proto3";
+package log;
+
+message LogEntry{
+  int64 index = 1;
+  int64 term = 2;
+  string key = 3;
+  string value = 4;
+}

server/log_replication_server.go

@@ -2,11 +2,12 @@ package server
 
 import (
 	"bufio"
-	"encoding/json"
 	"fmt"
+	"github.com/golang/protobuf/proto"
 	"os"
 	"path/filepath"
 	"raft/entity"
+	log "raft/pb"
 	"strconv"
 	"sync"
 )
@@ -16,8 +17,8 @@ import (
 // log复制，持久化到本地
 //
 type LogReplicationServer struct {
-	dataMap  map[string]entity.DBEntry
-	index    int
+	dataMap  map[string]log.LogEntry
+	index    int64
 	path     string
 	fileName string
 	mux      sync.RWMutex
@@ -26,13 +27,13 @@ type LogReplicationServer struct {
 func (l *LogReplicationServer) Save(command entity.CommandRequest) {
 	l.mux.Lock()
 	l.index = l.index + 1
-	entry := entity.DBEntry{
+	entry := &log.LogEntry{
 		Index: l.index,
 		Term:  0,
 		Key:   command.Key,
 		Value: command.Value,
 	}
-	l.dataMap[command.Key+strconv.Itoa(l.index)] = entry
+	l.dataMap[command.Key+strconv.FormatInt(l.index, 10)] = *entry
 	l.appendLog(entry)
 	l.mux.Unlock()
 	//l.debugDataMap()
@@ -45,15 +46,15 @@ func (l *LogReplicationServer) debugDataMap() {
 	}
 }
 
-func (l *LogReplicationServer) appendLog(logEntry entity.DBEntry) {
+func (l *LogReplicationServer) appendLog(logEntry *log.LogEntry) {
 	logFile, err := os.OpenFile(filepath.Join(l.path, l.fileName), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
 	if err != nil {
 		panic(err)
 	}
 	defer logFile.Close()
 	write := bufio.NewWriter(logFile)
-	jsonBytes, _ := json.Marshal(logEntry)
-	_, err = write.WriteString(string(jsonBytes))
+	jsonBytes, _ := proto.Marshal(logEntry)
+	_, err = write.Write(jsonBytes)
 
 	err = write.Flush()
 	if err != nil {
@@ -70,7 +71,7 @@ var LogReplication *LogReplicationServer
 
 func init() {
 	LogReplication = &LogReplicationServer{
-		dataMap:  make(map[string]entity.DBEntry, 0),
+		dataMap:  make(map[string]log.LogEntry, 0),
 		index:    0,
 		path:     "./log",
 		fileName: "log.raft",