Skip to content
This repository has been archived by the owner on Apr 5, 2023. It is now read-only.

Commit

Permalink
[#32] Add CouchDB implementation for block storage
Browse files Browse the repository at this point in the history
Signed-off-by: Firas Qutishat <[email protected]>
Signed-off-by: sudesh.shetty <[email protected]>
  • Loading branch information
sudeshrshetty authored and fqutishat committed May 11, 2019
1 parent 2f9256a commit cbafe5d
Show file tree
Hide file tree
Showing 26 changed files with 2,298 additions and 24 deletions.
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,6 @@ require (

)

replace github.com/hyperledger/fabric => github.com/trustbloc/fabric-mod v0.0.0-20190508134351-4beae6ee306b
replace github.com/hyperledger/fabric => github.com/trustbloc/fabric-mod v0.0.0-20190510235640-ff89b7580e81

replace github.com/hyperledger/fabric/extensions => github.com/trustbloc/fabric-mod/extensions v0.0.0-20190508134351-4beae6ee306b
replace github.com/hyperledger/fabric/extensions => github.com/trustbloc/fabric-mod/extensions v0.0.0-20190510235640-ff89b7580e81
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -182,8 +182,12 @@ github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc/go.mod h1:eyZnKCc955u
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/trustbloc/fabric-mod v0.0.0-20190508134351-4beae6ee306b h1:xZCwe/MyQYmCGZglxiayosPeDkM49OrLj9QwWyz/Wfw=
github.com/trustbloc/fabric-mod v0.0.0-20190508134351-4beae6ee306b/go.mod h1:y/yp/cF5M9+xddNmn6f24tz/PrDjgGCzOk5/0WCKLkU=
github.com/trustbloc/fabric-mod v0.0.0-20190510235640-ff89b7580e81 h1:CWmIqCQjIOmc12d8/byYfAYJjwwlUXIVJlRANX+syUE=
github.com/trustbloc/fabric-mod v0.0.0-20190510235640-ff89b7580e81/go.mod h1:y/yp/cF5M9+xddNmn6f24tz/PrDjgGCzOk5/0WCKLkU=
github.com/trustbloc/fabric-mod/extensions v0.0.0-20190508134351-4beae6ee306b h1:40O0t+2qIGqVKE6SMkx5FmKCcS4rsDF7QS3QGAHYIVw=
github.com/trustbloc/fabric-mod/extensions v0.0.0-20190508134351-4beae6ee306b/go.mod h1:SnMkFTM0DFEsFcZqZp4P0Gqm+ryD8iwx9cEGW8qrju0=
github.com/trustbloc/fabric-mod/extensions v0.0.0-20190510235640-ff89b7580e81 h1:eD2FCLOj1PWiqRM9NadS+LeHMu5UbiqE9D/cPN8bOUA=
github.com/trustbloc/fabric-mod/extensions v0.0.0-20190510235640-ff89b7580e81/go.mod h1:SnMkFTM0DFEsFcZqZp4P0Gqm+ryD8iwx9cEGW8qrju0=
github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/vishvananda/netlink v1.0.0 h1:bqNY2lgheFIu1meHUFSH3d7vG93AFyqg3oGbJCOJgSM=
Expand Down
11 changes: 8 additions & 3 deletions mod/peer/blkstorage/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,19 @@ package blkstorage
import (
"github.com/hyperledger/fabric/common/ledger/blkstorage"
"github.com/hyperledger/fabric/common/ledger/blkstorage/fsblkstorage"
"github.com/trustbloc/fabric-peer-ext/pkg/blkstorage/cdbblkstorage"
)

//NewProvider is redirect hook for fabric/fsblkstorage NewProvider()
//NewProvider returns couchdb blockstorage provider
func NewProvider(conf *fsblkstorage.Conf, indexConfig *blkstorage.IndexConfig) blkstorage.BlockStoreProvider {
return fsblkstorage.NewProvider(conf, indexConfig)
pvdr, err := cdbblkstorage.NewProvider(indexConfig)
if err != nil {
panic(err)
}
return pvdr
}

//NewConf is redirect hook for fabric/fsblkstorage NewConf()
//NewConf is returns file system based blockstorage conf
func NewConf(blockStorageDir string, maxBlockfileSize int) *fsblkstorage.Conf {
return fsblkstorage.NewConf(blockStorageDir, maxBlockfileSize)
}
16 changes: 3 additions & 13 deletions mod/peer/blkstorage/store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,16 @@ SPDX-License-Identifier: Apache-2.0
package blkstorage

import (
"io/ioutil"
"os"
"testing"

"github.com/hyperledger/fabric/common/ledger/blkstorage"
"github.com/hyperledger/fabric/core/ledger/ledgerconfig"
"github.com/spf13/viper"
"github.com/hyperledger/fabric/extensions/testutil"
"github.com/stretchr/testify/require"
)

func TestNewProvider(t *testing.T) {
cleanup := setupPath(t)
defer cleanup()
_, _, destroy := testutil.SetupExtTestEnv()
defer destroy()
require.NotEmpty(t, NewProvider(NewConf(ledgerconfig.GetBlockStorePath(), ledgerconfig.GetMaxBlockfileSize()), &blkstorage.IndexConfig{}))
}

func setupPath(t *testing.T) (cleanup func()) {
tempDir, err := ioutil.TempDir("", "transientstore")
require.NoError(t, err)

viper.Set("peer.fileSystemPath", tempDir)
return func() { os.RemoveAll(tempDir) }
}
2 changes: 1 addition & 1 deletion mod/peer/go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/trustbloc/fabric-peer-ext/mod/peer

replace github.com/hyperledger/fabric => github.com/trustbloc/fabric-mod v0.0.0-20190508134351-4beae6ee306b
replace github.com/hyperledger/fabric => github.com/trustbloc/fabric-mod v0.0.0-20190510235640-ff89b7580e81

replace github.com/hyperledger/fabric/extensions => ./

Expand Down
3 changes: 1 addition & 2 deletions mod/peer/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,7 @@ github.com/go-kit/kit v0.7.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
Expand Down Expand Up @@ -199,6 +197,7 @@ github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc/go.mod h1:eyZnKCc955u
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/trustbloc/fabric-mod v0.0.0-20190508134351-4beae6ee306b h1:xZCwe/MyQYmCGZglxiayosPeDkM49OrLj9QwWyz/Wfw=
github.com/trustbloc/fabric-mod v0.0.0-20190508134351-4beae6ee306b/go.mod h1:y/yp/cF5M9+xddNmn6f24tz/PrDjgGCzOk5/0WCKLkU=
github.com/trustbloc/fabric-mod v0.0.0-20190510235640-ff89b7580e81/go.mod h1:y/yp/cF5M9+xddNmn6f24tz/PrDjgGCzOk5/0WCKLkU=
github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/vishvananda/netlink v1.0.0 h1:bqNY2lgheFIu1meHUFSH3d7vG93AFyqg3oGbJCOJgSM=
Expand Down
67 changes: 67 additions & 0 deletions pkg/blkstorage/cdbblkstorage/block_serialization.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
Copyright SecureKey Technologies Inc. All Rights Reserved.
SPDX-License-Identifier: Apache-2.0
*/

package cdbblkstorage

import (
"github.com/pkg/errors"

"github.com/hyperledger/fabric/protos/common"
"github.com/hyperledger/fabric/protoutil"
)

func extractTxIDFromEnvelope(txEnvelope *common.Envelope) (string, error) {
payload, err := protoutil.GetPayload(txEnvelope)
if err != nil {
return "", nil
}

payloadHeader := payload.Header
channelHeader, err := protoutil.UnmarshalChannelHeader(payloadHeader.ChannelHeader)
if err != nil {
return "", err
}

return channelHeader.TxId, nil
}

func extractTxnEnvelopeFromBlock(block *common.Block, txID string) (*common.Envelope, error) {
blockData := block.GetData()
for _, txEnvelopeBytes := range blockData.GetData() {
envelope, err := protoutil.GetEnvelopeFromBlock(txEnvelopeBytes)
if err != nil {
return nil, err
}

id, err := extractTxIDFromEnvelope(envelope)
if err != nil {
return nil, err
}
if id != txID {
continue
}

txEnvelope, err := protoutil.GetEnvelopeFromBlock(txEnvelopeBytes)
if err != nil {
return nil, err
}

return txEnvelope, nil
}

return nil, errors.Errorf("transaction not found [%s]", txID)
}

func extractEnvelopeFromBlock(block *common.Block, tranNum uint64) (*common.Envelope, error) {
blockData := block.GetData()
envelopes := blockData.GetData()
envelopesLen := uint64(len(envelopes))
if envelopesLen-1 < tranNum {
blockNum := block.GetHeader().GetNumber()
return nil, errors.Errorf("transaction number is invalid [%d, %d, %d]", blockNum, envelopesLen, tranNum)
}
return protoutil.GetEnvelopeFromBlock(envelopes[tranNum])
}
73 changes: 73 additions & 0 deletions pkg/blkstorage/cdbblkstorage/blocks_itr.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
Copyright SecureKey Technologies Inc. All Rights Reserved.
SPDX-License-Identifier: Apache-2.0
*/

package cdbblkstorage

import (
"sync"

"github.com/hyperledger/fabric/common/ledger"
)

// blocksItr - an iterator for iterating over a sequence of blocks
type blocksItr struct {
cdbBlockStore *cdbBlockStore
maxBlockNumAvailable uint64
blockNumToRetrieve uint64
closeMarker bool
closeMarkerLock *sync.Mutex
}

func newBlockItr(cdbBlockStore *cdbBlockStore, startBlockNum uint64) *blocksItr {
return &blocksItr{cdbBlockStore, cdbBlockStore.cpInfo.lastBlockNumber, startBlockNum, false, &sync.Mutex{}}
}

func (itr *blocksItr) waitForBlock(blockNum uint64) uint64 {
itr.cdbBlockStore.cpInfoCond.L.Lock()
defer itr.cdbBlockStore.cpInfoCond.L.Unlock()
for itr.cdbBlockStore.cpInfo.lastBlockNumber < blockNum && !itr.shouldClose() {
logger.Debugf("Going to wait for newer blocks. maxAvailaBlockNumber=[%d], waitForBlockNum=[%d]",
itr.cdbBlockStore.cpInfo.lastBlockNumber, blockNum)
itr.cdbBlockStore.cpInfoCond.Wait()
logger.Debugf("Came out of wait. maxAvailaBlockNumber=[%d]", itr.cdbBlockStore.cpInfo.lastBlockNumber)
}
return itr.cdbBlockStore.cpInfo.lastBlockNumber
}

func (itr *blocksItr) shouldClose() bool {
itr.closeMarkerLock.Lock()
defer itr.closeMarkerLock.Unlock()
return itr.closeMarker
}

// Next moves the cursor to next block and returns true iff the iterator is not exhausted
func (itr *blocksItr) Next() (ledger.QueryResult, error) {
if itr.maxBlockNumAvailable < itr.blockNumToRetrieve {
itr.maxBlockNumAvailable = itr.waitForBlock(itr.blockNumToRetrieve)
}
itr.closeMarkerLock.Lock()
defer itr.closeMarkerLock.Unlock()
if itr.closeMarker {
return nil, nil
}

nextBlock, err := itr.cdbBlockStore.RetrieveBlockByNumber(itr.blockNumToRetrieve)
if err != nil {
return nil, err
}
itr.blockNumToRetrieve++
return nextBlock, nil
}

// Close releases any resources held by the iterator
func (itr *blocksItr) Close() {
itr.cdbBlockStore.cpInfoCond.L.Lock()
defer itr.cdbBlockStore.cpInfoCond.L.Unlock()
itr.closeMarkerLock.Lock()
defer itr.closeMarkerLock.Unlock()
itr.closeMarker = true
itr.cdbBlockStore.cpInfoCond.Broadcast()
}
Loading

0 comments on commit cbafe5d

Please sign in to comment.