Skip to content
This repository was archived by the owner on Aug 31, 2021. It is now read-only.

Commit eb866e9

Browse files
author
ericmeyer
authored
Merge pull request #59 from 8thlight/populate-historical-blocks
Populate historical blocks
2 parents 1ee4079 + 86e5582 commit eb866e9

16 files changed

+317
-19
lines changed

‎Gododir/main.go

+10
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,16 @@ func tasks(p *do.Project) {
2525
do.M{"environment": environment, "$in": "cmd/run"})
2626
})
2727

28+
p.Task("populateBlocks", nil, func(context *do.Context) {
29+
environment := parseEnvironment(context)
30+
startingNumber := context.Args.MayInt(-1, "starting-number")
31+
if startingNumber < 0 {
32+
log.Fatalln("--starting-number required")
33+
}
34+
context.Start(`go run main.go --environment={{.environment}} --starting-number={{.startingNumber}}`,
35+
do.M{"environment": environment, "startingNumber": startingNumber, "$in": "cmd/populate_blocks"})
36+
})
37+
2838
p.Task("migrate", nil, func(context *do.Context) {
2939
environment := parseEnvironment(context)
3040
cfg := config.NewConfig(environment)

‎README.md

+6
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,12 @@ The default location for Ethereum is:
6565
1. Start a blockchain.
6666
2. In a separate terminal start listener (ipcDir location)
6767
- `godo run -- --environment=<some-environment>`
68+
69+
## Retrieving Historical Data
70+
71+
1. Start a blockchain.
72+
2. In a separate terminal start listener (ipcDir location)
73+
- `godo populateBlocks -- --environment=<some-environment> --starting-number=<starting-block-number>`
6874

6975
### Configuring Additional Environments
7076

‎cmd/populate_blocks/main.go

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package main
2+
3+
import (
4+
"flag"
5+
6+
"log"
7+
8+
"fmt"
9+
10+
"github.com/8thlight/vulcanizedb/pkg/config"
11+
"github.com/8thlight/vulcanizedb/pkg/geth"
12+
"github.com/8thlight/vulcanizedb/pkg/history"
13+
"github.com/8thlight/vulcanizedb/pkg/repositories"
14+
"github.com/jmoiron/sqlx"
15+
)
16+
17+
func main() {
18+
environment := flag.String("environment", "", "Environment name")
19+
startingBlockNumber := flag.Int("starting-number", -1, "First block to fill from")
20+
flag.Parse()
21+
cfg := config.NewConfig(*environment)
22+
23+
blockchain := geth.NewGethBlockchain(cfg.Client.IPCPath)
24+
connectString := config.DbConnectionString(cfg.Database)
25+
db, err := sqlx.Connect("postgres", connectString)
26+
if err != nil {
27+
log.Fatalf("Error connecting to DB: %v\n", err)
28+
}
29+
repository := repositories.NewPostgres(db)
30+
numberOfBlocksCreated := history.PopulateBlocks(blockchain, repository, int64(*startingBlockNumber))
31+
fmt.Printf("Populated %d blocks", numberOfBlocksCreated)
32+
}

‎integration_test/geth_blockchain_test.go

+12-1
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,12 @@ var _ = Describe("Reading from the Geth blockchain", func() {
1414

1515
var listener blockchain_listener.BlockchainListener
1616
var observer *fakes.BlockchainObserver
17+
var blockchain *geth.GethBlockchain
1718

1819
BeforeEach(func() {
1920
observer = fakes.NewFakeBlockchainObserver()
2021
cfg := config.NewConfig("private")
21-
blockchain := geth.NewGethBlockchain(cfg.Client.IPCPath)
22+
blockchain = geth.NewGethBlockchain(cfg.Client.IPCPath)
2223
observers := []core.BlockchainObserver{observer}
2324
listener = blockchain_listener.NewBlockchainListener(blockchain, observers)
2425
})
@@ -43,4 +44,14 @@ var _ = Describe("Reading from the Geth blockchain", func() {
4344
close(done)
4445
}, 10)
4546

47+
It("retrieves the genesis block and first block", func(done Done) {
48+
genesisBlock := blockchain.GetBlockByNumber(int64(0))
49+
firstBlock := blockchain.GetBlockByNumber(int64(1))
50+
51+
Expect(genesisBlock.Number).To(Equal(int64(0)))
52+
Expect(firstBlock.Number).To(Equal(int64(1)))
53+
54+
close(done)
55+
}, 10)
56+
4657
})

‎pkg/blockchain_listener/blockchain_listener_test.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ var _ = Describe("Blockchain listeners", func() {
1212

1313
It("starts with no blocks", func(done Done) {
1414
observer := fakes.NewFakeBlockchainObserver()
15-
blockchain := &fakes.Blockchain{}
15+
blockchain := fakes.NewBlockchain()
1616

1717
blockchain_listener.NewBlockchainListener(blockchain, []core.BlockchainObserver{observer})
1818

@@ -22,7 +22,7 @@ var _ = Describe("Blockchain listeners", func() {
2222

2323
It("sees when one block was added", func(done Done) {
2424
observer := fakes.NewFakeBlockchainObserver()
25-
blockchain := &fakes.Blockchain{}
25+
blockchain := fakes.NewBlockchain()
2626
listener := blockchain_listener.NewBlockchainListener(blockchain, []core.BlockchainObserver{observer})
2727
go listener.Start()
2828

@@ -38,7 +38,7 @@ var _ = Describe("Blockchain listeners", func() {
3838

3939
It("sees a second block", func(done Done) {
4040
observer := fakes.NewFakeBlockchainObserver()
41-
blockchain := &fakes.Blockchain{}
41+
blockchain := fakes.NewBlockchain()
4242
listener := blockchain_listener.NewBlockchainListener(blockchain, []core.BlockchainObserver{observer})
4343
go listener.Start()
4444

@@ -56,7 +56,7 @@ var _ = Describe("Blockchain listeners", func() {
5656

5757
It("stops listening", func(done Done) {
5858
observer := fakes.NewFakeBlockchainObserver()
59-
blockchain := &fakes.Blockchain{}
59+
blockchain := fakes.NewBlockchain()
6060
listener := blockchain_listener.NewBlockchainListener(blockchain, []core.BlockchainObserver{observer})
6161
go listener.Start()
6262

‎pkg/config/config.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@ func NewConfig(environment string) Config {
2323
filenameWithExtension := fmt.Sprintf("%s.toml", environment)
2424
absolutePath := filepath.Join(ProjectRoot(), "pkg", "config", "environments", filenameWithExtension)
2525
config := parseConfigFile(absolutePath)
26-
config.Client.IPCPath = filepath.Join(ProjectRoot(), config.Client.IPCPath)
26+
if !filepath.IsAbs(config.Client.IPCPath) {
27+
config.Client.IPCPath = filepath.Join(ProjectRoot(), config.Client.IPCPath)
28+
}
2729
return config
2830
}
2931

‎pkg/core/blockchain.go

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package core
22

33
type Blockchain interface {
4+
GetBlockByNumber(blockNumber int64) Block
45
SubscribeToBlocks(blocks chan Block)
56
StartListening()
67
StopListening()

‎pkg/fakes/blockchain.go

+24-4
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,36 @@ package fakes
33
import "github.com/8thlight/vulcanizedb/pkg/core"
44

55
type Blockchain struct {
6-
outputBlocks chan core.Block
6+
blocks map[int64]core.Block
7+
blocksChannel chan core.Block
78
WasToldToStop bool
89
}
910

11+
func NewBlockchain() *Blockchain {
12+
return &Blockchain{blocks: make(map[int64]core.Block)}
13+
}
14+
15+
func NewBlockchainWithBlocks(blocks []core.Block) *Blockchain {
16+
blockNumberToBlocks := make(map[int64]core.Block)
17+
for _, block := range blocks {
18+
blockNumberToBlocks[block.Number] = block
19+
}
20+
return &Blockchain{
21+
blocks: blockNumberToBlocks,
22+
}
23+
}
24+
25+
func (blockchain *Blockchain) GetBlockByNumber(blockNumber int64) core.Block {
26+
return blockchain.blocks[blockNumber]
27+
}
28+
1029
func (blockchain *Blockchain) SubscribeToBlocks(outputBlocks chan core.Block) {
11-
blockchain.outputBlocks = outputBlocks
30+
blockchain.blocksChannel = outputBlocks
1231
}
1332

14-
func (blockchain Blockchain) AddBlock(block core.Block) {
15-
blockchain.outputBlocks <- block
33+
func (blockchain *Blockchain) AddBlock(block core.Block) {
34+
blockchain.blocks[block.Number] = block
35+
blockchain.blocksChannel <- block
1636
}
1737

1838
func (*Blockchain) StartListening() {}

‎pkg/geth/geth_blockchain.go

+8-3
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package geth
33
import (
44
"fmt"
55

6+
"math/big"
7+
68
"github.com/8thlight/vulcanizedb/pkg/core"
79
"github.com/ethereum/go-ethereum"
810
"github.com/ethereum/go-ethereum/core/types"
@@ -17,6 +19,11 @@ type GethBlockchain struct {
1719
newHeadSubscription ethereum.Subscription
1820
}
1921

22+
func (blockchain *GethBlockchain) GetBlockByNumber(blockNumber int64) core.Block {
23+
gethBlock, _ := blockchain.client.BlockByNumber(context.Background(), big.NewInt(blockNumber))
24+
return GethBlockToCoreBlock(gethBlock)
25+
}
26+
2027
func NewGethBlockchain(ipcPath string) *GethBlockchain {
2128
fmt.Printf("Creating Geth Blockchain to: %s\n", ipcPath)
2229
blockchain := GethBlockchain{}
@@ -36,10 +43,8 @@ func (blockchain *GethBlockchain) SubscribeToBlocks(blocks chan core.Block) {
3643
}
3744

3845
func (blockchain *GethBlockchain) StartListening() {
39-
myContext := context.Background()
4046
for header := range blockchain.readGethHeaders {
41-
gethBlock, _ := blockchain.client.BlockByNumber(myContext, header.Number)
42-
block := GethBlockToCoreBlock(gethBlock)
47+
block := blockchain.GetBlockByNumber(header.Number.Int64())
4348
blockchain.outputBlocks <- block
4449
}
4550
}

‎pkg/history/history_suite_test.go

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package history_test
2+
3+
import (
4+
. "github.com/onsi/ginkgo"
5+
. "github.com/onsi/gomega"
6+
7+
"testing"
8+
)
9+
10+
func TestHistory(t *testing.T) {
11+
RegisterFailHandler(Fail)
12+
RunSpecs(t, "History Suite")
13+
}

‎pkg/history/populate_blocks.go

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package history
2+
3+
import (
4+
"github.com/8thlight/vulcanizedb/pkg/core"
5+
"github.com/8thlight/vulcanizedb/pkg/repositories"
6+
)
7+
8+
func PopulateBlocks(blockchain core.Blockchain, repository repositories.Repository, startingBlockNumber int64) int {
9+
blockNumbers := repository.MissingBlockNumbers(startingBlockNumber, repository.MaxBlockNumber())
10+
for _, blockNumber := range blockNumbers {
11+
block := blockchain.GetBlockByNumber(blockNumber)
12+
repository.CreateBlock(block)
13+
}
14+
return len(blockNumbers)
15+
}

‎pkg/history/populate_blocks_test.go

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
package history_test
2+
3+
import (
4+
"github.com/8thlight/vulcanizedb/pkg/core"
5+
"github.com/8thlight/vulcanizedb/pkg/fakes"
6+
"github.com/8thlight/vulcanizedb/pkg/history"
7+
"github.com/8thlight/vulcanizedb/pkg/repositories"
8+
. "github.com/onsi/ginkgo"
9+
. "github.com/onsi/gomega"
10+
)
11+
12+
var _ = Describe("Populating blocks", func() {
13+
14+
It("fills in the only missing block", func() {
15+
blocks := []core.Block{{Number: 1, Hash: "x012343"}}
16+
blockchain := fakes.NewBlockchainWithBlocks(blocks)
17+
repository := repositories.NewInMemory()
18+
repository.CreateBlock(core.Block{Number: 2})
19+
20+
history.PopulateBlocks(blockchain, repository, 1)
21+
22+
block := repository.FindBlockByNumber(1)
23+
Expect(block).NotTo(BeNil())
24+
Expect(block.Hash).To(Equal("x012343"))
25+
})
26+
27+
It("fills in two missing blocks", func() {
28+
blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
29+
{Number: 4},
30+
{Number: 5},
31+
{Number: 8},
32+
{Number: 10},
33+
{Number: 13},
34+
})
35+
repository := repositories.NewInMemory()
36+
repository.CreateBlock(core.Block{Number: 1})
37+
repository.CreateBlock(core.Block{Number: 2})
38+
repository.CreateBlock(core.Block{Number: 3})
39+
repository.CreateBlock(core.Block{Number: 6})
40+
repository.CreateBlock(core.Block{Number: 7})
41+
repository.CreateBlock(core.Block{Number: 9})
42+
repository.CreateBlock(core.Block{Number: 11})
43+
repository.CreateBlock(core.Block{Number: 12})
44+
45+
history.PopulateBlocks(blockchain, repository, 5)
46+
47+
Expect(repository.BlockCount()).To(Equal(11))
48+
Expect(repository.FindBlockByNumber(4)).To(BeNil())
49+
Expect(repository.FindBlockByNumber(5)).NotTo(BeNil())
50+
Expect(repository.FindBlockByNumber(8)).NotTo(BeNil())
51+
Expect(repository.FindBlockByNumber(10)).NotTo(BeNil())
52+
Expect(repository.FindBlockByNumber(13)).To(BeNil())
53+
})
54+
55+
It("returns the number of blocks created", func() {
56+
blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
57+
{Number: 4},
58+
{Number: 5},
59+
})
60+
repository := repositories.NewInMemory()
61+
repository.CreateBlock(core.Block{Number: 3})
62+
repository.CreateBlock(core.Block{Number: 6})
63+
64+
numberOfBlocksCreated := history.PopulateBlocks(blockchain, repository, 3)
65+
66+
Expect(numberOfBlocksCreated).To(Equal(2))
67+
})
68+
69+
})

‎pkg/repositories/in_memory.go

+20
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,16 @@ type InMemory struct {
88
blocks map[int64]*core.Block
99
}
1010

11+
func (repository *InMemory) MissingBlockNumbers(startingBlockNumber int64, endingBlockNumber int64) []int64 {
12+
missingNumbers := []int64{}
13+
for blockNumber := int64(startingBlockNumber); blockNumber <= endingBlockNumber; blockNumber++ {
14+
if repository.blocks[blockNumber] == nil {
15+
missingNumbers = append(missingNumbers, blockNumber)
16+
}
17+
}
18+
return missingNumbers
19+
}
20+
1121
func NewInMemory() *InMemory {
1222
return &InMemory{
1323
blocks: make(map[int64]*core.Block),
@@ -25,3 +35,13 @@ func (repository *InMemory) BlockCount() int {
2535
func (repository *InMemory) FindBlockByNumber(blockNumber int64) *core.Block {
2636
return repository.blocks[blockNumber]
2737
}
38+
39+
func (repository *InMemory) MaxBlockNumber() int64 {
40+
highestBlockNumber := int64(-1)
41+
for key := range repository.blocks {
42+
if key > highestBlockNumber {
43+
highestBlockNumber = key
44+
}
45+
}
46+
return highestBlockNumber
47+
}

0 commit comments

Comments
 (0)