Skip to content

Commit c06ba5f

Browse files
authored
s2: Use sorted search for index (#555)
``` cpu: AMD Ryzen 9 3950X 16-Core Processor BenchmarkIndexFind BenchmarkIndexFind/blocks-1-32 194769370 6.180 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-2-32 199258790 6.057 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-4-32 204297220 5.879 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-8-32 190476280 6.321 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-16-32 158611744 7.537 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-32-32 100000000 10.04 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-64-32 79986668 14.90 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-128-32 74996250 14.95 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-256-32 19506928 60.93 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-512-32 8450894 142.1 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-1024-32 3858536 307.3 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-2048-32 2556153 470.4 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-4096-32 840476 1457 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-8192-32 571398 2091 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-16384-32 195525 6022 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-32768-32 138596 8665 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-65535-32 86328 13993 ns/op 0 B/op 0 allocs/op ``` after: ``` cpu: AMD Ryzen 9 3950X 16-Core Processor BenchmarkIndexFind BenchmarkIndexFind/blocks-1-32 195354979 6.054 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-2-32 193981434 6.140 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-4-32 201894782 6.023 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-8-32 180996812 6.978 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-16-32 148147708 8.016 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-32-32 100000000 10.43 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-64-32 75000000 15.95 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-128-32 75000000 15.59 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-256-32 49979174 22.83 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-512-32 46302012 25.00 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-1024-32 44444937 27.43 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-2048-32 38709552 29.62 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-4096-32 37005741 31.86 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-8192-32 34297766 34.43 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-16384-32 30763077 36.89 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-32768-32 29656209 39.45 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-65535-32 27980180 42.53 ns/op 0 B/op 0 allocs/op ``` Fixes #554
1 parent 458f435 commit c06ba5f

File tree

2 files changed

+47
-0
lines changed

2 files changed

+47
-0
lines changed

s2/encode_test.go

+37
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,43 @@ func TestIndex(t *testing.T) {
351351
}
352352
}
353353

354+
func BenchmarkIndexFind(b *testing.B) {
355+
fatalErr := func(t testing.TB, err error) {
356+
if err != nil {
357+
t.Fatal(err)
358+
}
359+
}
360+
for blocks := 1; blocks <= 65536; blocks *= 2 {
361+
if blocks == 65536 {
362+
blocks = 65535
363+
}
364+
365+
var index Index
366+
index.reset(100)
367+
index.TotalUncompressed = int64(blocks) * 100
368+
index.TotalCompressed = int64(blocks) * 100
369+
for i := 0; i < blocks; i++ {
370+
err := index.add(int64(i*100), int64(i*100))
371+
fatalErr(b, err)
372+
}
373+
374+
rng := rand.New(rand.NewSource(0xabeefcafe))
375+
b.Run(fmt.Sprintf("blocks-%d", len(index.info)), func(b *testing.B) {
376+
b.ResetTimer()
377+
b.ReportAllocs()
378+
const prime4bytes = 2654435761
379+
rng2 := rng.Int63()
380+
for i := 0; i < b.N; i++ {
381+
rng2 = ((rng2 + prime4bytes) * prime4bytes) >> 32
382+
// Find offset:
383+
_, _, err := index.Find(rng2 % (int64(blocks) * 100))
384+
fatalErr(b, err)
385+
}
386+
})
387+
}
388+
389+
}
390+
354391
func TestWriterPadding(t *testing.T) {
355392
n := 100
356393
if testing.Short() {

s2/index.go

+10
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"encoding/json"
1111
"fmt"
1212
"io"
13+
"sort"
1314
)
1415

1516
const (
@@ -100,6 +101,15 @@ func (i *Index) Find(offset int64) (compressedOff, uncompressedOff int64, err er
100101
if offset > i.TotalUncompressed {
101102
return 0, 0, io.ErrUnexpectedEOF
102103
}
104+
if len(i.info) > 200 {
105+
n := sort.Search(len(i.info), func(n int) bool {
106+
return i.info[n].uncompressedOffset > offset
107+
})
108+
if n == 0 {
109+
n = 1
110+
}
111+
return i.info[n-1].compressedOffset, i.info[n-1].uncompressedOffset, nil
112+
}
103113
for _, info := range i.info {
104114
if info.uncompressedOffset > offset {
105115
break

0 commit comments

Comments
 (0)