Skip to content

Commit 619c68b

Browse files
dsnetdsnet
committed
internal/cgo: add wrappers for flate, bzip2, and brotli
Re-implement the cgo wrappers for flate, bzip2, and brotli to properly follow the C-to-Go pointer rules outlined in (golang/go#12416). Also, move the wrappers to seperate packages and update the bench tool to use the new wrappers.
1 parent b83065f commit 619c68b

17 files changed

+846
-524
lines changed

‎internal/cgo/brotli/brotli.go

+211
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
// Copyright 2016, Joe Tsai. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE.md file.
4+
5+
// +build cgo
6+
7+
// Package brotli implements the Brotli compressed data format using C wrappers.
8+
package brotli
9+
10+
/*
11+
// This relies upon the shared library built from github.com/google/brotli
12+
// at revision 7e347a7c849db05acad20304f5e9b29071ecec7c.
13+
//
14+
// The steps to build and install the shared library is as follows:
15+
// mkdir out && cd out && ../configure-cmake && make
16+
// make test
17+
// make install
18+
19+
#cgo LDFLAGS: -lbrotlidec
20+
#cgo LDFLAGS: -lbrotlienc
21+
22+
#include "brotli/decode.h"
23+
#include "brotli/encode.h"
24+
25+
BrotliDecoderState* zbDecCreate() {
26+
return BrotliDecoderCreateInstance(NULL, NULL, NULL);
27+
}
28+
29+
BrotliDecoderResult zbDecStream(
30+
BrotliDecoderState* state,
31+
size_t* avail_in, const uint8_t* next_in,
32+
size_t* avail_out, uint8_t* next_out
33+
) {
34+
return BrotliDecoderDecompressStream(
35+
state, avail_in, &next_in, avail_out, &next_out, NULL
36+
);
37+
}
38+
39+
void zbDecDestroy(BrotliDecoderState* state) {
40+
return BrotliDecoderDestroyInstance(state);
41+
}
42+
43+
BrotliEncoderState* zbEncCreate(int level) {
44+
BrotliEncoderState* state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
45+
if (state != NULL) {
46+
BrotliEncoderSetParameter(state, BROTLI_PARAM_QUALITY, level);
47+
}
48+
return state;
49+
}
50+
51+
BROTLI_BOOL zbEncStream(
52+
BrotliEncoderState* state, BrotliEncoderOperation op,
53+
size_t* avail_in, const uint8_t* next_in,
54+
size_t* avail_out, uint8_t* next_out
55+
) {
56+
return BrotliEncoderCompressStream(
57+
state, op, avail_in, &next_in, avail_out, &next_out, NULL
58+
);
59+
}
60+
61+
void zbEncDestroy(BrotliEncoderState* state) {
62+
return BrotliEncoderDestroyInstance(state);
63+
}
64+
*/
65+
import "C"
66+
67+
import (
68+
"errors"
69+
"io"
70+
"unsafe"
71+
)
72+
73+
type Reader struct {
74+
r io.Reader
75+
err error
76+
state *C.BrotliDecoderState
77+
buf []byte
78+
arr [1 << 14]byte
79+
}
80+
81+
func NewReader(r io.Reader) io.ReadCloser {
82+
zr := &Reader{r: r, state: C.zbDecCreate()}
83+
if zr.state == nil {
84+
panic("brotli: could not allocate decoder state")
85+
}
86+
return zr
87+
}
88+
89+
func (zr *Reader) Read(buf []byte) (int, error) {
90+
if zr.state == nil {
91+
return 0, io.ErrClosedPipe
92+
}
93+
94+
var n int
95+
for zr.err == nil && (len(buf) > 0 && n == 0) {
96+
availIn, availOut, ptrIn, ptrOut := sizePtrs(zr.buf, buf)
97+
ret := C.zbDecStream(zr.state, &availIn, ptrIn, &availOut, ptrOut)
98+
n += len(buf) - int(availOut)
99+
buf = buf[len(buf)-int(availOut):]
100+
zr.buf = zr.buf[len(zr.buf)-int(availIn):]
101+
102+
switch ret {
103+
case C.BROTLI_DECODER_RESULT_ERROR:
104+
zr.err = errors.New("brotli: corrupted input")
105+
case C.BROTLI_DECODER_RESULT_SUCCESS:
106+
return n, io.EOF
107+
case C.BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
108+
n1 := copy(zr.arr[:], zr.buf)
109+
n2, err := zr.r.Read(zr.arr[n1:])
110+
if n2 > 0 {
111+
zr.buf = zr.arr[:n1+n2]
112+
} else if err != nil {
113+
if err == io.EOF {
114+
err = io.ErrUnexpectedEOF
115+
}
116+
zr.err = err
117+
}
118+
case C.BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
119+
return n, nil
120+
default:
121+
zr.err = errors.New("brotli: unknown decoder error")
122+
}
123+
}
124+
return n, zr.err
125+
}
126+
127+
func (zr *Reader) Close() error {
128+
if zr.state != nil {
129+
defer func() {
130+
C.zbDecDestroy(zr.state)
131+
zr.state = nil
132+
}()
133+
}
134+
return zr.err
135+
}
136+
137+
type Writer struct {
138+
w io.Writer
139+
err error
140+
state *C.BrotliEncoderState
141+
buf []byte
142+
arr [1 << 14]byte
143+
}
144+
145+
func NewWriter(w io.Writer, level int) io.WriteCloser {
146+
if level < C.BROTLI_MIN_QUALITY || level > C.BROTLI_MAX_QUALITY {
147+
panic("brotli: invalid compression level")
148+
}
149+
150+
zw := &Writer{w: w, state: C.zbEncCreate(C.int(level))}
151+
if zw.state == nil {
152+
panic("brotli: could not allocate encoder state")
153+
}
154+
return zw
155+
}
156+
157+
func (zw *Writer) Write(buf []byte) (int, error) {
158+
return zw.write(buf, C.BROTLI_OPERATION_PROCESS)
159+
}
160+
161+
func (zw *Writer) write(buf []byte, op C.BrotliEncoderOperation) (int, error) {
162+
if zw.state == nil {
163+
return 0, io.ErrClosedPipe
164+
}
165+
166+
var n int
167+
flush := op != C.BROTLI_OPERATION_PROCESS
168+
for zw.err == nil && (len(buf) > 0 || flush) {
169+
availIn, availOut, ptrIn, ptrOut := sizePtrs(buf, zw.arr[:])
170+
ret := C.zbEncStream(zw.state, op, &availIn, ptrIn, &availOut, ptrOut)
171+
n += len(buf) - int(availIn)
172+
buf = buf[len(buf)-int(availIn):]
173+
zw.buf = zw.arr[:len(zw.arr)-int(availOut)]
174+
175+
if len(zw.buf) > 0 {
176+
if _, err := zw.w.Write(zw.buf); err != nil {
177+
zw.err = err
178+
}
179+
}
180+
if ret == 0 && zw.err == nil {
181+
zw.err = errors.New("brotli: compression error")
182+
}
183+
if flush && C.BrotliEncoderHasMoreOutput(zw.state) == 0 {
184+
break
185+
}
186+
}
187+
return n, zw.err
188+
}
189+
190+
func (zw *Writer) Close() error {
191+
if zw.state != nil {
192+
defer func() {
193+
C.zbEncDestroy(zw.state)
194+
zw.state = nil
195+
}()
196+
zw.write(nil, C.BROTLI_OPERATION_FINISH)
197+
}
198+
return zw.err
199+
}
200+
201+
func sizePtrs(in, out []byte) (sizeIn, sizeOut C.size_t, ptrIn, ptrOut *C.uint8_t) {
202+
sizeIn = C.size_t(len(in))
203+
sizeOut = C.size_t(len(out))
204+
if len(in) > 0 {
205+
ptrIn = (*C.uint8_t)(unsafe.Pointer(&in[0]))
206+
}
207+
if len(out) > 0 {
208+
ptrOut = (*C.uint8_t)(unsafe.Pointer(&out[0]))
209+
}
210+
return
211+
}

‎internal/cgo/brotli/brotli_test.go

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
// Copyright 2016, Joe Tsai. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE.md file.
4+
5+
// +build cgo
6+
7+
package brotli
8+
9+
import (
10+
"bytes"
11+
"hash/crc32"
12+
"io"
13+
"path/filepath"
14+
"testing"
15+
16+
"github.com/dsnet/compress/internal/testutil"
17+
)
18+
19+
func TestRoundTrip(t *testing.T) {
20+
files := []string{
21+
"binary.bin", "digits.txt", "huffman.txt", "random.bin", "repeats.bin", "twain.txt", "zeros.bin",
22+
}
23+
for _, f := range files {
24+
input := testutil.MustLoadFile(filepath.Join("../../../testdata/", f))
25+
26+
buf := new(bytes.Buffer)
27+
zw := NewWriter(buf, 6)
28+
if _, err := io.Copy(zw, bytes.NewReader(input)); err != nil {
29+
t.Errorf("test %s, unexpected Copy error: %v", f, err)
30+
continue
31+
}
32+
if err := zw.Close(); err != nil {
33+
t.Errorf("test %s, unexpected Close error: %v", f, err)
34+
continue
35+
}
36+
37+
hash := crc32.NewIEEE()
38+
zr := NewReader(buf)
39+
if _, err := io.Copy(hash, zr); err != nil {
40+
t.Errorf("test %s, unexpected Copy error: %v", f, err)
41+
continue
42+
}
43+
if err := zr.Close(); err != nil {
44+
t.Errorf("test %s, unexpected Close error: %v", f, err)
45+
continue
46+
}
47+
48+
if got, want := hash.Sum32(), crc32.ChecksumIEEE(input); got != want {
49+
t.Errorf("test %s, mismatching checksum: got 0x%08x, want 0x%08x", f, got, want)
50+
}
51+
}
52+
}

0 commit comments

Comments
 (0)