Add a lib/cgolz4 package

This commit is contained in:
Nigel Tao committed 2019-09-15 13:38:40 +10:00
1 parent 8e0b5aab3b
commit 5e957c1161
3 files changed
+674

No files matched your search

+521
View File
@@ -0,0 +1,521 @@
// Copyright 2019 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------
// Package cgolz4 wraps the C "lz4" library.
//
// It speaks the LZ4 frame format, not the LZ4 block format.
package cgolz4
// TODO: dictionaries. See https://github.com/lz4/lz4/issues/791
/*
#cgo pkg-config: liblz4
#include "lz4frame.h"
#include <stdint.h>
typedef struct {
uint32_t ndst;
uint32_t nsrc;
uint32_t eof;
} advances;
LZ4F_cctx* cgolz4_compress_new() {
LZ4F_cctx* ret = NULL;
if (LZ4F_isError(LZ4F_createCompressionContext(&ret, LZ4F_VERSION))) {
return NULL;
}
return ret;
}
LZ4F_dctx* cgolz4_decompress_new() {
LZ4F_dctx* ret = NULL;
if (LZ4F_isError(LZ4F_createDecompressionContext(&ret, LZ4F_VERSION))) {
return NULL;
}
return ret;
}
uint64_t cgolz4_compress_min_dst_len(uint32_t srcSize) {
return LZ4F_compressBound(srcSize, NULL);
}
uint64_t cgolz4_compress_begin(LZ4F_cctx* z,
advances* a,
uint8_t* dst_ptr,
uint32_t dst_len) {
size_t result = LZ4F_compressBegin(z, dst_ptr, dst_len, NULL);
if (LZ4F_isError(result)) {
a->ndst = 0;
a->nsrc = 0;
a->eof = 0;
return result;
}
a->ndst = result;
a->nsrc = 0;
a->eof = 0;
return 0;
}
uint64_t cgolz4_compress_update(LZ4F_cctx* z,
advances* a,
uint8_t* dst_ptr,
uint32_t dst_len,
uint8_t* src_ptr,
uint32_t src_len) {
size_t result = LZ4F_compressUpdate(z, dst_ptr, dst_len, src_ptr, src_len, NULL);
if (LZ4F_isError(result)) {
a->ndst = 0;
a->nsrc = 0;
a->eof = 0;
return result;
}
a->ndst = result;
a->nsrc = src_len;
a->eof = 0;
return 0;
}
uint64_t cgolz4_compress_end(LZ4F_cctx* z,
advances* a,
uint8_t* dst_ptr,
uint32_t dst_len) {
size_t result = LZ4F_compressEnd(z, dst_ptr, dst_len, NULL);
if (LZ4F_isError(result)) {
a->ndst = 0;
a->nsrc = 0;
a->eof = 0;
return result;
}
a->ndst = result;
a->nsrc = 0;
a->eof = 1;
return 0;
}
uint64_t cgolz4_decompress_update(LZ4F_dctx* z,
advances* a,
uint8_t* dst_ptr,
uint32_t dst_len,
uint8_t* src_ptr,
uint32_t src_len) {
size_t d = dst_len;
size_t s = src_len;
size_t result = LZ4F_decompress(z, dst_ptr, &d, src_ptr, &s, NULL);
a->ndst = d;
a->nsrc = s;
a->eof = (result == 0) ? 1 : 0;
return LZ4F_isError(result) ? result : 0;
}
*/
import "C"
import (
"errors"
"io"
"unsafe"
"github.com/google/wuffs/lib/compression"
)
const cgoEnabled = true
// maxLen avoids overflow concerns when converting C and Go integer types.
const maxLen = 1 << 30
// blockMaxLen is the size that Writer.Write's []byte argument is sliced into.
// LZ4 is fundamentally a block-based API, and compressing source data of size
// N requires O(N) memory. For Writer to have a fixed size buffer, we impose a
// cap on the source data size.
const blockMaxLen = 65536
var (
errMissingResetCall = errors.New("cgolz4: missing Reset call")
errNilIOReader = errors.New("cgolz4: nil io.Reader")
errNilIOWriter = errors.New("cgolz4: nil io.Writer")
errNilReceiver = errors.New("cgolz4: nil receiver")
errOutOfMemoryVersionMismatch = errors.New("cgolz4: out of memory / version mismatch")
errWriteBufferIsTooSmall = errors.New("cgolz4: write buffer is too small")
)
type errCode uint64
func (e errCode) Error() string {
if s := C.GoString(C.LZ4F_getErrorName(C.LZ4F_errorCode_t(e))); s != "" {
return "cgolz4: " + s
}
return "cgolz4: unknown error"
}
// ReaderRecycler can lessen the new memory allocated when calling Reader.Reset
// on a bound Reader.
//
// It is not safe to use a ReaderRecycler and a Reader concurrently.
type ReaderRecycler struct {
z *C.LZ4F_dctx
closed bool
}
// Bind lets r re-use the memory that is manually managed by c. Call c.Close to
// free that memory.
func (c *ReaderRecycler) Bind(r *Reader) {
r.recycler = c
}
// Close implements io.Closer.
func (c *ReaderRecycler) Close() error {
c.closed = true
if c.z != nil {
C.LZ4F_freeDecompressionContext(c.z)
c.z = nil
}
return nil
}
// Reader decompresses from the lz4 format.
//
// The zero value is not usable until Reset is called.
type Reader struct {
buf [blockMaxLen]byte
i, j uint32
r io.Reader
readErr error
lz4Err error
recycler *ReaderRecycler
z *C.LZ4F_dctx
a C.advances
}
// Reset implements compression.Reader.
func (r *Reader) Reset(reader io.Reader, dictionary []byte) error {
if r == nil {
return errNilReceiver
}
if err := r.Close(); err != nil {
return err
}
if reader == nil {
return errNilIOReader
}
r.r = reader
return nil
}
// Close implements compression.Reader.
func (r *Reader) Close() error {
if r == nil {
return errNilReceiver
}
if r.r == nil {
return nil
}
r.i = 0
r.j = 0
r.r = nil
r.readErr = nil
r.lz4Err = nil
if r.z != nil {
if (r.recycler != nil) && !r.recycler.closed && (r.recycler.z == nil) {
r.recycler.z, r.z = r.z, nil
} else {
C.LZ4F_freeDecompressionContext(r.z)
r.z = nil
}
}
return nil
}
// Read implements compression.Reader.
func (r *Reader) Read(p []byte) (int, error) {
if r == nil {
return 0, errNilReceiver
}
if r.r == nil {
return 0, errMissingResetCall
}
if r.z == nil {
if (r.recycler != nil) && !r.recycler.closed && (r.recycler.z != nil) {
r.z, r.recycler.z = r.recycler.z, nil
C.LZ4F_resetDecompressionContext(r.z)
} else {
r.z = C.cgolz4_decompress_new()
if r.z == nil {
return 0, errOutOfMemoryVersionMismatch
}
}
}
if len(p) > maxLen {
p = p[:maxLen]
}
for numRead := 0; ; {
if r.lz4Err != nil {
return numRead, r.lz4Err
}
if len(p) == 0 {
return numRead, nil
}
if r.i >= r.j {
if r.readErr != nil {
return numRead, r.readErr
}
n, err := r.r.Read(r.buf[:])
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
r.i, r.j, r.readErr = 0, uint32(n), err
continue
}
e := errCode(C.cgolz4_decompress_update(r.z, &r.a,
(*C.uint8_t)(unsafe.Pointer(&p[0])),
(C.uint32_t)(len(p)),
(*C.uint8_t)(unsafe.Pointer(&r.buf[r.i])),
(C.uint32_t)(r.j-r.i),
))
numRead += int(r.a.ndst)
p = p[int(r.a.ndst):]
r.i += uint32(r.a.nsrc)
if e == 0 {
if r.a.eof == 0 {
continue
}
r.lz4Err = io.EOF
} else {
r.lz4Err = e
}
return numRead, r.lz4Err
}
}
// WriterRecycler can lessen the new memory allocated when calling Writer.Reset
// on a bound Writer.
//
// It is not safe to use a WriterRecycler and a Writer concurrently.
type WriterRecycler struct {
z *C.LZ4F_cctx
closed bool
}
// Bind lets w re-use the memory that is manually managed by c. Call c.Close to
// free that memory.
func (c *WriterRecycler) Bind(w *Writer) {
w.recycler = c
}
// Close implements io.Closer.
func (c *WriterRecycler) Close() error {
c.closed = true
if c.z != nil {
C.LZ4F_freeCompressionContext(c.z)
c.z = nil
}
return nil
}
// Writer compresses to the lz4 format.
//
// Compressed bytes may be buffered and not sent to the underlying io.Writer
// until Close is called.
//
// The zero value is not usable until Reset is called.
type Writer struct {
buf [2 * blockMaxLen]byte
j uint32
w io.Writer
begun bool
writeErr error
recycler *WriterRecycler
z *C.LZ4F_cctx
a C.advances
}
// Reset implements compression.Writer.
func (w *Writer) Reset(writer io.Writer, dictionary []byte, level compression.Level) error {
if w == nil {
return errNilReceiver
}
w.close()
if writer == nil {
return errNilIOWriter
}
w.w = writer
w.begun = false
return nil
}
// Close implements compression.Writer.
func (w *Writer) Close() error {
if w == nil {
return errNilReceiver
}
err := w.flush(true)
w.close()
return err
}
func (w *Writer) flush(final bool) error {
if w.w == nil {
return nil
}
if final {
if err := w.write(nil, true); err != nil {
return err
}
}
if w.j == 0 {
return nil
}
_, err := w.w.Write(w.buf[:w.j])
w.j = 0
return err
}
func (w *Writer) close() {
if w.w == nil {
return
}
w.j = 0
w.w = nil
w.writeErr = nil
if w.z != nil {
if (w.recycler != nil) && !w.recycler.closed && (w.recycler.z == nil) {
w.recycler.z, w.z = w.z, nil
} else {
C.LZ4F_freeCompressionContext(w.z)
w.z = nil
}
}
}
// Write implements compression.Writer.
func (w *Writer) Write(p []byte) (int, error) {
if w == nil {
return 0, errNilReceiver
}
if w.w == nil {
return 0, errMissingResetCall
}
if w.writeErr != nil {
return 0, w.writeErr
}
originalLenP := len(p)
for {
remaining := []byte(nil)
if blockMaxLen > maxLen {
panic("unreachable")
} else if len(p) > blockMaxLen {
p, remaining = p[:blockMaxLen], p[blockMaxLen:]
}
if err := w.write(p, false); err != nil {
return 0, err
}
p, remaining = remaining, nil
if len(p) == 0 {
return originalLenP, nil
}
}
}
func (w *Writer) write(p []byte, final bool) error {
if n := len(p); (n > blockMaxLen) || (n > maxLen) {
panic("unreachable")
}
if w.z == nil {
if (w.recycler != nil) && !w.recycler.closed && (w.recycler.z != nil) {
w.z, w.recycler.z = w.recycler.z, nil
} else {
w.z = C.cgolz4_compress_new()
if w.z == nil {
return errOutOfMemoryVersionMismatch
}
}
}
if !w.begun {
w.begun = true
e := errCode(C.cgolz4_compress_begin(w.z, &w.a,
(*C.uint8_t)(unsafe.Pointer(&w.buf[w.j])),
(C.uint32_t)(uint32(len(w.buf))-w.j),
))
w.j += uint32(w.a.ndst)
p = p[uint32(w.a.nsrc):]
if e != 0 {
w.writeErr = e
return w.writeErr
}
}
for (len(p) > 0) || final {
minDstLen := uint64(C.cgolz4_compress_min_dst_len(C.uint32_t(len(p))))
for (uint64(len(w.buf)) - uint64(w.j)) < minDstLen {
if w.j == 0 {
w.writeErr = errWriteBufferIsTooSmall
return w.writeErr
}
if err := w.flush(false); err != nil {
w.writeErr = err
return w.writeErr
}
}
e := errCode(0)
if final {
e = errCode(C.cgolz4_compress_end(w.z, &w.a,
(*C.uint8_t)(unsafe.Pointer(&w.buf[w.j])),
(C.uint32_t)(uint32(len(w.buf))-w.j),
))
final = w.a.eof == 0
} else {
e = errCode(C.cgolz4_compress_update(w.z, &w.a,
(*C.uint8_t)(unsafe.Pointer(&w.buf[w.j])),
(C.uint32_t)(uint32(len(w.buf))-w.j),
(*C.uint8_t)(unsafe.Pointer(&p[0])),
(C.uint32_t)(len(p)),
))
}
w.j += uint32(w.a.ndst)
p = p[uint32(w.a.nsrc):]
if e != 0 {
w.writeErr = e
return w.writeErr
}
}
return nil
}
+91
View File
@@ -0,0 +1,91 @@
// Copyright 2019 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cgolz4
import (
"bytes"
"io/ioutil"
"strings"
"testing"
)
const (
// compressedMore is 21 bytes of lz4 frame:
// \x04\x22\x4d\x18 \x40 \x40 \xc0 \x06\x00\x00\x80 \x4d\x6f\x72\x65\x21\x0a \x00\x00\x00\x00
// Magic----------- FLG- BD-- HC- BlockSize------- BlockData--------------- EndMark---------
//
// FLG: Version=1.
// BD: BlockMaxSize=64K.
// HC: checksum.
// Block Size: 6 (uncompressed).
// BlockData: the literal bytes "More!\n".
compressedMore = "\x04\x22\x4d\x18\x40\x40\xc0\x06\x00\x00\x80\x4d\x6f\x72\x65\x21\x0a\x00\x00\x00\x00"
uncompressedMore = "More!\n"
)
func TestRoundTrip(t *testing.T) {
if !cgoEnabled {
t.Skip("cgo is not enabled")
}
wr := &WriterRecycler{}
defer wr.Close()
w := &Writer{}
wr.Bind(w)
rr := &ReaderRecycler{}
defer rr.Close()
r := &Reader{}
rr.Bind(r)
for i := 0; i < 3; i++ {
buf := &bytes.Buffer{}
// Compress.
{
w.Reset(buf, nil, 0)
if _, err := w.Write([]byte(uncompressedMore)); err != nil {
w.Close()
t.Fatalf("i=%d: Write: %v", i, err)
}
if err := w.Close(); err != nil {
t.Fatalf("i=%d: Close: %v", i, err)
}
}
compressed := buf.String()
if compressed != compressedMore {
t.Fatalf("i=%d: compressed\ngot % 02x\nwant % 02x", i, compressed, compressedMore)
}
// Uncompress.
{
r.Reset(strings.NewReader(compressed), nil)
gotBytes, err := ioutil.ReadAll(r)
if err != nil {
r.Close()
t.Fatalf("i=%d: ReadAll: %v", i, err)
}
if got, want := string(gotBytes), uncompressedMore; got != want {
r.Close()
t.Fatalf("i=%d:\ngot %q\nwant %q", i, got, want)
}
if err := r.Close(); err != nil {
t.Fatalf("i=%d: Close: %v", i, err)
}
}
}
}
+62
View File
@@ -0,0 +1,62 @@
// Copyright 2019 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------
// +build !cgo
package cgolz4
// This file contains placeholder types and funcs so that the package still
// builds (with the same API) when CGO_ENABLED=0. The package doesn't work
// without cgo, but it will fail at run time, not compile time.
//
// In particular, the build stays green regardless of whether CGO_ENABLED is on
// or off. Installing and testing every package in the whole repository will
// not fail. The tests in this package don't pass, but they are skipped.
import (
"errors"
"io"
"github.com/google/wuffs/lib/compression"
)
const cgoEnabled = false
var (
errCgoIsNotEnabled = errors.New("cgolz4: cgo is not enabled")
)
type ReaderRecycler struct{}
func (c *ReaderRecycler) Bind(*Reader) {}
func (c *ReaderRecycler) Close() error { return errCgoIsNotEnabled }
type Reader struct{}
func (r *Reader) Close() error { return errCgoIsNotEnabled }
func (r *Reader) Read([]byte) (int, error) { return 0, errCgoIsNotEnabled }
func (r *Reader) Reset(io.Reader, []byte) error { return errCgoIsNotEnabled }
type WriterRecycler struct{}
func (c *WriterRecycler) Bind(*Writer) {}
func (c *WriterRecycler) Close() error { return errCgoIsNotEnabled }
type Writer struct{}
func (w *Writer) Close() error { return errCgoIsNotEnabled }
func (w *Writer) Reset(io.Writer, []byte, compression.Level) error { return errCgoIsNotEnabled }
func (w *Writer) Write([]byte) (int, error) { return 0, errCgoIsNotEnabled }