mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-17 16:32:11 +01:00
Add a lib/cgozstd package
This commit is contained in:
1 parent
99197b3f09
commit
3adcfc4d99
3 files changed
+627
No files matched your search
@@ -0,0 +1,475 @@
|
||||
// Copyright 2019 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// ----------------
|
||||
|
||||
// Package cgozstd wraps the C "zstd" library.
|
||||
package cgozstd
|
||||
|
||||
// TODO: dictionaries. See https://github.com/facebook/zstd/issues/1776
|
||||
|
||||
/*
|
||||
#cgo pkg-config: libzstd
|
||||
#include "zstd.h"
|
||||
#include "zstd_errors.h"
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
typedef struct {
|
||||
uint32_t ndst;
|
||||
uint32_t nsrc;
|
||||
uint32_t eof;
|
||||
} advances;
|
||||
|
||||
int32_t cgozstd_compress(ZSTD_CCtx* z,
|
||||
advances* a,
|
||||
uint8_t* dst_ptr,
|
||||
uint32_t dst_len,
|
||||
uint8_t* src_ptr,
|
||||
uint32_t src_len) {
|
||||
ZSTD_outBuffer ob;
|
||||
ob.dst = dst_ptr;
|
||||
ob.size = dst_len;
|
||||
ob.pos = 0;
|
||||
|
||||
ZSTD_inBuffer ib;
|
||||
ib.src = src_ptr;
|
||||
ib.size = src_len;
|
||||
ib.pos = 0;
|
||||
|
||||
size_t result = ZSTD_compressStream(z, &ob, &ib);
|
||||
|
||||
a->ndst = ob.pos;
|
||||
a->nsrc = ib.pos;
|
||||
a->eof = 0;
|
||||
|
||||
return ZSTD_getErrorCode(result);
|
||||
}
|
||||
|
||||
int32_t cgozstd_compress_end(ZSTD_CCtx* z,
|
||||
advances* a,
|
||||
uint8_t* dst_ptr,
|
||||
uint32_t dst_len) {
|
||||
ZSTD_outBuffer ob;
|
||||
ob.dst = dst_ptr;
|
||||
ob.size = dst_len;
|
||||
ob.pos = 0;
|
||||
|
||||
size_t result = ZSTD_endStream(z, &ob);
|
||||
|
||||
a->ndst = ob.pos;
|
||||
a->nsrc = 0;
|
||||
a->eof = (result == 0) ? 1 : 0;
|
||||
|
||||
return ZSTD_getErrorCode(result);
|
||||
}
|
||||
|
||||
int32_t cgozstd_decompress(ZSTD_DCtx* z,
|
||||
advances* a,
|
||||
uint8_t* dst_ptr,
|
||||
uint32_t dst_len,
|
||||
uint8_t* src_ptr,
|
||||
uint32_t src_len) {
|
||||
ZSTD_outBuffer ob;
|
||||
ob.dst = dst_ptr;
|
||||
ob.size = dst_len;
|
||||
ob.pos = 0;
|
||||
|
||||
ZSTD_inBuffer ib;
|
||||
ib.src = src_ptr;
|
||||
ib.size = src_len;
|
||||
ib.pos = 0;
|
||||
|
||||
size_t result = ZSTD_decompressStream(z, &ob, &ib);
|
||||
|
||||
a->ndst = ob.pos;
|
||||
a->nsrc = ib.pos;
|
||||
a->eof = (result == 0) ? 1 : 0;
|
||||
|
||||
return ZSTD_getErrorCode(result);
|
||||
}
|
||||
*/
|
||||
import "C"
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"unsafe"
|
||||
|
||||
"github.com/google/wuffs/lib/compression"
|
||||
)
|
||||
|
||||
const cgoEnabled = true
|
||||
|
||||
// maxLen avoids overflow concerns when converting C and Go integer types.
|
||||
const maxLen = 1 << 30
|
||||
|
||||
var (
|
||||
errMissingResetCall = errors.New("cgozstd: missing Reset call")
|
||||
errNilIOReader = errors.New("cgozstd: nil io.Reader")
|
||||
errNilIOWriter = errors.New("cgozstd: nil io.Writer")
|
||||
errNilReceiver = errors.New("cgozstd: nil receiver")
|
||||
errOutOfMemory = errors.New("cgozstd: out of memory")
|
||||
)
|
||||
|
||||
type errCode int32
|
||||
|
||||
func (e errCode) Error() string {
|
||||
if s := C.GoString(C.ZSTD_getErrorString(C.ZSTD_ErrorCode(e))); s != "" {
|
||||
return "cgozstd: " + s
|
||||
}
|
||||
return "cgozstd: unknown error"
|
||||
}
|
||||
|
||||
// ReaderRecycler can lessen the new memory allocated when calling Reader.Reset
|
||||
// on a bound Reader.
|
||||
//
|
||||
// It is not safe to use a ReaderRecycler and a Reader concurrently.
|
||||
type ReaderRecycler struct {
|
||||
z *C.ZSTD_DCtx
|
||||
closed bool
|
||||
}
|
||||
|
||||
// Bind lets r re-use the memory that is manually managed by c. Call c.Close to
|
||||
// free that memory.
|
||||
func (c *ReaderRecycler) Bind(r *Reader) {
|
||||
r.recycler = c
|
||||
}
|
||||
|
||||
// Close implements io.Closer.
|
||||
func (c *ReaderRecycler) Close() error {
|
||||
c.closed = true
|
||||
if c.z != nil {
|
||||
C.ZSTD_freeDCtx(c.z)
|
||||
c.z = nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reader decompresses from the zstd format.
|
||||
//
|
||||
// The zero value is not usable until Reset is called.
|
||||
type Reader struct {
|
||||
buf [4096]byte
|
||||
i, j uint32
|
||||
r io.Reader
|
||||
|
||||
readErr error
|
||||
zstdErr error
|
||||
|
||||
recycler *ReaderRecycler
|
||||
|
||||
z *C.ZSTD_DCtx
|
||||
a C.advances
|
||||
}
|
||||
|
||||
// Reset implements compression.Reader.
|
||||
func (r *Reader) Reset(reader io.Reader, dictionary []byte) error {
|
||||
if r == nil {
|
||||
return errNilReceiver
|
||||
}
|
||||
if err := r.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if reader == nil {
|
||||
return errNilIOReader
|
||||
}
|
||||
r.r = reader
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close implements compression.Reader.
|
||||
func (r *Reader) Close() error {
|
||||
if r == nil {
|
||||
return errNilReceiver
|
||||
}
|
||||
if r.r == nil {
|
||||
return nil
|
||||
}
|
||||
r.i = 0
|
||||
r.j = 0
|
||||
r.r = nil
|
||||
r.readErr = nil
|
||||
r.zstdErr = nil
|
||||
if r.z != nil {
|
||||
if (r.recycler != nil) && !r.recycler.closed && (r.recycler.z == nil) {
|
||||
r.recycler.z, r.z = r.z, nil
|
||||
} else {
|
||||
C.ZSTD_freeDCtx(r.z)
|
||||
r.z = nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read implements compression.Reader.
|
||||
func (r *Reader) Read(p []byte) (int, error) {
|
||||
if r == nil {
|
||||
return 0, errNilReceiver
|
||||
}
|
||||
if r.r == nil {
|
||||
return 0, errMissingResetCall
|
||||
}
|
||||
|
||||
if r.z == nil {
|
||||
if (r.recycler != nil) && !r.recycler.closed && (r.recycler.z != nil) {
|
||||
r.z, r.recycler.z = r.recycler.z, nil
|
||||
C.ZSTD_initDStream(r.z)
|
||||
} else {
|
||||
r.z = C.ZSTD_createDStream()
|
||||
if r.z == nil {
|
||||
return 0, errOutOfMemory
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(p) > maxLen {
|
||||
p = p[:maxLen]
|
||||
}
|
||||
|
||||
for numRead := 0; ; {
|
||||
if r.zstdErr != nil {
|
||||
return numRead, r.zstdErr
|
||||
}
|
||||
if len(p) == 0 {
|
||||
return numRead, nil
|
||||
}
|
||||
|
||||
if r.i >= r.j {
|
||||
if r.readErr != nil {
|
||||
return numRead, r.readErr
|
||||
}
|
||||
|
||||
n, err := r.r.Read(r.buf[:])
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
r.i, r.j, r.readErr = 0, uint32(n), err
|
||||
continue
|
||||
}
|
||||
|
||||
e := C.cgozstd_decompress(r.z, &r.a,
|
||||
(*C.uint8_t)(unsafe.Pointer(&p[0])),
|
||||
(C.uint32_t)(len(p)),
|
||||
(*C.uint8_t)(unsafe.Pointer(&r.buf[r.i])),
|
||||
(C.uint32_t)(r.j-r.i),
|
||||
)
|
||||
|
||||
numRead += int(r.a.ndst)
|
||||
p = p[int(r.a.ndst):]
|
||||
|
||||
r.i += uint32(r.a.nsrc)
|
||||
|
||||
if e == 0 {
|
||||
if r.a.eof == 0 {
|
||||
continue
|
||||
}
|
||||
r.zstdErr = io.EOF
|
||||
} else {
|
||||
r.zstdErr = errCode(e)
|
||||
}
|
||||
return numRead, r.zstdErr
|
||||
}
|
||||
}
|
||||
|
||||
// WriterRecycler can lessen the new memory allocated when calling Writer.Reset
|
||||
// on a bound Writer.
|
||||
//
|
||||
// It is not safe to use a WriterRecycler and a Writer concurrently.
|
||||
type WriterRecycler struct {
|
||||
z *C.ZSTD_CCtx
|
||||
closed bool
|
||||
}
|
||||
|
||||
// Bind lets w re-use the memory that is manually managed by c. Call c.Close to
|
||||
// free that memory.
|
||||
func (c *WriterRecycler) Bind(w *Writer) {
|
||||
w.recycler = c
|
||||
}
|
||||
|
||||
// Close implements io.Closer.
|
||||
func (c *WriterRecycler) Close() error {
|
||||
c.closed = true
|
||||
if c.z != nil {
|
||||
C.ZSTD_freeCCtx(c.z)
|
||||
c.z = nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Writer compresses to the zstd format.
|
||||
//
|
||||
// Compressed bytes may be buffered and not sent to the underlying io.Writer
|
||||
// until Close is called.
|
||||
//
|
||||
// The zero value is not usable until Reset is called.
|
||||
type Writer struct {
|
||||
buf [4096]byte
|
||||
j uint32
|
||||
w io.Writer
|
||||
level compression.Level
|
||||
|
||||
writeErr error
|
||||
|
||||
recycler *WriterRecycler
|
||||
|
||||
z *C.ZSTD_CCtx
|
||||
a C.advances
|
||||
}
|
||||
|
||||
func (w *Writer) zstdCompressionLevel() int32 {
|
||||
return w.level.Interpolate(1, 2, 3, 15, 22)
|
||||
}
|
||||
|
||||
// Reset implements compression.Writer.
|
||||
func (w *Writer) Reset(writer io.Writer, dictionary []byte, level compression.Level) error {
|
||||
if w == nil {
|
||||
return errNilReceiver
|
||||
}
|
||||
w.close()
|
||||
if writer == nil {
|
||||
return errNilIOWriter
|
||||
}
|
||||
w.w = writer
|
||||
w.level = level
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close implements compression.Writer.
|
||||
func (w *Writer) Close() error {
|
||||
if w == nil {
|
||||
return errNilReceiver
|
||||
}
|
||||
err := w.flush(true)
|
||||
w.close()
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) flush(final bool) error {
|
||||
if w.w == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if final {
|
||||
if err := w.write(nil, true); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if w.j == 0 {
|
||||
return nil
|
||||
}
|
||||
_, err := w.w.Write(w.buf[:w.j])
|
||||
w.j = 0
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) close() {
|
||||
if w.w == nil {
|
||||
return
|
||||
}
|
||||
w.j = 0
|
||||
w.w = nil
|
||||
w.writeErr = nil
|
||||
if w.z != nil {
|
||||
if (w.recycler != nil) && !w.recycler.closed && (w.recycler.z == nil) {
|
||||
w.recycler.z, w.z = w.z, nil
|
||||
} else {
|
||||
C.ZSTD_freeCCtx(w.z)
|
||||
w.z = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Write implements compression.Writer.
|
||||
func (w *Writer) Write(p []byte) (int, error) {
|
||||
if w == nil {
|
||||
return 0, errNilReceiver
|
||||
}
|
||||
if w.w == nil {
|
||||
return 0, errMissingResetCall
|
||||
}
|
||||
if w.writeErr != nil {
|
||||
return 0, w.writeErr
|
||||
}
|
||||
|
||||
originalLenP := len(p)
|
||||
for {
|
||||
remaining := []byte(nil)
|
||||
if len(p) > maxLen {
|
||||
p, remaining = p[:maxLen], p[maxLen:]
|
||||
}
|
||||
|
||||
if err := w.write(p, false); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
p, remaining = remaining, nil
|
||||
if len(p) == 0 {
|
||||
return originalLenP, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Writer) write(p []byte, final bool) error {
|
||||
if len(p) > maxLen {
|
||||
panic("unreachable")
|
||||
}
|
||||
|
||||
if w.z == nil {
|
||||
if (w.recycler != nil) && !w.recycler.closed && (w.recycler.z != nil) {
|
||||
w.z, w.recycler.z = w.recycler.z, nil
|
||||
} else {
|
||||
w.z = C.ZSTD_createCStream()
|
||||
if w.z == nil {
|
||||
return errOutOfMemory
|
||||
}
|
||||
}
|
||||
C.ZSTD_initCStream(w.z, C.int(w.zstdCompressionLevel()))
|
||||
}
|
||||
|
||||
for (len(p) > 0) || final {
|
||||
if w.j == uint32(len(w.buf)) {
|
||||
if err := w.flush(false); err != nil {
|
||||
w.writeErr = err
|
||||
return w.writeErr
|
||||
}
|
||||
}
|
||||
|
||||
e := int32(0)
|
||||
if final {
|
||||
e = int32(C.cgozstd_compress_end(w.z, &w.a,
|
||||
(*C.uint8_t)(unsafe.Pointer(&w.buf[w.j])),
|
||||
(C.uint32_t)(uint32(len(w.buf))-w.j),
|
||||
))
|
||||
final = w.a.eof == 0
|
||||
} else {
|
||||
e = int32(C.cgozstd_compress(w.z, &w.a,
|
||||
(*C.uint8_t)(unsafe.Pointer(&w.buf[w.j])),
|
||||
(C.uint32_t)(uint32(len(w.buf))-w.j),
|
||||
(*C.uint8_t)(unsafe.Pointer(&p[0])),
|
||||
(C.uint32_t)(len(p)),
|
||||
))
|
||||
}
|
||||
|
||||
w.j += uint32(w.a.ndst)
|
||||
p = p[uint32(w.a.nsrc):]
|
||||
|
||||
if e != 0 {
|
||||
w.writeErr = errCode(e)
|
||||
return w.writeErr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
// Copyright 2019 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cgozstd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const (
|
||||
// compressedMore is 15 bytes of zstd frame:
|
||||
// \x28\xb5\x2f\xfd \x00 \x58 \x31\x00\x00 \x4d\x6f\x72\x65\x21\x0a"
|
||||
// Magic----------- FHD- WD-- BH---------- BlockData----------------
|
||||
//
|
||||
// Frame Header Descriptor 0x00 means no flags set.
|
||||
// Window Descriptor: Exponent=11, Mantissa=0, Window_Size=2MiB.
|
||||
// Block Header: Last_Block=1, Block_Type=0 (Raw_Block), Block_Size=6.
|
||||
// BlockData is the literal bytes "More!\n".
|
||||
compressedMore = "\x28\xb5\x2f\xfd\x00\x58\x31\x00\x00\x4d\x6f\x72\x65\x21\x0a"
|
||||
|
||||
uncompressedMore = "More!\n"
|
||||
)
|
||||
|
||||
func TestRoundTrip(t *testing.T) {
|
||||
if !cgoEnabled {
|
||||
t.Skip("cgo is not enabled")
|
||||
}
|
||||
|
||||
wr := &WriterRecycler{}
|
||||
defer wr.Close()
|
||||
w := &Writer{}
|
||||
wr.Bind(w)
|
||||
|
||||
rr := &ReaderRecycler{}
|
||||
defer rr.Close()
|
||||
r := &Reader{}
|
||||
rr.Bind(r)
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
buf := &bytes.Buffer{}
|
||||
|
||||
// Compress.
|
||||
{
|
||||
w.Reset(buf, nil, 0)
|
||||
if _, err := w.Write([]byte(uncompressedMore)); err != nil {
|
||||
w.Close()
|
||||
t.Fatalf("i=%d: Write: %v", i, err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatalf("i=%d: Close: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
compressed := buf.String()
|
||||
if compressed != compressedMore {
|
||||
t.Fatalf("i=%d: compressed\ngot % 02x\nwant % 02x", i, compressed, compressedMore)
|
||||
}
|
||||
|
||||
// Uncompress.
|
||||
{
|
||||
r.Reset(strings.NewReader(compressed), nil)
|
||||
gotBytes, err := ioutil.ReadAll(r)
|
||||
if err != nil {
|
||||
r.Close()
|
||||
t.Fatalf("i=%d: ReadAll: %v", i, err)
|
||||
}
|
||||
if got, want := string(gotBytes), uncompressedMore; got != want {
|
||||
r.Close()
|
||||
t.Fatalf("i=%d:\ngot %q\nwant %q", i, got, want)
|
||||
}
|
||||
if err := r.Close(); err != nil {
|
||||
t.Fatalf("i=%d: Close: %v", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
// Copyright 2019 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// ----------------
|
||||
|
||||
// +build !cgo
|
||||
|
||||
package cgozstd
|
||||
|
||||
// This file contains placeholder types and funcs so that the package still
|
||||
// builds (with the same API) when CGO_ENABLED=0. The package doesn't work
|
||||
// without cgo, but it will fail at run time, not compile time.
|
||||
//
|
||||
// In particular, the build stays green regardless of whether CGO_ENABLED is on
|
||||
// or off. Installing and testing every package in the whole repository will
|
||||
// not fail. The tests in this package don't pass, but they are skipped.
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/google/wuffs/lib/compression"
|
||||
)
|
||||
|
||||
const cgoEnabled = false
|
||||
|
||||
var (
|
||||
errCgoIsNotEnabled = errors.New("cgozstd: cgo is not enabled")
|
||||
)
|
||||
|
||||
type ReaderRecycler struct{}
|
||||
|
||||
func (c *ReaderRecycler) Bind(*Reader) {}
|
||||
func (c *ReaderRecycler) Close() error { return errCgoIsNotEnabled }
|
||||
|
||||
type Reader struct{}
|
||||
|
||||
func (r *Reader) Close() error { return errCgoIsNotEnabled }
|
||||
func (r *Reader) Read([]byte) (int, error) { return 0, errCgoIsNotEnabled }
|
||||
func (r *Reader) Reset(io.Reader, []byte) error { return errCgoIsNotEnabled }
|
||||
|
||||
type WriterRecycler struct{}
|
||||
|
||||
func (c *WriterRecycler) Bind(*Writer) {}
|
||||
func (c *WriterRecycler) Close() error { return errCgoIsNotEnabled }
|
||||
|
||||
type Writer struct{}
|
||||
|
||||
func (w *Writer) Close() error { return errCgoIsNotEnabled }
|
||||
func (w *Writer) Reset(io.Writer, []byte, compression.Level) error { return errCgoIsNotEnabled }
|
||||
func (w *Writer) Write([]byte) (int, error) { return 0, errCgoIsNotEnabled }
|
||||
Reference in new issue
Block a user