mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-17 16:32:11 +01:00
The LICENSE has changed from a single license (Apache2) to a dual license (Apache2 or MIT, at your option). If you've previously used Wuffs under the Apache2 license, you are able to keep doing so. No further action required by you. The two files listed below (previous releases) are exceptions, as they are older snapshots, frozen in time. Their license has not changed: release/c/wuffs-v0.2.c release/c/wuffs-v0.3.c Every name in the AUTHORS file, other than Google Inc., has signed Google's CLA (Contributor License Agreement).
605 lines
12 KiB
Go
605 lines
12 KiB
Go
// Copyright 2019 The Wuffs Authors.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
|
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
|
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
|
|
// option. This file may not be copied, modified, or distributed
|
|
// except according to those terms.
|
|
|
|
// ----------------
|
|
|
|
// Package cgozstd wraps the C "zstd" library.
|
|
package cgozstd
|
|
|
|
/*
|
|
#cgo pkg-config: libzstd
|
|
#include "zstd.h"
|
|
#include "zstd_errors.h"
|
|
|
|
#include <stdint.h>
|
|
|
|
// --------
|
|
#if (ZSTD_VERSION_MAJOR < 1) || (ZSTD_VERSION_MINOR < 3)
|
|
|
|
// For zstd version 1.2 and below, dictionaries simply aren't supported.
|
|
|
|
int32_t cgozstd_compress_start(ZSTD_CCtx* z,
|
|
uint8_t* dict_ptr,
|
|
uint32_t dict_len,
|
|
int compression_level) {
|
|
if (dict_len > 0) {
|
|
return -1;
|
|
}
|
|
return ZSTD_getErrorCode(ZSTD_initCStream(z, compression_level));
|
|
}
|
|
|
|
int32_t cgozstd_decompress_start(ZSTD_DCtx* z,
|
|
uint8_t* dict_ptr,
|
|
uint32_t dict_len) {
|
|
if (dict_len > 0) {
|
|
return -1;
|
|
}
|
|
return ZSTD_getErrorCode(ZSTD_initDStream(z));
|
|
}
|
|
|
|
#elif (ZSTD_VERSION_MAJOR < 1) || (ZSTD_VERSION_MINOR < 4)
|
|
|
|
// For zstd version 1.3 (but not 1.4 and above), the ZSTD_initFoo_usingDict
|
|
// functions are present in the dynamic library, but not part of the stable
|
|
// zstd API. We have to explicitly write their function prototypes (or #define
|
|
// ZSTD_STATIC_LINKING_ONLY) so that cgo knows about them.
|
|
//
|
|
// For example, Ubuntu "bionic" 18.04 LTS ships zstd version 1.3.3.
|
|
|
|
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(
|
|
ZSTD_CCtx* z,
|
|
const void* dict_ptr,
|
|
size_t dict_len,
|
|
int compression_level);
|
|
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(
|
|
ZSTD_DCtx* z,
|
|
const void* dict_ptr,
|
|
size_t dict_len);
|
|
|
|
int32_t cgozstd_compress_start(ZSTD_CCtx* z,
|
|
uint8_t* dict_ptr,
|
|
uint32_t dict_len,
|
|
int compression_level) {
|
|
return ZSTD_getErrorCode(ZSTD_initCStream_usingDict(
|
|
z, dict_ptr, dict_len, compression_level));
|
|
}
|
|
|
|
int32_t cgozstd_decompress_start(ZSTD_DCtx* z,
|
|
uint8_t* dict_ptr,
|
|
uint32_t dict_len) {
|
|
return ZSTD_getErrorCode(ZSTD_initDStream_usingDict(
|
|
z, dict_ptr, dict_len));
|
|
}
|
|
|
|
#else
|
|
|
|
// For zstd version 1.4 and above, the ZSTD_initFoo_usingDict functions are
|
|
// deprecated, and their replacements are part of the stable zstd API.
|
|
|
|
int32_t cgozstd_compress_start(ZSTD_CCtx* z,
|
|
uint8_t* dict_ptr,
|
|
uint32_t dict_len,
|
|
int compression_level) {
|
|
ZSTD_ErrorCode e;
|
|
e = ZSTD_getErrorCode(ZSTD_CCtx_reset(z, ZSTD_reset_session_only));
|
|
if (e) {
|
|
return e;
|
|
}
|
|
e = ZSTD_getErrorCode(ZSTD_CCtx_setParameter(
|
|
z, ZSTD_c_compressionLevel, compression_level));
|
|
if (e) {
|
|
return e;
|
|
}
|
|
e = ZSTD_getErrorCode(ZSTD_CCtx_loadDictionary(z, dict_ptr, dict_len));
|
|
if (e) {
|
|
return e;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int32_t cgozstd_decompress_start(ZSTD_DCtx* z,
|
|
uint8_t* dict_ptr,
|
|
uint32_t dict_len) {
|
|
ZSTD_ErrorCode e;
|
|
e = ZSTD_getErrorCode(ZSTD_DCtx_reset(z, ZSTD_reset_session_only));
|
|
if (e) {
|
|
return e;
|
|
}
|
|
e = ZSTD_getErrorCode(ZSTD_DCtx_loadDictionary(z, dict_ptr, dict_len));
|
|
if (e) {
|
|
return e;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
#endif
|
|
// --------
|
|
|
|
typedef struct {
|
|
uint32_t ndst;
|
|
uint32_t nsrc;
|
|
uint32_t eof;
|
|
} advances;
|
|
|
|
int32_t cgozstd_compress(ZSTD_CCtx* z,
|
|
advances* a,
|
|
uint8_t* dst_ptr,
|
|
uint32_t dst_len,
|
|
uint8_t* src_ptr,
|
|
uint32_t src_len) {
|
|
ZSTD_outBuffer ob;
|
|
ob.dst = dst_ptr;
|
|
ob.size = dst_len;
|
|
ob.pos = 0;
|
|
|
|
ZSTD_inBuffer ib;
|
|
ib.src = src_ptr;
|
|
ib.size = src_len;
|
|
ib.pos = 0;
|
|
|
|
size_t result = ZSTD_compressStream(z, &ob, &ib);
|
|
|
|
a->ndst = ob.pos;
|
|
a->nsrc = ib.pos;
|
|
a->eof = 0;
|
|
|
|
return ZSTD_getErrorCode(result);
|
|
}
|
|
|
|
int32_t cgozstd_compress_end(ZSTD_CCtx* z,
|
|
advances* a,
|
|
uint8_t* dst_ptr,
|
|
uint32_t dst_len) {
|
|
ZSTD_outBuffer ob;
|
|
ob.dst = dst_ptr;
|
|
ob.size = dst_len;
|
|
ob.pos = 0;
|
|
|
|
size_t result = ZSTD_endStream(z, &ob);
|
|
|
|
a->ndst = ob.pos;
|
|
a->nsrc = 0;
|
|
a->eof = (result == 0) ? 1 : 0;
|
|
|
|
return ZSTD_getErrorCode(result);
|
|
}
|
|
|
|
int32_t cgozstd_decompress(ZSTD_DCtx* z,
|
|
advances* a,
|
|
uint8_t* dst_ptr,
|
|
uint32_t dst_len,
|
|
uint8_t* src_ptr,
|
|
uint32_t src_len) {
|
|
ZSTD_outBuffer ob;
|
|
ob.dst = dst_ptr;
|
|
ob.size = dst_len;
|
|
ob.pos = 0;
|
|
|
|
ZSTD_inBuffer ib;
|
|
ib.src = src_ptr;
|
|
ib.size = src_len;
|
|
ib.pos = 0;
|
|
|
|
size_t result = ZSTD_decompressStream(z, &ob, &ib);
|
|
|
|
a->ndst = ob.pos;
|
|
a->nsrc = ib.pos;
|
|
a->eof = (result == 0) ? 1 : 0;
|
|
|
|
return ZSTD_getErrorCode(result);
|
|
}
|
|
*/
|
|
import "C"
|
|
|
|
import (
|
|
"errors"
|
|
"io"
|
|
"unsafe"
|
|
|
|
"github.com/google/wuffs/lib/compression"
|
|
)
|
|
|
|
const cgoEnabled = true
|
|
|
|
// maxLen avoids overflow concerns when converting C and Go integer types.
|
|
const maxLen = 1 << 30
|
|
|
|
var (
|
|
errMissingResetCall = errors.New("cgozstd: missing Reset call")
|
|
errNilIOReader = errors.New("cgozstd: nil io.Reader")
|
|
errNilIOWriter = errors.New("cgozstd: nil io.Writer")
|
|
errNilReceiver = errors.New("cgozstd: nil receiver")
|
|
errOutOfMemory = errors.New("cgozstd: out of memory")
|
|
errZstdVersionTooSmall = errors.New("cgozstd: zstd version too small (1.3 minimum)")
|
|
)
|
|
|
|
type errCode int32
|
|
|
|
func (e errCode) Error() string {
|
|
if s := C.GoString(C.ZSTD_getErrorString(C.ZSTD_ErrorCode(e))); s != "" {
|
|
return "cgozstd: " + s
|
|
}
|
|
return "cgozstd: unknown error"
|
|
}
|
|
|
|
func slicePointer(s []uint8) unsafe.Pointer {
|
|
if len(s) == 0 {
|
|
return nil
|
|
}
|
|
return unsafe.Pointer(&s[0])
|
|
}
|
|
|
|
// ReaderRecycler can lessen the new memory allocated when calling Reader.Reset
|
|
// on a bound Reader.
|
|
//
|
|
// It is not safe to use a ReaderRecycler and a Reader concurrently.
|
|
type ReaderRecycler struct {
|
|
z *C.ZSTD_DCtx
|
|
closed bool
|
|
}
|
|
|
|
// Bind lets r re-use the memory that is manually managed by c. Call c.Close to
|
|
// free that memory.
|
|
func (c *ReaderRecycler) Bind(r *Reader) {
|
|
r.recycler = c
|
|
}
|
|
|
|
// Close implements io.Closer.
|
|
func (c *ReaderRecycler) Close() error {
|
|
c.closed = true
|
|
if c.z != nil {
|
|
C.ZSTD_freeDCtx(c.z)
|
|
c.z = nil
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Reader decompresses from the zstd format.
|
|
//
|
|
// The zero value is not usable until Reset is called.
|
|
type Reader struct {
|
|
buf [65536]byte
|
|
i, j uint32
|
|
r io.Reader
|
|
|
|
readErr error
|
|
zstdErr error
|
|
|
|
recycler *ReaderRecycler
|
|
|
|
z *C.ZSTD_DCtx
|
|
a C.advances
|
|
}
|
|
|
|
// Reset implements compression.Reader.
|
|
func (r *Reader) Reset(reader io.Reader, dictionary []byte) error {
|
|
if r == nil {
|
|
return errNilReceiver
|
|
}
|
|
if err := r.Close(); err != nil {
|
|
return err
|
|
}
|
|
if reader == nil {
|
|
return errNilIOReader
|
|
}
|
|
if len(dictionary) > maxLen {
|
|
dictionary = dictionary[len(dictionary)-maxLen:]
|
|
}
|
|
|
|
z := (*C.ZSTD_DCtx)(nil)
|
|
if (r.recycler != nil) && !r.recycler.closed && (r.recycler.z != nil) {
|
|
z, r.recycler.z = r.recycler.z, nil
|
|
} else {
|
|
z = C.ZSTD_createDStream()
|
|
if z == nil {
|
|
return errOutOfMemory
|
|
}
|
|
}
|
|
|
|
if e := errCode(C.cgozstd_decompress_start(z,
|
|
(*C.uint8_t)(slicePointer(dictionary)),
|
|
(C.uint32_t)(len(dictionary)),
|
|
)); e != 0 {
|
|
C.ZSTD_freeDCtx(z)
|
|
if e < 0 {
|
|
return errZstdVersionTooSmall
|
|
}
|
|
return e
|
|
}
|
|
|
|
r.r = reader
|
|
r.z = z
|
|
return nil
|
|
}
|
|
|
|
// Close implements compression.Reader.
|
|
func (r *Reader) Close() error {
|
|
if r == nil {
|
|
return errNilReceiver
|
|
}
|
|
if r.r == nil {
|
|
return nil
|
|
}
|
|
r.i = 0
|
|
r.j = 0
|
|
r.r = nil
|
|
r.readErr = nil
|
|
r.zstdErr = nil
|
|
if r.z != nil {
|
|
if (r.recycler != nil) && !r.recycler.closed && (r.recycler.z == nil) {
|
|
r.recycler.z, r.z = r.z, nil
|
|
} else {
|
|
C.ZSTD_freeDCtx(r.z)
|
|
r.z = nil
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Read implements compression.Reader.
|
|
func (r *Reader) Read(p []byte) (int, error) {
|
|
if r == nil {
|
|
return 0, errNilReceiver
|
|
}
|
|
if r.r == nil {
|
|
return 0, errMissingResetCall
|
|
}
|
|
|
|
if len(p) > maxLen {
|
|
p = p[:maxLen]
|
|
}
|
|
|
|
for numRead := 0; ; {
|
|
if r.zstdErr != nil {
|
|
return numRead, r.zstdErr
|
|
}
|
|
if len(p) == 0 {
|
|
return numRead, nil
|
|
}
|
|
|
|
if r.i >= r.j {
|
|
if r.readErr != nil {
|
|
return numRead, r.readErr
|
|
}
|
|
|
|
n, err := r.r.Read(r.buf[:])
|
|
if err == io.EOF {
|
|
err = io.ErrUnexpectedEOF
|
|
}
|
|
r.i, r.j, r.readErr = 0, uint32(n), err
|
|
continue
|
|
}
|
|
|
|
e := errCode(C.cgozstd_decompress(r.z, &r.a,
|
|
(*C.uint8_t)(unsafe.Pointer(&p[0])),
|
|
(C.uint32_t)(len(p)),
|
|
(*C.uint8_t)(unsafe.Pointer(&r.buf[r.i])),
|
|
(C.uint32_t)(r.j-r.i),
|
|
))
|
|
|
|
numRead += int(r.a.ndst)
|
|
p = p[int(r.a.ndst):]
|
|
|
|
r.i += uint32(r.a.nsrc)
|
|
|
|
if e == 0 {
|
|
if r.a.eof == 0 {
|
|
continue
|
|
}
|
|
r.zstdErr = io.EOF
|
|
} else {
|
|
r.zstdErr = e
|
|
}
|
|
return numRead, r.zstdErr
|
|
}
|
|
}
|
|
|
|
// WriterRecycler can lessen the new memory allocated when calling Writer.Reset
|
|
// on a bound Writer.
|
|
//
|
|
// It is not safe to use a WriterRecycler and a Writer concurrently.
|
|
type WriterRecycler struct {
|
|
z *C.ZSTD_CCtx
|
|
closed bool
|
|
}
|
|
|
|
// Bind lets w re-use the memory that is manually managed by c. Call c.Close to
|
|
// free that memory.
|
|
func (c *WriterRecycler) Bind(w *Writer) {
|
|
w.recycler = c
|
|
}
|
|
|
|
// Close implements io.Closer.
|
|
func (c *WriterRecycler) Close() error {
|
|
c.closed = true
|
|
if c.z != nil {
|
|
C.ZSTD_freeCCtx(c.z)
|
|
c.z = nil
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Writer compresses to the zstd format.
|
|
//
|
|
// Compressed bytes may be buffered and not sent to the underlying io.Writer
|
|
// until Close is called.
|
|
//
|
|
// The zero value is not usable until Reset is called.
|
|
type Writer struct {
|
|
buf [65536]byte
|
|
j uint32
|
|
w io.Writer
|
|
|
|
writeErr error
|
|
|
|
recycler *WriterRecycler
|
|
|
|
z *C.ZSTD_CCtx
|
|
a C.advances
|
|
}
|
|
|
|
func zstdCompressionLevel(level compression.Level) int32 {
|
|
return level.Interpolate(1, 2, 3, 15, 22)
|
|
}
|
|
|
|
// Reset implements compression.Writer.
|
|
func (w *Writer) Reset(writer io.Writer, dictionary []byte, level compression.Level) error {
|
|
if w == nil {
|
|
return errNilReceiver
|
|
}
|
|
w.close()
|
|
if writer == nil {
|
|
return errNilIOWriter
|
|
}
|
|
if len(dictionary) > maxLen {
|
|
dictionary = dictionary[len(dictionary)-maxLen:]
|
|
}
|
|
|
|
z := (*C.ZSTD_CCtx)(nil)
|
|
if (w.recycler != nil) && !w.recycler.closed && (w.recycler.z != nil) {
|
|
z, w.recycler.z = w.recycler.z, nil
|
|
} else {
|
|
z = C.ZSTD_createCStream()
|
|
if z == nil {
|
|
return errOutOfMemory
|
|
}
|
|
}
|
|
|
|
if e := errCode(C.cgozstd_compress_start(z,
|
|
(*C.uint8_t)(slicePointer(dictionary)),
|
|
(C.uint32_t)(len(dictionary)),
|
|
C.int(zstdCompressionLevel(level)),
|
|
)); e != 0 {
|
|
C.ZSTD_freeCCtx(z)
|
|
if e < 0 {
|
|
return errZstdVersionTooSmall
|
|
}
|
|
return e
|
|
}
|
|
|
|
w.w = writer
|
|
w.z = z
|
|
return nil
|
|
}
|
|
|
|
// Close implements compression.Writer.
|
|
func (w *Writer) Close() error {
|
|
if w == nil {
|
|
return errNilReceiver
|
|
}
|
|
err := w.flush(true)
|
|
w.close()
|
|
return err
|
|
}
|
|
|
|
func (w *Writer) flush(final bool) error {
|
|
if w.w == nil {
|
|
return nil
|
|
}
|
|
|
|
if final {
|
|
if err := w.write(nil, true); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if w.j == 0 {
|
|
return nil
|
|
}
|
|
_, err := w.w.Write(w.buf[:w.j])
|
|
w.j = 0
|
|
return err
|
|
}
|
|
|
|
func (w *Writer) close() {
|
|
if w.w == nil {
|
|
return
|
|
}
|
|
w.j = 0
|
|
w.w = nil
|
|
w.writeErr = nil
|
|
if w.z != nil {
|
|
if (w.recycler != nil) && !w.recycler.closed && (w.recycler.z == nil) {
|
|
w.recycler.z, w.z = w.z, nil
|
|
} else {
|
|
C.ZSTD_freeCCtx(w.z)
|
|
w.z = nil
|
|
}
|
|
}
|
|
}
|
|
|
|
// Write implements compression.Writer.
|
|
func (w *Writer) Write(p []byte) (int, error) {
|
|
if w == nil {
|
|
return 0, errNilReceiver
|
|
}
|
|
if w.w == nil {
|
|
return 0, errMissingResetCall
|
|
}
|
|
if w.writeErr != nil {
|
|
return 0, w.writeErr
|
|
}
|
|
|
|
originalLenP := len(p)
|
|
for {
|
|
remaining := []byte(nil)
|
|
if len(p) > maxLen {
|
|
p, remaining = p[:maxLen], p[maxLen:]
|
|
}
|
|
|
|
if err := w.write(p, false); err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
p, remaining = remaining, nil
|
|
if len(p) == 0 {
|
|
return originalLenP, nil
|
|
}
|
|
}
|
|
}
|
|
|
|
func (w *Writer) write(p []byte, final bool) error {
|
|
if len(p) > maxLen {
|
|
panic("unreachable")
|
|
}
|
|
|
|
for (len(p) > 0) || final {
|
|
if w.j == uint32(len(w.buf)) {
|
|
if err := w.flush(false); err != nil {
|
|
w.writeErr = err
|
|
return w.writeErr
|
|
}
|
|
}
|
|
|
|
e := errCode(0)
|
|
if final {
|
|
e = errCode(C.cgozstd_compress_end(w.z, &w.a,
|
|
(*C.uint8_t)(unsafe.Pointer(&w.buf[w.j])),
|
|
(C.uint32_t)(uint32(len(w.buf))-w.j),
|
|
))
|
|
final = w.a.eof == 0
|
|
} else {
|
|
e = errCode(C.cgozstd_compress(w.z, &w.a,
|
|
(*C.uint8_t)(unsafe.Pointer(&w.buf[w.j])),
|
|
(C.uint32_t)(uint32(len(w.buf))-w.j),
|
|
(*C.uint8_t)(unsafe.Pointer(&p[0])),
|
|
(C.uint32_t)(len(p)),
|
|
))
|
|
}
|
|
|
|
w.j += uint32(w.a.ndst)
|
|
p = p[uint32(w.a.nsrc):]
|
|
|
|
if e != 0 {
|
|
w.writeErr = e
|
|
return w.writeErr
|
|
}
|
|
}
|
|
return nil
|
|
}
|