mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 08:32:11 +01:00
Add a lib/raczstd package
This commit is contained in:
1 parent
74614a2921
commit
3a3d9a3ca8
2 files changed
+261
No files matched your search
@@ -0,0 +1,111 @@
|
||||
// Copyright 2019 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package raczstd_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
|
||||
"github.com/google/wuffs/lib/rac"
|
||||
"github.com/google/wuffs/lib/raczstd"
|
||||
)
|
||||
|
||||
// Example_roundTrip demonstrates compressing (using a raczstd.Writer) and
|
||||
// decompressing (using a rac.Reader configured with a raczstd.CodecReader).
|
||||
// This includes decompressing an excerpt of the original data, exercising the
|
||||
// "random access" part of RAC.
|
||||
func Example_roundTrip() {
|
||||
// Create some test data.
|
||||
oBuf := &bytes.Buffer{}
|
||||
for i := 99; i > 0; i-- {
|
||||
fmt.Fprintf(oBuf, "%d bottles of beer on the wall, %d bottles of beer.\n"+
|
||||
"Take one down, pass it around, %d bottles of beer on the wall.\n",
|
||||
i, i, i-1)
|
||||
}
|
||||
original := oBuf.Bytes()
|
||||
|
||||
// Create the RAC file.
|
||||
cBuf := &bytes.Buffer{}
|
||||
w := &rac.Writer{
|
||||
Writer: cBuf,
|
||||
CodecWriter: &raczstd.CodecWriter{},
|
||||
// It's not necessary to explicitly declare the DChunkSize. The zero
|
||||
// value implies a reasonable default. Nonetheless, using a 1 KiB
|
||||
// DChunkSize (which is relatively small) makes for a more interesting
|
||||
// test, as the resultant RAC file then contains more than one chunk.
|
||||
DChunkSize: 1024,
|
||||
// We also use the default IndexLocation value, which makes for a
|
||||
// simpler example, but if you're copy/pasting this code, note that
|
||||
// using an explicit IndexLocationAtStart can result in slightly more
|
||||
// efficient RAC files, at the cost of using more memory to encode.
|
||||
}
|
||||
if _, err := w.Write(original); err != nil {
|
||||
log.Fatalf("Write: %v", err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
log.Fatalf("Close: %v", err)
|
||||
}
|
||||
compressed := cBuf.Bytes()
|
||||
|
||||
// The exact compression ratio depends on the zstd encoder's algorithm,
|
||||
// which can change across Go standard library releases, but it should be
|
||||
// at least a 4x ratio. It'd be larger if we didn't specify an explicit
|
||||
// (but relatively small) DChunkSize.
|
||||
if ratio := len(original) / len(compressed); ratio < 4 {
|
||||
log.Fatalf("compression ratio (%dx) was too small", ratio)
|
||||
}
|
||||
|
||||
// Prepare to decompress.
|
||||
r := &rac.Reader{
|
||||
ReadSeeker: bytes.NewReader(compressed),
|
||||
CompressedSize: int64(len(compressed)),
|
||||
CodecReaders: []rac.CodecReader{&raczstd.CodecReader{}},
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
// Read the whole file.
|
||||
wBuf := &bytes.Buffer{}
|
||||
if _, err := io.Copy(wBuf, r); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
wholeFile := wBuf.Bytes()
|
||||
if !bytes.Equal(wholeFile, original) {
|
||||
log.Fatal("round trip did not preserve whole file")
|
||||
} else {
|
||||
fmt.Printf("Whole file preserved (%d bytes).\n", len(wholeFile))
|
||||
}
|
||||
|
||||
// Read an excerpt.
|
||||
const offset, length = 3000, 1200
|
||||
want := original[offset : offset+length]
|
||||
got := make([]byte, length)
|
||||
if _, err := r.Seek(offset, io.SeekStart); err != nil {
|
||||
log.Fatalf("Seek: %v", err)
|
||||
}
|
||||
if _, err := io.ReadFull(r, got); err != nil {
|
||||
log.Fatalf("ReadFull: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
log.Fatal("round trip did not preserve excerpt")
|
||||
} else {
|
||||
fmt.Printf("Excerpt preserved (%d bytes).\n", len(got))
|
||||
}
|
||||
|
||||
// Output:
|
||||
// Whole file preserved (11357 bytes).
|
||||
// Excerpt preserved (1200 bytes).
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
// Copyright 2019 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// ----------------
|
||||
|
||||
// Package raczstd provides access to RAC (Random Access Compression) files
|
||||
// with the Zstd compression codec.
|
||||
//
|
||||
// The RAC specification is at
|
||||
// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md
|
||||
package raczstd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/google/wuffs/lib/cgozstd"
|
||||
"github.com/google/wuffs/lib/compression"
|
||||
"github.com/google/wuffs/lib/rac"
|
||||
)
|
||||
|
||||
var (
|
||||
errCannotCut = errors.New("raczstd: cannot cut")
|
||||
errInvalidCodec = errors.New("raczstd: invalid codec")
|
||||
)
|
||||
|
||||
// CodecReader specializes a rac.Reader to decode Zstd-compressed chunks.
|
||||
type CodecReader struct {
|
||||
cachedReader compression.Reader
|
||||
recycler cgozstd.ReaderRecycler
|
||||
}
|
||||
|
||||
// Close implements rac.CodecReader.
|
||||
func (r *CodecReader) Close() error {
|
||||
return r.recycler.Close()
|
||||
}
|
||||
|
||||
// Accepts implements rac.CodecReader.
|
||||
func (r *CodecReader) Accepts(c rac.Codec) bool {
|
||||
return c == rac.CodecZstandard
|
||||
}
|
||||
|
||||
// Clone implements rac.CodecReader.
|
||||
func (r *CodecReader) Clone() rac.CodecReader {
|
||||
return &CodecReader{}
|
||||
}
|
||||
|
||||
// MakeDecompressor implements rac.CodecReader.
|
||||
func (r *CodecReader) MakeDecompressor(compressed io.Reader, rctx rac.ReaderContext) (io.Reader, error) {
|
||||
if r.cachedReader == nil {
|
||||
zr := &cgozstd.Reader{}
|
||||
r.cachedReader = zr
|
||||
r.recycler.Bind(zr)
|
||||
}
|
||||
if err := r.cachedReader.Reset(compressed, rctx.Secondary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.cachedReader, nil
|
||||
}
|
||||
|
||||
// MakeReaderContext implements rac.CodecReader.
|
||||
func (r *CodecReader) MakeReaderContext(rs io.ReadSeeker, chunk rac.Chunk) (rac.ReaderContext, error) {
|
||||
return rac.ReaderContext{}, nil
|
||||
}
|
||||
|
||||
// CodecWriter specializes a rac.Writer to encode Zstd-compressed chunks.
|
||||
type CodecWriter struct {
|
||||
compressed bytes.Buffer
|
||||
cachedWriter compression.Writer
|
||||
recycler cgozstd.WriterRecycler
|
||||
}
|
||||
|
||||
// Close implements rac.CodecWriter.
|
||||
func (w *CodecWriter) Close() error {
|
||||
return w.recycler.Close()
|
||||
}
|
||||
|
||||
// Clone implements rac.CodecWriter.
|
||||
func (w *CodecWriter) Clone() rac.CodecWriter {
|
||||
return &CodecWriter{}
|
||||
}
|
||||
|
||||
// Compress implements rac.CodecWriter.
|
||||
func (w *CodecWriter) Compress(p []byte, q []byte, resourcesData [][]byte) (
|
||||
codec rac.Codec, compressed []byte, secondaryResource int, tertiaryResource int, retErr error) {
|
||||
|
||||
// Compress p+q without any shared dictionary.
|
||||
baseline, err := w.compress(p, q, nil)
|
||||
if err != nil {
|
||||
return 0, nil, 0, 0, err
|
||||
}
|
||||
return rac.CodecZstandard, baseline, rac.NoResourceUsed, rac.NoResourceUsed, nil
|
||||
}
|
||||
|
||||
func (w *CodecWriter) compress(p []byte, q []byte, dict []byte) ([]byte, error) {
|
||||
w.compressed.Reset()
|
||||
if w.cachedWriter == nil {
|
||||
zw := &cgozstd.Writer{}
|
||||
w.cachedWriter = zw
|
||||
w.recycler.Bind(zw)
|
||||
}
|
||||
if err := w.cachedWriter.Reset(&w.compressed, dict, compression.LevelSmall); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(p) > 0 {
|
||||
if _, err := w.cachedWriter.Write(p); err != nil {
|
||||
w.cachedWriter.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if len(q) > 0 {
|
||||
if _, err := w.cachedWriter.Write(q); err != nil {
|
||||
w.cachedWriter.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := w.cachedWriter.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return w.compressed.Bytes(), nil
|
||||
}
|
||||
|
||||
// CanCut implements rac.CodecWriter.
|
||||
func (w *CodecWriter) CanCut() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Cut implements rac.CodecWriter.
|
||||
func (w *CodecWriter) Cut(codec rac.Codec, encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, retErr error) {
|
||||
return 0, 0, errCannotCut
|
||||
}
|
||||
|
||||
// WrapResource implements rac.CodecWriter.
|
||||
func (w *CodecWriter) WrapResource(raw []byte) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
Reference in new issue
Block a user