From 3a3d9a3ca8d6d170513f28c40427a981cef73c83 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sat, 14 Sep 2019 12:57:15 +1000 Subject: [PATCH] Add a lib/raczstd package --- lib/raczstd/example_test.go | 111 ++++++++++++++++++++++++++ lib/raczstd/raczstd.go | 150 ++++++++++++++++++++++++++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 lib/raczstd/example_test.go create mode 100644 lib/raczstd/raczstd.go diff --git a/lib/raczstd/example_test.go b/lib/raczstd/example_test.go new file mode 100644 index 00000000..1d0e2bc2 --- /dev/null +++ b/lib/raczstd/example_test.go @@ -0,0 +1,111 @@ +// Copyright 2019 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raczstd_test + +import ( + "bytes" + "fmt" + "io" + "log" + + "github.com/google/wuffs/lib/rac" + "github.com/google/wuffs/lib/raczstd" +) + +// Example_roundTrip demonstrates compressing (using a raczstd.Writer) and +// decompressing (using a rac.Reader configured with a raczstd.CodecReader). +// This includes decompressing an excerpt of the original data, exercising the +// "random access" part of RAC. +func Example_roundTrip() { + // Create some test data. + oBuf := &bytes.Buffer{} + for i := 99; i > 0; i-- { + fmt.Fprintf(oBuf, "%d bottles of beer on the wall, %d bottles of beer.\n"+ + "Take one down, pass it around, %d bottles of beer on the wall.\n", + i, i, i-1) + } + original := oBuf.Bytes() + + // Create the RAC file. + cBuf := &bytes.Buffer{} + w := &rac.Writer{ + Writer: cBuf, + CodecWriter: &raczstd.CodecWriter{}, + // It's not necessary to explicitly declare the DChunkSize. The zero + // value implies a reasonable default. Nonetheless, using a 1 KiB + // DChunkSize (which is relatively small) makes for a more interesting + // test, as the resultant RAC file then contains more than one chunk. + DChunkSize: 1024, + // We also use the default IndexLocation value, which makes for a + // simpler example, but if you're copy/pasting this code, note that + // using an explicit IndexLocationAtStart can result in slightly more + // efficient RAC files, at the cost of using more memory to encode. + } + if _, err := w.Write(original); err != nil { + log.Fatalf("Write: %v", err) + } + if err := w.Close(); err != nil { + log.Fatalf("Close: %v", err) + } + compressed := cBuf.Bytes() + + // The exact compression ratio depends on the zstd encoder's algorithm, + // which can change across Go standard library releases, but it should be + // at least a 4x ratio. It'd be larger if we didn't specify an explicit + // (but relatively small) DChunkSize. + if ratio := len(original) / len(compressed); ratio < 4 { + log.Fatalf("compression ratio (%dx) was too small", ratio) + } + + // Prepare to decompress. + r := &rac.Reader{ + ReadSeeker: bytes.NewReader(compressed), + CompressedSize: int64(len(compressed)), + CodecReaders: []rac.CodecReader{&raczstd.CodecReader{}}, + } + defer r.Close() + + // Read the whole file. + wBuf := &bytes.Buffer{} + if _, err := io.Copy(wBuf, r); err != nil { + log.Fatal(err) + } + wholeFile := wBuf.Bytes() + if !bytes.Equal(wholeFile, original) { + log.Fatal("round trip did not preserve whole file") + } else { + fmt.Printf("Whole file preserved (%d bytes).\n", len(wholeFile)) + } + + // Read an excerpt. + const offset, length = 3000, 1200 + want := original[offset : offset+length] + got := make([]byte, length) + if _, err := r.Seek(offset, io.SeekStart); err != nil { + log.Fatalf("Seek: %v", err) + } + if _, err := io.ReadFull(r, got); err != nil { + log.Fatalf("ReadFull: %v", err) + } + if !bytes.Equal(got, want) { + log.Fatal("round trip did not preserve excerpt") + } else { + fmt.Printf("Excerpt preserved (%d bytes).\n", len(got)) + } + + // Output: + // Whole file preserved (11357 bytes). + // Excerpt preserved (1200 bytes). +} diff --git a/lib/raczstd/raczstd.go b/lib/raczstd/raczstd.go new file mode 100644 index 00000000..28c029d1 --- /dev/null +++ b/lib/raczstd/raczstd.go @@ -0,0 +1,150 @@ +// Copyright 2019 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ---------------- + +// Package raczstd provides access to RAC (Random Access Compression) files +// with the Zstd compression codec. +// +// The RAC specification is at +// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md +package raczstd + +import ( + "bytes" + "errors" + "io" + + "github.com/google/wuffs/lib/cgozstd" + "github.com/google/wuffs/lib/compression" + "github.com/google/wuffs/lib/rac" +) + +var ( + errCannotCut = errors.New("raczstd: cannot cut") + errInvalidCodec = errors.New("raczstd: invalid codec") +) + +// CodecReader specializes a rac.Reader to decode Zstd-compressed chunks. +type CodecReader struct { + cachedReader compression.Reader + recycler cgozstd.ReaderRecycler +} + +// Close implements rac.CodecReader. +func (r *CodecReader) Close() error { + return r.recycler.Close() +} + +// Accepts implements rac.CodecReader. +func (r *CodecReader) Accepts(c rac.Codec) bool { + return c == rac.CodecZstandard +} + +// Clone implements rac.CodecReader. +func (r *CodecReader) Clone() rac.CodecReader { + return &CodecReader{} +} + +// MakeDecompressor implements rac.CodecReader. +func (r *CodecReader) MakeDecompressor(compressed io.Reader, rctx rac.ReaderContext) (io.Reader, error) { + if r.cachedReader == nil { + zr := &cgozstd.Reader{} + r.cachedReader = zr + r.recycler.Bind(zr) + } + if err := r.cachedReader.Reset(compressed, rctx.Secondary); err != nil { + return nil, err + } + return r.cachedReader, nil +} + +// MakeReaderContext implements rac.CodecReader. +func (r *CodecReader) MakeReaderContext(rs io.ReadSeeker, chunk rac.Chunk) (rac.ReaderContext, error) { + return rac.ReaderContext{}, nil +} + +// CodecWriter specializes a rac.Writer to encode Zstd-compressed chunks. +type CodecWriter struct { + compressed bytes.Buffer + cachedWriter compression.Writer + recycler cgozstd.WriterRecycler +} + +// Close implements rac.CodecWriter. +func (w *CodecWriter) Close() error { + return w.recycler.Close() +} + +// Clone implements rac.CodecWriter. +func (w *CodecWriter) Clone() rac.CodecWriter { + return &CodecWriter{} +} + +// Compress implements rac.CodecWriter. +func (w *CodecWriter) Compress(p []byte, q []byte, resourcesData [][]byte) ( + codec rac.Codec, compressed []byte, secondaryResource int, tertiaryResource int, retErr error) { + + // Compress p+q without any shared dictionary. + baseline, err := w.compress(p, q, nil) + if err != nil { + return 0, nil, 0, 0, err + } + return rac.CodecZstandard, baseline, rac.NoResourceUsed, rac.NoResourceUsed, nil +} + +func (w *CodecWriter) compress(p []byte, q []byte, dict []byte) ([]byte, error) { + w.compressed.Reset() + if w.cachedWriter == nil { + zw := &cgozstd.Writer{} + w.cachedWriter = zw + w.recycler.Bind(zw) + } + if err := w.cachedWriter.Reset(&w.compressed, dict, compression.LevelSmall); err != nil { + return nil, err + } + + if len(p) > 0 { + if _, err := w.cachedWriter.Write(p); err != nil { + w.cachedWriter.Close() + return nil, err + } + } + if len(q) > 0 { + if _, err := w.cachedWriter.Write(q); err != nil { + w.cachedWriter.Close() + return nil, err + } + } + + if err := w.cachedWriter.Close(); err != nil { + return nil, err + } + return w.compressed.Bytes(), nil +} + +// CanCut implements rac.CodecWriter. +func (w *CodecWriter) CanCut() bool { + return false +} + +// Cut implements rac.CodecWriter. +func (w *CodecWriter) Cut(codec rac.Codec, encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, retErr error) { + return 0, 0, errCannotCut +} + +// WrapResource implements rac.CodecWriter. +func (w *CodecWriter) WrapResource(raw []byte) ([]byte, error) { + return nil, nil +}