Add a lib/raclz4 package

This commit is contained in:
Nigel Tao committed 2019-09-15 14:13:45 +10:00
1 parent 01ebdae372
commit 00b6d22333
3 files changed
+267

No files matched your search

+4
View File
@@ -193,6 +193,7 @@ import (
"strings"
"github.com/google/wuffs/lib/rac"
"github.com/google/wuffs/lib/raclz4"
"github.com/google/wuffs/lib/raczlib"
"github.com/google/wuffs/lib/raczstd"
)
@@ -377,6 +378,7 @@ func decode(inFile *os.File) error {
ReadSeeker: rs,
CompressedSize: compressedSize,
CodecReaders: []rac.CodecReader{
&raclz4.CodecReader{},
&raczlib.CodecReader{},
&raczstd.CodecReader{},
},
@@ -446,6 +448,8 @@ func encode(r io.Reader) error {
DChunkSize: uint64(dchunksize),
}
switch *codecFlag {
case "lz4":
w.CodecWriter = &raclz4.CodecWriter{}
case "zlib":
w.CodecWriter = &raczlib.CodecWriter{}
case "zstd":
+111
View File
@@ -0,0 +1,111 @@
// Copyright 2019 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package raclz4_test
import (
"bytes"
"fmt"
"io"
"log"
"github.com/google/wuffs/lib/rac"
"github.com/google/wuffs/lib/raclz4"
)
// Example_roundTrip demonstrates compressing (using a rac.Writer and a
// raclz4.CodecWriter) and decompressing (using a rac.Reader and a
// raclz4.CodecReader). This includes decompressing an excerpt of the original
// data, exercising the "random access" part of RAC.
func Example_roundTrip() {
// Create some test data.
oBuf := &bytes.Buffer{}
for i := 99; i > 0; i-- {
fmt.Fprintf(oBuf, "%d bottles of beer on the wall, %d bottles of beer.\n"+
"Take one down, pass it around, %d bottles of beer on the wall.\n",
i, i, i-1)
}
original := oBuf.Bytes()
// Create the RAC file.
cBuf := &bytes.Buffer{}
w := &rac.Writer{
Writer: cBuf,
CodecWriter: &raclz4.CodecWriter{},
// It's not necessary to explicitly declare the DChunkSize. The zero
// value implies a reasonable default. Nonetheless, using a 1 KiB
// DChunkSize (which is relatively small) makes for a more interesting
// test, as the resultant RAC file then contains more than one chunk.
DChunkSize: 1024,
// We also use the default IndexLocation value, which makes for a
// simpler example, but if you're copy/pasting this code, note that
// using an explicit IndexLocationAtStart can result in slightly more
// efficient RAC files, at the cost of using more memory to encode.
}
if _, err := w.Write(original); err != nil {
log.Fatalf("Write: %v", err)
}
if err := w.Close(); err != nil {
log.Fatalf("Close: %v", err)
}
compressed := cBuf.Bytes()
// The exact compression ratio depends on the lz4 encoder's algorithm,
// which can change across Go standard library releases, but it should be
// at least a 3x ratio. It'd be larger if we didn't specify an explicit
// (but relatively small) DChunkSize.
if ratio := len(original) / len(compressed); ratio < 3 {
log.Fatalf("compression ratio (%dx) was too small", ratio)
}
// Prepare to decompress.
r := &rac.Reader{
ReadSeeker: bytes.NewReader(compressed),
CompressedSize: int64(len(compressed)),
CodecReaders: []rac.CodecReader{&raclz4.CodecReader{}},
}
defer r.Close()
// Read the whole file.
wBuf := &bytes.Buffer{}
if _, err := io.Copy(wBuf, r); err != nil {
log.Fatal(err)
}
wholeFile := wBuf.Bytes()
if !bytes.Equal(wholeFile, original) {
log.Fatal("round trip did not preserve whole file")
} else {
fmt.Printf("Whole file preserved (%d bytes).\n", len(wholeFile))
}
// Read an excerpt.
const offset, length = 3000, 1200
want := original[offset : offset+length]
got := make([]byte, length)
if _, err := r.Seek(offset, io.SeekStart); err != nil {
log.Fatalf("Seek: %v", err)
}
if _, err := io.ReadFull(r, got); err != nil {
log.Fatalf("ReadFull: %v", err)
}
if !bytes.Equal(got, want) {
log.Fatal("round trip did not preserve excerpt")
} else {
fmt.Printf("Excerpt preserved (%d bytes).\n", len(got))
}
// Output:
// Whole file preserved (11357 bytes).
// Excerpt preserved (1200 bytes).
}
+152
View File
@@ -0,0 +1,152 @@
// Copyright 2019 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------
// Package raclz4 provides access to RAC (Random Access Compression) files with
// the LZ4 compression codec.
//
// The RAC specification is at
// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md
package raclz4
import (
"bytes"
"errors"
"io"
"github.com/google/wuffs/lib/cgolz4"
"github.com/google/wuffs/lib/compression"
"github.com/google/wuffs/lib/rac"
)
var (
errCannotCut = errors.New("raclz4: cannot cut")
errInvalidCodec = errors.New("raclz4: invalid codec")
)
// CodecReader specializes a rac.Reader to decode LZ4-compressed chunks.
type CodecReader struct {
cachedReader compression.Reader
recycler cgolz4.ReaderRecycler
lim io.LimitedReader
}
// Close implements rac.CodecReader.
func (r *CodecReader) Close() error {
return r.recycler.Close()
}
// Accepts implements rac.CodecReader.
func (r *CodecReader) Accepts(c rac.Codec) bool {
return c == rac.CodecLZ4
}
// Clone implements rac.CodecReader.
func (r *CodecReader) Clone() rac.CodecReader {
return &CodecReader{}
}
// MakeDecompressor implements rac.CodecReader.
func (r *CodecReader) MakeDecompressor(racFile io.ReadSeeker, chunk rac.Chunk) (io.Reader, error) {
if _, err := racFile.Seek(chunk.CPrimary[0], io.SeekStart); err != nil {
return nil, err
}
r.lim.R = racFile
r.lim.N = chunk.CPrimary.Size()
if r.cachedReader == nil {
zr := &cgolz4.Reader{}
r.cachedReader = zr
r.recycler.Bind(zr)
}
if err := r.cachedReader.Reset(&r.lim, nil); err != nil {
return nil, err
}
return r.cachedReader, nil
}
// CodecWriter specializes a rac.Writer to encode LZ4-compressed chunks.
type CodecWriter struct {
compressed bytes.Buffer
cachedWriter compression.Writer
recycler cgolz4.WriterRecycler
}
// Close implements rac.CodecWriter.
func (w *CodecWriter) Close() error {
return w.recycler.Close()
}
// Clone implements rac.CodecWriter.
func (w *CodecWriter) Clone() rac.CodecWriter {
return &CodecWriter{}
}
// Compress implements rac.CodecWriter.
func (w *CodecWriter) Compress(p []byte, q []byte, resourcesData [][]byte) (
codec rac.Codec, compressed []byte, secondaryResource int, tertiaryResource int, retErr error) {
// Compress p+q without any shared dictionary.
baseline, err := w.compress(p, q, nil)
if err != nil {
return 0, nil, 0, 0, err
}
return rac.CodecLZ4, baseline, rac.NoResourceUsed, rac.NoResourceUsed, nil
}
func (w *CodecWriter) compress(p []byte, q []byte, dict []byte) ([]byte, error) {
w.compressed.Reset()
if w.cachedWriter == nil {
zw := &cgolz4.Writer{}
w.cachedWriter = zw
w.recycler.Bind(zw)
}
if err := w.cachedWriter.Reset(&w.compressed, dict, compression.LevelSmall); err != nil {
return nil, err
}
if len(p) > 0 {
if _, err := w.cachedWriter.Write(p); err != nil {
w.cachedWriter.Close()
return nil, err
}
}
if len(q) > 0 {
if _, err := w.cachedWriter.Write(q); err != nil {
w.cachedWriter.Close()
return nil, err
}
}
if err := w.cachedWriter.Close(); err != nil {
return nil, err
}
return w.compressed.Bytes(), nil
}
// CanCut implements rac.CodecWriter.
func (w *CodecWriter) CanCut() bool {
return false
}
// Cut implements rac.CodecWriter.
func (w *CodecWriter) Cut(codec rac.Codec, encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, retErr error) {
return 0, 0, errCannotCut
}
// WrapResource implements rac.CodecWriter.
func (w *CodecWriter) WrapResource(raw []byte) ([]byte, error) {
return nil, nil
}