diff --git a/cmd/ractool/main.go b/cmd/ractool/main.go index fc390ce6..5268394e 100644 --- a/cmd/ractool/main.go +++ b/cmd/ractool/main.go @@ -193,6 +193,7 @@ import ( "strings" "github.com/google/wuffs/lib/rac" + "github.com/google/wuffs/lib/raclz4" "github.com/google/wuffs/lib/raczlib" "github.com/google/wuffs/lib/raczstd" ) @@ -377,6 +378,7 @@ func decode(inFile *os.File) error { ReadSeeker: rs, CompressedSize: compressedSize, CodecReaders: []rac.CodecReader{ + &raclz4.CodecReader{}, &raczlib.CodecReader{}, &raczstd.CodecReader{}, }, @@ -446,6 +448,8 @@ func encode(r io.Reader) error { DChunkSize: uint64(dchunksize), } switch *codecFlag { + case "lz4": + w.CodecWriter = &raclz4.CodecWriter{} case "zlib": w.CodecWriter = &raczlib.CodecWriter{} case "zstd": diff --git a/lib/raclz4/example_test.go b/lib/raclz4/example_test.go new file mode 100644 index 00000000..1579d4f9 --- /dev/null +++ b/lib/raclz4/example_test.go @@ -0,0 +1,111 @@ +// Copyright 2019 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raclz4_test + +import ( + "bytes" + "fmt" + "io" + "log" + + "github.com/google/wuffs/lib/rac" + "github.com/google/wuffs/lib/raclz4" +) + +// Example_roundTrip demonstrates compressing (using a rac.Writer and a +// raclz4.CodecWriter) and decompressing (using a rac.Reader and a +// raclz4.CodecReader). This includes decompressing an excerpt of the original +// data, exercising the "random access" part of RAC. +func Example_roundTrip() { + // Create some test data. + oBuf := &bytes.Buffer{} + for i := 99; i > 0; i-- { + fmt.Fprintf(oBuf, "%d bottles of beer on the wall, %d bottles of beer.\n"+ + "Take one down, pass it around, %d bottles of beer on the wall.\n", + i, i, i-1) + } + original := oBuf.Bytes() + + // Create the RAC file. + cBuf := &bytes.Buffer{} + w := &rac.Writer{ + Writer: cBuf, + CodecWriter: &raclz4.CodecWriter{}, + // It's not necessary to explicitly declare the DChunkSize. The zero + // value implies a reasonable default. Nonetheless, using a 1 KiB + // DChunkSize (which is relatively small) makes for a more interesting + // test, as the resultant RAC file then contains more than one chunk. + DChunkSize: 1024, + // We also use the default IndexLocation value, which makes for a + // simpler example, but if you're copy/pasting this code, note that + // using an explicit IndexLocationAtStart can result in slightly more + // efficient RAC files, at the cost of using more memory to encode. + } + if _, err := w.Write(original); err != nil { + log.Fatalf("Write: %v", err) + } + if err := w.Close(); err != nil { + log.Fatalf("Close: %v", err) + } + compressed := cBuf.Bytes() + + // The exact compression ratio depends on the lz4 encoder's algorithm, + // which can change across Go standard library releases, but it should be + // at least a 3x ratio. It'd be larger if we didn't specify an explicit + // (but relatively small) DChunkSize. + if ratio := len(original) / len(compressed); ratio < 3 { + log.Fatalf("compression ratio (%dx) was too small", ratio) + } + + // Prepare to decompress. + r := &rac.Reader{ + ReadSeeker: bytes.NewReader(compressed), + CompressedSize: int64(len(compressed)), + CodecReaders: []rac.CodecReader{&raclz4.CodecReader{}}, + } + defer r.Close() + + // Read the whole file. + wBuf := &bytes.Buffer{} + if _, err := io.Copy(wBuf, r); err != nil { + log.Fatal(err) + } + wholeFile := wBuf.Bytes() + if !bytes.Equal(wholeFile, original) { + log.Fatal("round trip did not preserve whole file") + } else { + fmt.Printf("Whole file preserved (%d bytes).\n", len(wholeFile)) + } + + // Read an excerpt. + const offset, length = 3000, 1200 + want := original[offset : offset+length] + got := make([]byte, length) + if _, err := r.Seek(offset, io.SeekStart); err != nil { + log.Fatalf("Seek: %v", err) + } + if _, err := io.ReadFull(r, got); err != nil { + log.Fatalf("ReadFull: %v", err) + } + if !bytes.Equal(got, want) { + log.Fatal("round trip did not preserve excerpt") + } else { + fmt.Printf("Excerpt preserved (%d bytes).\n", len(got)) + } + + // Output: + // Whole file preserved (11357 bytes). + // Excerpt preserved (1200 bytes). +} diff --git a/lib/raclz4/raclz4.go b/lib/raclz4/raclz4.go new file mode 100644 index 00000000..047c09d8 --- /dev/null +++ b/lib/raclz4/raclz4.go @@ -0,0 +1,152 @@ +// Copyright 2019 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ---------------- + +// Package raclz4 provides access to RAC (Random Access Compression) files with +// the LZ4 compression codec. +// +// The RAC specification is at +// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md +package raclz4 + +import ( + "bytes" + "errors" + "io" + + "github.com/google/wuffs/lib/cgolz4" + "github.com/google/wuffs/lib/compression" + "github.com/google/wuffs/lib/rac" +) + +var ( + errCannotCut = errors.New("raclz4: cannot cut") + errInvalidCodec = errors.New("raclz4: invalid codec") +) + +// CodecReader specializes a rac.Reader to decode LZ4-compressed chunks. +type CodecReader struct { + cachedReader compression.Reader + recycler cgolz4.ReaderRecycler + lim io.LimitedReader +} + +// Close implements rac.CodecReader. +func (r *CodecReader) Close() error { + return r.recycler.Close() +} + +// Accepts implements rac.CodecReader. +func (r *CodecReader) Accepts(c rac.Codec) bool { + return c == rac.CodecLZ4 +} + +// Clone implements rac.CodecReader. +func (r *CodecReader) Clone() rac.CodecReader { + return &CodecReader{} +} + +// MakeDecompressor implements rac.CodecReader. +func (r *CodecReader) MakeDecompressor(racFile io.ReadSeeker, chunk rac.Chunk) (io.Reader, error) { + if _, err := racFile.Seek(chunk.CPrimary[0], io.SeekStart); err != nil { + return nil, err + } + r.lim.R = racFile + r.lim.N = chunk.CPrimary.Size() + + if r.cachedReader == nil { + zr := &cgolz4.Reader{} + r.cachedReader = zr + r.recycler.Bind(zr) + } + if err := r.cachedReader.Reset(&r.lim, nil); err != nil { + return nil, err + } + return r.cachedReader, nil +} + +// CodecWriter specializes a rac.Writer to encode LZ4-compressed chunks. +type CodecWriter struct { + compressed bytes.Buffer + cachedWriter compression.Writer + recycler cgolz4.WriterRecycler +} + +// Close implements rac.CodecWriter. +func (w *CodecWriter) Close() error { + return w.recycler.Close() +} + +// Clone implements rac.CodecWriter. +func (w *CodecWriter) Clone() rac.CodecWriter { + return &CodecWriter{} +} + +// Compress implements rac.CodecWriter. +func (w *CodecWriter) Compress(p []byte, q []byte, resourcesData [][]byte) ( + codec rac.Codec, compressed []byte, secondaryResource int, tertiaryResource int, retErr error) { + + // Compress p+q without any shared dictionary. + baseline, err := w.compress(p, q, nil) + if err != nil { + return 0, nil, 0, 0, err + } + return rac.CodecLZ4, baseline, rac.NoResourceUsed, rac.NoResourceUsed, nil +} + +func (w *CodecWriter) compress(p []byte, q []byte, dict []byte) ([]byte, error) { + w.compressed.Reset() + if w.cachedWriter == nil { + zw := &cgolz4.Writer{} + w.cachedWriter = zw + w.recycler.Bind(zw) + } + if err := w.cachedWriter.Reset(&w.compressed, dict, compression.LevelSmall); err != nil { + return nil, err + } + + if len(p) > 0 { + if _, err := w.cachedWriter.Write(p); err != nil { + w.cachedWriter.Close() + return nil, err + } + } + if len(q) > 0 { + if _, err := w.cachedWriter.Write(q); err != nil { + w.cachedWriter.Close() + return nil, err + } + } + + if err := w.cachedWriter.Close(); err != nil { + return nil, err + } + return w.compressed.Bytes(), nil +} + +// CanCut implements rac.CodecWriter. +func (w *CodecWriter) CanCut() bool { + return false +} + +// Cut implements rac.CodecWriter. +func (w *CodecWriter) Cut(codec rac.Codec, encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, retErr error) { + return 0, 0, errCannotCut +} + +// WrapResource implements rac.CodecWriter. +func (w *CodecWriter) WrapResource(raw []byte) ([]byte, error) { + return nil, nil +}