Have raczlib.Reader support shared dictionaries

This commit is contained in:
Nigel Tao committed 2019-08-05 18:09:05 +10:00
1 parent 566e8e0424
commit 0eb23c67b6
2 files changed
+151 -1

No files matched your search

+58
View File
@@ -0,0 +1,58 @@
// Copyright 2019 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package raczlib
import (
"bytes"
"io"
"strings"
"testing"
)
func TestReaderWithDictionary(t *testing.T) {
// encoded and decoded come from Example_indexLocationAtStart in
// "../rac/example_test.go".
const (
encoded = "" +
"\x72\xC3\x63\x04\x71\xB5\x00\xFF\x00\x00\x00\x00\x00\x00\x00\xFF" +
"\x0B\x00\x00\x00\x00\x00\x00\xFF\x16\x00\x00\x00\x00\x00\x00\xFF" +
"\x23\x00\x00\x00\x00\x00\x00\x01\x50\x00\x00\x00\x00\x00\x01\xFF" +
"\x5E\x00\x00\x00\x00\x00\x01\x00\x73\x00\x00\x00\x00\x00\x01\x00" +
"\x88\x00\x00\x00\x00\x00\x01\x00\x9F\x00\x00\x00\x00\x00\x01\x04" +
"\x08\x00\x20\x73\x68\x65\x65\x70\x2E\x0A\x0B\xE0\x02\x6E\x78\xF9" +
"\x0B\xE0\x02\x6E\xF2\xCF\x4B\x85\x31\x01\x01\x00\x00\xFF\xFF\x17" +
"\x21\x03\x90\x78\xF9\x0B\xE0\x02\x6E\x0A\x29\xCF\x87\x31\x01\x01" +
"\x00\x00\xFF\xFF\x18\x0C\x03\xA8\x78\xF9\x0B\xE0\x02\x6E\x0A\xC9" +
"\x28\x4A\x4D\x85\x71\x00\x01\x00\x00\xFF\xFF\x21\x6E\x04\x66"
decoded = "" +
"One sheep.\n" +
"Two sheep.\n" +
"Three sheep.\n"
)
buf := &bytes.Buffer{}
r := &Reader{
ReadSeeker: strings.NewReader(encoded),
CompressedSize: int64(len(encoded)),
}
if _, err := io.Copy(buf, r); err != nil {
t.Fatalf("Copy: %v", err)
}
if got, want := buf.String(), decoded; got != want {
t.Fatalf("got:\n%s\nwant:\n%s", got, want)
}
}
+93 -1
View File
@@ -17,6 +17,7 @@ package raczlib
import (
"compress/zlib"
"errors"
"hash/adler32"
"io"
"github.com/google/wuffs/lib/rac"
@@ -27,11 +28,35 @@ var (
errInvalidChunk = errors.New("raczlib: invalid chunk")
errInvalidChunkTooLarge = errors.New("raczlib: invalid chunk (too large)")
errInvalidChunkTruncated = errors.New("raczlib: invalid chunk (truncated)")
errInvalidDictionary = errors.New("raczlib: invalid dictionary")
errInvalidReadSeeker = errors.New("raczlib: invalid ReadSeeker")
errInternalInconsistentPosition = errors.New("raczlib: internal error: inconsistent position")
)
func u32BE(b []byte) uint32 {
_ = b[3] // Early bounds check to guarantee safety of reads below.
return (uint32(b[0]) << 24) | (uint32(b[1]) << 16) | (uint32(b[2]) << 8) | (uint32(b[3]))
}
// readAt calls ReadAt if it is available, otherwise it falls back to calling
// Seek and then ReadFull. Calling ReadAt is presumably slightly more
// efficient, e.g. one syscall instead of two.
func readAt(r io.ReadSeeker, p []byte, offset int64) error {
if a, ok := r.(io.ReaderAt); ok && false {
n, err := a.ReadAt(p, offset)
if (n == len(p)) && (err == io.EOF) {
err = nil
}
return err
}
if _, err := r.Seek(offset, io.SeekStart); err != nil {
return err
}
_, err := io.ReadFull(r, p)
return err
}
// Reader reads a RAC file.
//
// Do not modify its exported fields after calling any of its methods.
@@ -84,6 +109,9 @@ type Reader struct {
zlibReader io.ReadCloser
inImplicitZeroes bool
// buf is a scratch buffer.
buf [2]byte
// cachedZlibReader lets us re-use the memory allocated for a zlib reader,
// when decompressing multiple chunks.
cachedZlibReader zlib.Resetter
@@ -116,6 +144,10 @@ type Reader struct {
// In "State A", the dRange is empty and unused, other than trivially
// maintaining the invariant.
dRange rac.Range
// These fields contain the most recently used shared dictionary.
cachedDictionary []byte
cachedDictionaryCRange rac.Range
}
func (r *Reader) initialize() error {
@@ -284,7 +316,9 @@ func (r *Reader) nextChunk() error {
dict := []byte(nil)
if !chunk.CSecondary.Empty() {
panic("TODO: dictionary support")
if dict, err = r.loadDictionary(chunk.CSecondary); err != nil {
return err
}
}
if _, err := r.ReadSeeker.Seek(chunk.CPrimary[0], io.SeekStart); err != nil {
@@ -320,6 +354,64 @@ func (r *Reader) nextChunk() error {
return nil
}
// loadDictionary loads a dictionary given a chunk's CSecondary range.
//
// For a description of the RAC+Zlib secondary-data format, see
// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib
func (r *Reader) loadDictionary(cRange rac.Range) ([]byte, error) {
// Load from the MRU cache, if it was loaded from the same cRange.
if (cRange == r.cachedDictionaryCRange) && !cRange.Empty() {
return r.cachedDictionary, nil
}
r.cachedDictionaryCRange = rac.Range{}
// Check the cRange size.
if (cRange.Size() < 6) || (cRange[1] > r.CompressedSize) {
r.err = errInvalidDictionary
return nil, r.err
}
// Read the dictionary size.
if err := readAt(r.ReadSeeker, r.buf[:2], cRange[0]); err != nil {
r.err = err
return nil, r.err
}
dictSize := int64(r.buf[0]) | (int64(r.buf[1]) << 8)
// Check the size. The +6 is for the 2 byte prefix (dictionary size) and
// the 4 byte suffix (checksum).
if (dictSize + 6) > cRange.Size() {
r.err = errInvalidDictionary
return nil, r.err
}
// Allocate or re-use the cachedDictionary buffer.
if n := dictSize + 4; int64(cap(r.cachedDictionary)) >= n {
r.cachedDictionary = r.cachedDictionary[:n]
} else {
r.cachedDictionary = make([]byte, n)
}
// Read the dictionary and checksum.
if err := readAt(r.ReadSeeker, r.cachedDictionary, cRange[0]+2); err != nil {
r.err = err
return nil, r.err
}
// Verify the checksum and trim the cachedDictionary buffer.
checksum := r.cachedDictionary[dictSize:]
r.cachedDictionary = r.cachedDictionary[:dictSize]
if u32BE(checksum) != adler32.Checksum(r.cachedDictionary) {
r.err = errInvalidDictionary
return nil, r.err
}
// Save to the MRU cache and return.
r.cachedDictionary = r.cachedDictionary
r.cachedDictionaryCRange = cRange
return r.cachedDictionary, nil
}
// Seek implements io.Seeker.
func (r *Reader) Seek(offset int64, whence int) (int64, error) {
if err := r.initialize(); err != nil {