diff --git a/lib/raczlib/raczlib_test.go b/lib/raczlib/raczlib_test.go new file mode 100644 index 00000000..af2b7969 --- /dev/null +++ b/lib/raczlib/raczlib_test.go @@ -0,0 +1,58 @@ +// Copyright 2019 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raczlib + +import ( + "bytes" + "io" + "strings" + "testing" +) + +func TestReaderWithDictionary(t *testing.T) { + // encoded and decoded come from Example_indexLocationAtStart in + // "../rac/example_test.go". + const ( + encoded = "" + + "\x72\xC3\x63\x04\x71\xB5\x00\xFF\x00\x00\x00\x00\x00\x00\x00\xFF" + + "\x0B\x00\x00\x00\x00\x00\x00\xFF\x16\x00\x00\x00\x00\x00\x00\xFF" + + "\x23\x00\x00\x00\x00\x00\x00\x01\x50\x00\x00\x00\x00\x00\x01\xFF" + + "\x5E\x00\x00\x00\x00\x00\x01\x00\x73\x00\x00\x00\x00\x00\x01\x00" + + "\x88\x00\x00\x00\x00\x00\x01\x00\x9F\x00\x00\x00\x00\x00\x01\x04" + + "\x08\x00\x20\x73\x68\x65\x65\x70\x2E\x0A\x0B\xE0\x02\x6E\x78\xF9" + + "\x0B\xE0\x02\x6E\xF2\xCF\x4B\x85\x31\x01\x01\x00\x00\xFF\xFF\x17" + + "\x21\x03\x90\x78\xF9\x0B\xE0\x02\x6E\x0A\x29\xCF\x87\x31\x01\x01" + + "\x00\x00\xFF\xFF\x18\x0C\x03\xA8\x78\xF9\x0B\xE0\x02\x6E\x0A\xC9" + + "\x28\x4A\x4D\x85\x71\x00\x01\x00\x00\xFF\xFF\x21\x6E\x04\x66" + + decoded = "" + + "One sheep.\n" + + "Two sheep.\n" + + "Three sheep.\n" + ) + + buf := &bytes.Buffer{} + r := &Reader{ + ReadSeeker: strings.NewReader(encoded), + CompressedSize: int64(len(encoded)), + } + if _, err := io.Copy(buf, r); err != nil { + t.Fatalf("Copy: %v", err) + } + + if got, want := buf.String(), decoded; got != want { + t.Fatalf("got:\n%s\nwant:\n%s", got, want) + } +} diff --git a/lib/raczlib/reader.go b/lib/raczlib/reader.go index 60967094..bc3cd13c 100644 --- a/lib/raczlib/reader.go +++ b/lib/raczlib/reader.go @@ -17,6 +17,7 @@ package raczlib import ( "compress/zlib" "errors" + "hash/adler32" "io" "github.com/google/wuffs/lib/rac" @@ -27,11 +28,35 @@ var ( errInvalidChunk = errors.New("raczlib: invalid chunk") errInvalidChunkTooLarge = errors.New("raczlib: invalid chunk (too large)") errInvalidChunkTruncated = errors.New("raczlib: invalid chunk (truncated)") + errInvalidDictionary = errors.New("raczlib: invalid dictionary") errInvalidReadSeeker = errors.New("raczlib: invalid ReadSeeker") errInternalInconsistentPosition = errors.New("raczlib: internal error: inconsistent position") ) +func u32BE(b []byte) uint32 { + _ = b[3] // Early bounds check to guarantee safety of reads below. + return (uint32(b[0]) << 24) | (uint32(b[1]) << 16) | (uint32(b[2]) << 8) | (uint32(b[3])) +} + +// readAt calls ReadAt if it is available, otherwise it falls back to calling +// Seek and then ReadFull. Calling ReadAt is presumably slightly more +// efficient, e.g. one syscall instead of two. +func readAt(r io.ReadSeeker, p []byte, offset int64) error { + if a, ok := r.(io.ReaderAt); ok && false { + n, err := a.ReadAt(p, offset) + if (n == len(p)) && (err == io.EOF) { + err = nil + } + return err + } + if _, err := r.Seek(offset, io.SeekStart); err != nil { + return err + } + _, err := io.ReadFull(r, p) + return err +} + // Reader reads a RAC file. // // Do not modify its exported fields after calling any of its methods. @@ -84,6 +109,9 @@ type Reader struct { zlibReader io.ReadCloser inImplicitZeroes bool + // buf is a scratch buffer. + buf [2]byte + // cachedZlibReader lets us re-use the memory allocated for a zlib reader, // when decompressing multiple chunks. cachedZlibReader zlib.Resetter @@ -116,6 +144,10 @@ type Reader struct { // In "State A", the dRange is empty and unused, other than trivially // maintaining the invariant. dRange rac.Range + + // These fields contain the most recently used shared dictionary. + cachedDictionary []byte + cachedDictionaryCRange rac.Range } func (r *Reader) initialize() error { @@ -284,7 +316,9 @@ func (r *Reader) nextChunk() error { dict := []byte(nil) if !chunk.CSecondary.Empty() { - panic("TODO: dictionary support") + if dict, err = r.loadDictionary(chunk.CSecondary); err != nil { + return err + } } if _, err := r.ReadSeeker.Seek(chunk.CPrimary[0], io.SeekStart); err != nil { @@ -320,6 +354,64 @@ func (r *Reader) nextChunk() error { return nil } +// loadDictionary loads a dictionary given a chunk's CSecondary range. +// +// For a description of the RAC+Zlib secondary-data format, see +// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib +func (r *Reader) loadDictionary(cRange rac.Range) ([]byte, error) { + // Load from the MRU cache, if it was loaded from the same cRange. + if (cRange == r.cachedDictionaryCRange) && !cRange.Empty() { + return r.cachedDictionary, nil + } + r.cachedDictionaryCRange = rac.Range{} + + // Check the cRange size. + if (cRange.Size() < 6) || (cRange[1] > r.CompressedSize) { + r.err = errInvalidDictionary + return nil, r.err + } + + // Read the dictionary size. + if err := readAt(r.ReadSeeker, r.buf[:2], cRange[0]); err != nil { + r.err = err + return nil, r.err + } + dictSize := int64(r.buf[0]) | (int64(r.buf[1]) << 8) + + // Check the size. The +6 is for the 2 byte prefix (dictionary size) and + // the 4 byte suffix (checksum). + if (dictSize + 6) > cRange.Size() { + r.err = errInvalidDictionary + return nil, r.err + } + + // Allocate or re-use the cachedDictionary buffer. + if n := dictSize + 4; int64(cap(r.cachedDictionary)) >= n { + r.cachedDictionary = r.cachedDictionary[:n] + } else { + r.cachedDictionary = make([]byte, n) + } + + // Read the dictionary and checksum. + if err := readAt(r.ReadSeeker, r.cachedDictionary, cRange[0]+2); err != nil { + r.err = err + return nil, r.err + } + + // Verify the checksum and trim the cachedDictionary buffer. + checksum := r.cachedDictionary[dictSize:] + r.cachedDictionary = r.cachedDictionary[:dictSize] + if u32BE(checksum) != adler32.Checksum(r.cachedDictionary) { + r.err = errInvalidDictionary + return nil, r.err + } + + // Save to the MRU cache and return. + r.cachedDictionary = r.cachedDictionary + r.cachedDictionaryCRange = cRange + return r.cachedDictionary, nil +} + // Seek implements io.Seeker. func (r *Reader) Seek(offset int64, whence int) (int64, error) { if err := r.initialize(); err != nil {