mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 08:32:11 +01:00
Have raczlib.Reader support shared dictionaries
This commit is contained in:
1 parent
566e8e0424
commit
0eb23c67b6
2 files changed
+151
-1
No files matched your search
@@ -0,0 +1,58 @@
|
||||
// Copyright 2019 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package raczlib
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReaderWithDictionary(t *testing.T) {
|
||||
// encoded and decoded come from Example_indexLocationAtStart in
|
||||
// "../rac/example_test.go".
|
||||
const (
|
||||
encoded = "" +
|
||||
"\x72\xC3\x63\x04\x71\xB5\x00\xFF\x00\x00\x00\x00\x00\x00\x00\xFF" +
|
||||
"\x0B\x00\x00\x00\x00\x00\x00\xFF\x16\x00\x00\x00\x00\x00\x00\xFF" +
|
||||
"\x23\x00\x00\x00\x00\x00\x00\x01\x50\x00\x00\x00\x00\x00\x01\xFF" +
|
||||
"\x5E\x00\x00\x00\x00\x00\x01\x00\x73\x00\x00\x00\x00\x00\x01\x00" +
|
||||
"\x88\x00\x00\x00\x00\x00\x01\x00\x9F\x00\x00\x00\x00\x00\x01\x04" +
|
||||
"\x08\x00\x20\x73\x68\x65\x65\x70\x2E\x0A\x0B\xE0\x02\x6E\x78\xF9" +
|
||||
"\x0B\xE0\x02\x6E\xF2\xCF\x4B\x85\x31\x01\x01\x00\x00\xFF\xFF\x17" +
|
||||
"\x21\x03\x90\x78\xF9\x0B\xE0\x02\x6E\x0A\x29\xCF\x87\x31\x01\x01" +
|
||||
"\x00\x00\xFF\xFF\x18\x0C\x03\xA8\x78\xF9\x0B\xE0\x02\x6E\x0A\xC9" +
|
||||
"\x28\x4A\x4D\x85\x71\x00\x01\x00\x00\xFF\xFF\x21\x6E\x04\x66"
|
||||
|
||||
decoded = "" +
|
||||
"One sheep.\n" +
|
||||
"Two sheep.\n" +
|
||||
"Three sheep.\n"
|
||||
)
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
r := &Reader{
|
||||
ReadSeeker: strings.NewReader(encoded),
|
||||
CompressedSize: int64(len(encoded)),
|
||||
}
|
||||
if _, err := io.Copy(buf, r); err != nil {
|
||||
t.Fatalf("Copy: %v", err)
|
||||
}
|
||||
|
||||
if got, want := buf.String(), decoded; got != want {
|
||||
t.Fatalf("got:\n%s\nwant:\n%s", got, want)
|
||||
}
|
||||
}
|
||||
+93
-1
@@ -17,6 +17,7 @@ package raczlib
|
||||
import (
|
||||
"compress/zlib"
|
||||
"errors"
|
||||
"hash/adler32"
|
||||
"io"
|
||||
|
||||
"github.com/google/wuffs/lib/rac"
|
||||
@@ -27,11 +28,35 @@ var (
|
||||
errInvalidChunk = errors.New("raczlib: invalid chunk")
|
||||
errInvalidChunkTooLarge = errors.New("raczlib: invalid chunk (too large)")
|
||||
errInvalidChunkTruncated = errors.New("raczlib: invalid chunk (truncated)")
|
||||
errInvalidDictionary = errors.New("raczlib: invalid dictionary")
|
||||
errInvalidReadSeeker = errors.New("raczlib: invalid ReadSeeker")
|
||||
|
||||
errInternalInconsistentPosition = errors.New("raczlib: internal error: inconsistent position")
|
||||
)
|
||||
|
||||
func u32BE(b []byte) uint32 {
|
||||
_ = b[3] // Early bounds check to guarantee safety of reads below.
|
||||
return (uint32(b[0]) << 24) | (uint32(b[1]) << 16) | (uint32(b[2]) << 8) | (uint32(b[3]))
|
||||
}
|
||||
|
||||
// readAt calls ReadAt if it is available, otherwise it falls back to calling
|
||||
// Seek and then ReadFull. Calling ReadAt is presumably slightly more
|
||||
// efficient, e.g. one syscall instead of two.
|
||||
func readAt(r io.ReadSeeker, p []byte, offset int64) error {
|
||||
if a, ok := r.(io.ReaderAt); ok && false {
|
||||
n, err := a.ReadAt(p, offset)
|
||||
if (n == len(p)) && (err == io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if _, err := r.Seek(offset, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := io.ReadFull(r, p)
|
||||
return err
|
||||
}
|
||||
|
||||
// Reader reads a RAC file.
|
||||
//
|
||||
// Do not modify its exported fields after calling any of its methods.
|
||||
@@ -84,6 +109,9 @@ type Reader struct {
|
||||
zlibReader io.ReadCloser
|
||||
inImplicitZeroes bool
|
||||
|
||||
// buf is a scratch buffer.
|
||||
buf [2]byte
|
||||
|
||||
// cachedZlibReader lets us re-use the memory allocated for a zlib reader,
|
||||
// when decompressing multiple chunks.
|
||||
cachedZlibReader zlib.Resetter
|
||||
@@ -116,6 +144,10 @@ type Reader struct {
|
||||
// In "State A", the dRange is empty and unused, other than trivially
|
||||
// maintaining the invariant.
|
||||
dRange rac.Range
|
||||
|
||||
// These fields contain the most recently used shared dictionary.
|
||||
cachedDictionary []byte
|
||||
cachedDictionaryCRange rac.Range
|
||||
}
|
||||
|
||||
func (r *Reader) initialize() error {
|
||||
@@ -284,7 +316,9 @@ func (r *Reader) nextChunk() error {
|
||||
|
||||
dict := []byte(nil)
|
||||
if !chunk.CSecondary.Empty() {
|
||||
panic("TODO: dictionary support")
|
||||
if dict, err = r.loadDictionary(chunk.CSecondary); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := r.ReadSeeker.Seek(chunk.CPrimary[0], io.SeekStart); err != nil {
|
||||
@@ -320,6 +354,64 @@ func (r *Reader) nextChunk() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadDictionary loads a dictionary given a chunk's CSecondary range.
|
||||
//
|
||||
// For a description of the RAC+Zlib secondary-data format, see
|
||||
// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib
|
||||
func (r *Reader) loadDictionary(cRange rac.Range) ([]byte, error) {
|
||||
// Load from the MRU cache, if it was loaded from the same cRange.
|
||||
if (cRange == r.cachedDictionaryCRange) && !cRange.Empty() {
|
||||
return r.cachedDictionary, nil
|
||||
}
|
||||
r.cachedDictionaryCRange = rac.Range{}
|
||||
|
||||
// Check the cRange size.
|
||||
if (cRange.Size() < 6) || (cRange[1] > r.CompressedSize) {
|
||||
r.err = errInvalidDictionary
|
||||
return nil, r.err
|
||||
}
|
||||
|
||||
// Read the dictionary size.
|
||||
if err := readAt(r.ReadSeeker, r.buf[:2], cRange[0]); err != nil {
|
||||
r.err = err
|
||||
return nil, r.err
|
||||
}
|
||||
dictSize := int64(r.buf[0]) | (int64(r.buf[1]) << 8)
|
||||
|
||||
// Check the size. The +6 is for the 2 byte prefix (dictionary size) and
|
||||
// the 4 byte suffix (checksum).
|
||||
if (dictSize + 6) > cRange.Size() {
|
||||
r.err = errInvalidDictionary
|
||||
return nil, r.err
|
||||
}
|
||||
|
||||
// Allocate or re-use the cachedDictionary buffer.
|
||||
if n := dictSize + 4; int64(cap(r.cachedDictionary)) >= n {
|
||||
r.cachedDictionary = r.cachedDictionary[:n]
|
||||
} else {
|
||||
r.cachedDictionary = make([]byte, n)
|
||||
}
|
||||
|
||||
// Read the dictionary and checksum.
|
||||
if err := readAt(r.ReadSeeker, r.cachedDictionary, cRange[0]+2); err != nil {
|
||||
r.err = err
|
||||
return nil, r.err
|
||||
}
|
||||
|
||||
// Verify the checksum and trim the cachedDictionary buffer.
|
||||
checksum := r.cachedDictionary[dictSize:]
|
||||
r.cachedDictionary = r.cachedDictionary[:dictSize]
|
||||
if u32BE(checksum) != adler32.Checksum(r.cachedDictionary) {
|
||||
r.err = errInvalidDictionary
|
||||
return nil, r.err
|
||||
}
|
||||
|
||||
// Save to the MRU cache and return.
|
||||
r.cachedDictionary = r.cachedDictionary
|
||||
r.cachedDictionaryCRange = cRange
|
||||
return r.cachedDictionary, nil
|
||||
}
|
||||
|
||||
// Seek implements io.Seeker.
|
||||
func (r *Reader) Seek(offset int64, whence int) (int64, error) {
|
||||
if err := r.initialize(); err != nil {
|
||||
|
||||
Reference in new issue
Block a user