Add a rac.CodecReader abstraction

This commit is contained in:
Nigel Tao committed 2019-08-08 12:09:25 +10:00
1 parent 635c3c8fe9
commit cdf49196b1
4 files changed
+71 -128

No files matched your search

+35 -22
View File
@@ -16,15 +16,15 @@ package rac
import (
"errors"
"fmt"
"io"
)
var (
errInvalidChunk = errors.New("rac: invalid chunk")
errInvalidChunkTooLarge = errors.New("rac: invalid chunk (too large)")
errInvalidChunkTruncated = errors.New("rac: invalid chunk (truncated)")
errInvalidMakeXxxFunction = errors.New("rac: invalid MakeXxx function")
errInvalidReadSeeker = errors.New("rac: invalid ReadSeeker")
errInvalidChunk = errors.New("rac: invalid chunk")
errInvalidChunkTooLarge = errors.New("rac: invalid chunk (too large)")
errInvalidChunkTruncated = errors.New("rac: invalid chunk (truncated)")
errInvalidReadSeeker = errors.New("rac: invalid ReadSeeker")
errInternalInconsistentPosition = errors.New("rac: internal error: inconsistent position")
)
@@ -40,11 +40,23 @@ type ReaderContext struct {
Extra interface{}
}
// CodecReader specializes a Reader to decode a specific compression codec.
type CodecReader interface {
// Accepts returns whether this CodecReader can decode a Codec.
Accepts(c Codec) bool
// MakeDecompressor returns the Codec-specific io.Reader for a chunk.
//
// The returned io.Reader may optionally implement the io.Closer interface,
// in which case this Reader will call Close when has finished the chunk.
MakeDecompressor(r io.Reader, rctx ReaderContext) (io.Reader, error)
// MakeReaderContext returns the Codec-specific ReaderContext for a chunk.
MakeReaderContext(r io.ReadSeeker, compressedSize int64, c Chunk) (ReaderContext, error)
}
// Reader reads a RAC file.
//
// Users typically do not refer to this type directly. Instead, they use higher
// level packages like the sibling "raczlib" package.
//
// Do not modify its exported fields after calling any of its methods.
type Reader struct {
// ReadSeeker is where the RAC-encoded data is read from.
@@ -66,14 +78,7 @@ type Reader struct {
// Zero is an invalid value, as an empty file is not a valid RAC file.
CompressedSize int64
// MakeDecompressor returns the Codec-specific Decompressor for a chunk.
//
// The returned io.Reader may optionally implement the io.Closer interface,
// in which case this Reader will call Close when has finished the chunk.
MakeDecompressor func(io.Reader, ReaderContext) (io.Reader, error)
// MakeReaderContext returns the Codec-specific ReaderContext for a chunk.
MakeReaderContext func(Chunk) (ReaderContext, error)
CodecReaders []CodecReader
// err is the first error encountered. It is sticky: once a non-nil error
// occurs, all public methods will return that error.
@@ -146,10 +151,6 @@ func (r *Reader) initialize() error {
r.err = errInvalidReadSeeker
return r.err
}
if (r.MakeDecompressor == nil) || (r.MakeReaderContext == nil) {
r.err = errInvalidMakeXxxFunction
return r.err
}
r.parser.ReadSeeker = r.ReadSeeker
r.parser.CompressedSize = r.CompressedSize
@@ -300,7 +301,19 @@ func (r *Reader) nextChunk() error {
return r.err
}
rctx, err := r.MakeReaderContext(chunk)
codecReader := CodecReader(nil)
for _, cr := range r.CodecReaders {
if cr.Accepts(chunk.Codec) {
codecReader = cr
break
}
}
if codecReader == nil {
r.err = fmt.Errorf("rac: no matching CodecReader for Codec 0x%02X", chunk.Codec)
return r.err
}
rctx, err := codecReader.MakeReaderContext(r.ReadSeeker, r.CompressedSize, chunk)
if err != nil {
r.err = err
return r.err
@@ -316,7 +329,7 @@ func (r *Reader) nextChunk() error {
r.currChunk.N = chunk.CPrimary.Size()
r.dRange = chunk.DRange
decompressor, err := r.MakeDecompressor(&r.currChunk, rctx)
decompressor, err := codecReader.MakeDecompressor(&r.currChunk, rctx)
if err != nil {
r.err = err
return r.err
+6 -3
View File
@@ -20,12 +20,14 @@ import (
"io"
"log"
"github.com/google/wuffs/lib/rac"
"github.com/google/wuffs/lib/raczlib"
)
// Example_roundTrip demonstrates compressing (using a raczlib.Writer) and
// decompressing (using a raczlib.Reader). This includes decompressing an
// excerpt of the original data, exercising the "random access" part of RAC.
// decompressing (using a rac.Reader configured with a raczlib.CodecReader).
// This includes decompressing an excerpt of the original data, exercising the
// "random access" part of RAC.
func Example_roundTrip() {
// Create some test data.
oBuf := &bytes.Buffer{}
@@ -67,9 +69,10 @@ func Example_roundTrip() {
}
// Prepare to decompress.
r := &raczlib.Reader{
r := &rac.Reader{
ReadSeeker: bytes.NewReader(compressed),
CompressedSize: int64(len(compressed)),
CodecReaders: []rac.CodecReader{&raczlib.CodecReader{}},
}
// Read the whole file.
+4 -2
View File
@@ -58,9 +58,10 @@ const (
func testReader(t *testing.T, decoded string, encoded string) {
buf := &bytes.Buffer{}
r := &Reader{
r := &rac.Reader{
ReadSeeker: strings.NewReader(encoded),
CompressedSize: int64(len(encoded)),
CodecReaders: []rac.CodecReader{&CodecReader{}},
}
if _, err := io.Copy(buf, r); err != nil {
t.Fatalf("Copy: %v", err)
@@ -163,9 +164,10 @@ func TestZeroedBytes(t *testing.T) {
}
compressed := cBuf.Bytes()
r := &Reader{
r := &rac.Reader{
ReadSeeker: bytes.NewReader(compressed),
CompressedSize: int64(len(compressed)),
CodecReaders: []rac.CodecReader{&CodecReader{}},
}
for i := 0; i <= len(original); i++ {
want := original[i:]
+26 -101
View File
@@ -24,8 +24,6 @@ import (
)
var (
errInvalidCodec = errors.New("raczlib: invalid Codec (expected rac.CodecZlib)")
errInvalidReadSeeker = errors.New("raczlib: invalid ReadSeeker")
errInvalidDictionary = errors.New("raczlib: invalid dictionary")
)
@@ -52,39 +50,8 @@ func readAt(r io.ReadSeeker, p []byte, offset int64) error {
return err
}
// Reader reads a RAC file.
//
// Do not modify its exported fields after calling any of its methods.
type Reader struct {
// ReadSeeker is where the RAC-encoded data is read from.
//
// It may also implement io.ReaderAt, in which case its ReadAt method will
// be preferred over combining Read and Seek, as the former is presumably
// more efficient. This is optional: io.ReaderAt is a stronger contract
// than io.ReadSeeker, as multiple concurrent ReadAt calls must not
// interfere with each other.
//
// For example, this type itself only implements io.ReadSeeker, not
// io.ReaderAt, as it is not safe for concurrent use.
//
// Nil is an invalid value.
ReadSeeker io.ReadSeeker
// CompressedSize is the size of the RAC file.
//
// Zero is an invalid value, as an empty file is not a valid RAC file.
CompressedSize int64
// err is the first error encountered. It is sticky: once a non-nil error
// occurs, all public methods will return that error.
err error
// reader is the low-level (Codec-agnostic) RAC reader.
racReader rac.Reader
// buf is a scratch buffer.
buf [2]byte
// CodecReader specializes a rac.Reader to decode Zlib-compressed chunks.
type CodecReader struct {
// cachedZlibReader lets us re-use the memory allocated for a zlib reader,
// when decompressing multiple chunks.
cachedZlibReader zlib.Resetter
@@ -92,60 +59,26 @@ type Reader struct {
// These fields contain the most recently used shared dictionary.
cachedDictionary []byte
cachedDictionaryCRange rac.Range
// buf is a scratch buffer.
buf [2]byte
}
func (r *Reader) initialize() error {
if r.err != nil {
return r.err
}
if r.racReader.ReadSeeker != nil {
// We're already initialized.
return nil
}
if r.ReadSeeker == nil {
r.err = errInvalidReadSeeker
return r.err
}
r.racReader.ReadSeeker = r.ReadSeeker
r.racReader.CompressedSize = r.CompressedSize
r.racReader.MakeDecompressor = r.makeDecompressor
r.racReader.MakeReaderContext = r.makeReaderContext
return nil
// Accepts implements rac.CodecReader.
func (r *CodecReader) Accepts(c rac.Codec) bool {
return c == rac.CodecZlib
}
// Read implements io.Reader.
func (r *Reader) Read(p []byte) (int, error) {
if err := r.initialize(); err != nil {
return 0, err
}
n, err := r.racReader.Read(p)
if (err != nil) && (err != io.EOF) {
r.err = err
}
return n, err
}
// MakeDecompressor implements rac.CodecReader.
func (r *CodecReader) MakeDecompressor(
compressed io.Reader, rctx rac.ReaderContext) (io.Reader, error) {
// Seek implements io.Seeker.
func (r *Reader) Seek(offset int64, whence int) (int64, error) {
if err := r.initialize(); err != nil {
return 0, err
}
n, err := r.racReader.Seek(offset, whence)
if (err != nil) && (err != io.EOF) {
r.err = err
}
return n, err
}
func (r *Reader) makeDecompressor(compressed io.Reader, rctx rac.ReaderContext) (io.Reader, error) {
if r.cachedZlibReader != nil {
if err := r.cachedZlibReader.Reset(compressed, rctx.Secondary); err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
r.err = err
return nil, r.err
return nil, err
}
return r.cachedZlibReader.(io.Reader), nil
}
@@ -155,24 +88,21 @@ func (r *Reader) makeDecompressor(compressed io.Reader, rctx rac.ReaderContext)
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
r.err = err
return nil, r.err
return nil, err
}
r.cachedZlibReader = zlibReader.(zlib.Resetter)
return zlibReader, nil
}
func (r *Reader) makeReaderContext(chunk rac.Chunk) (rac.ReaderContext, error) {
// MakeReaderContext implements rac.CodecReader.
func (r *CodecReader) MakeReaderContext(
rs io.ReadSeeker, compressedSize int64, chunk rac.Chunk) (rac.ReaderContext, error) {
// For a description of the RAC+Zlib secondary-data format, see
// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib
if chunk.Codec != rac.CodecZlib {
r.err = errInvalidCodec
return rac.ReaderContext{}, r.err
}
if !chunk.CTertiary.Empty() {
r.err = errInvalidDictionary
return rac.ReaderContext{}, r.err
return rac.ReaderContext{}, errInvalidDictionary
}
if chunk.CSecondary.Empty() {
return rac.ReaderContext{}, nil
@@ -185,23 +115,20 @@ func (r *Reader) makeReaderContext(chunk rac.Chunk) (rac.ReaderContext, error) {
}
// Check the cRange size and the tTag.
if (cRange.Size() < 6) || (cRange[1] > r.CompressedSize) || (chunk.TTag != 0xFF) {
r.err = errInvalidDictionary
return rac.ReaderContext{}, r.err
if (cRange.Size() < 6) || (cRange[1] > compressedSize) || (chunk.TTag != 0xFF) {
return rac.ReaderContext{}, errInvalidDictionary
}
// Read the dictionary size.
if err := readAt(r.ReadSeeker, r.buf[:2], cRange[0]); err != nil {
r.err = err
return rac.ReaderContext{}, r.err
if err := readAt(rs, r.buf[:2], cRange[0]); err != nil {
return rac.ReaderContext{}, errInvalidDictionary
}
dictSize := int64(r.buf[0]) | (int64(r.buf[1]) << 8)
// Check the size. The +6 is for the 2 byte prefix (dictionary size) and
// the 4 byte suffix (checksum).
if (dictSize + 6) > cRange.Size() {
r.err = errInvalidDictionary
return rac.ReaderContext{}, r.err
return rac.ReaderContext{}, errInvalidDictionary
}
// Allocate or re-use the cachedDictionary buffer.
@@ -213,16 +140,14 @@ func (r *Reader) makeReaderContext(chunk rac.Chunk) (rac.ReaderContext, error) {
}
// Read the dictionary and checksum.
if err := readAt(r.ReadSeeker, buffer, cRange[0]+2); err != nil {
r.err = err
return rac.ReaderContext{}, r.err
if err := readAt(rs, buffer, cRange[0]+2); err != nil {
return rac.ReaderContext{}, errInvalidDictionary
}
// Verify the checksum.
dict, checksum := buffer[:dictSize], buffer[dictSize:]
if u32BE(checksum) != adler32.Checksum(dict) {
r.err = errInvalidDictionary
return rac.ReaderContext{}, r.err
return rac.ReaderContext{}, errInvalidDictionary
}
// Save to the MRU cache and return.