From cde91019d9f9b44293f847559776668134c6e86a Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sun, 15 Sep 2019 10:16:59 +1000 Subject: [PATCH] Remove the rac.ReaderContext type --- lib/rac/reader.go | 46 +++++--------------------------------- lib/raczlib/make_cgo.go | 6 ++--- lib/raczlib/make_notcgo.go | 8 +++---- lib/raczlib/raczlib.go | 40 +++++++++++++++++++++++---------- lib/raczstd/raczstd.go | 16 +++++++------ 5 files changed, 47 insertions(+), 69 deletions(-) diff --git a/lib/rac/reader.go b/lib/rac/reader.go index 6431527f..503809a9 100644 --- a/lib/rac/reader.go +++ b/lib/rac/reader.go @@ -37,14 +37,6 @@ func (z *zeroesReader) Read(p []byte) (int, error) { return len(p), nil } -// ReaderContext contains the decoded Codec-specific metadata (non-primary -// data) associated with a RAC chunk. -type ReaderContext struct { - Secondary []byte - Tertiary []byte - Extra interface{} -} - // CodecReader specializes a Reader to decode a specific compression codec. // // Instances are not required to support concurrent use. @@ -62,10 +54,7 @@ type CodecReader interface { // // The returned io.Reader may optionally implement the io.Closer interface, // in which case this Reader will call Close when has finished the chunk. - MakeDecompressor(r io.Reader, rctx ReaderContext) (io.Reader, error) - - // MakeReaderContext returns the Codec-specific ReaderContext for a chunk. - MakeReaderContext(r io.ReadSeeker, c Chunk) (ReaderContext, error) + MakeDecompressor(racFile io.ReadSeeker, c Chunk) (io.Reader, error) } // Reader reads a RAC file. @@ -121,7 +110,7 @@ type Reader struct { // - "State A" (both fields are zero): no RAC chunk is loaded. // // - "State B" (decompressor is non-zero, inImplicitZeroes is zero): a RAC - // chunk is loaded, but not fully exhausted: decompressing the zlib + // chunk is loaded, but not fully exhausted: decompressing the e.g. zlib // stream has not hit io.EOF yet. // // - "State C" (decompressor is zero, inImplicitZeroes is non-zero): a RAC @@ -141,11 +130,6 @@ type Reader struct { // closed is whether this Reader is closed. closed bool - // currChunk is an io.Reader for the current chunk, used while in "State - // B". It serves zlib-compressed data, which the (non-nil) decompressor - // turns into decompressed data. - currChunk io.LimitedReader - // pos is the current position, in DSpace. It is the base value when Seek // is called with io.SeekCurrent. pos int64 @@ -195,7 +179,6 @@ func (r *Reader) initialize() error { r.err = err return r.err } - r.currChunk.R = r.chunkReader.readSeeker r.posLimit = r.chunkReader.decompressedSize r.concReader.initialize(r) return nil @@ -268,7 +251,7 @@ func (r *Reader) Read(p []byte) (int, error) { } } -// readExplicitData serves the zlib-compressed data in a chunk. +// readExplicitData serves the compressed data in a chunk. func (r *Reader) readExplicitData(p []byte) (int, error) { // If the chunk started before r.pos, discard the opening bytes of the // chunk's decompressed data. @@ -375,7 +358,6 @@ func (r *Reader) nextChunk() error { } if (chunk.Codec == CodecZeroes) || (chunk.Codec == codecLongZeroes) { - r.currChunk.N = 0 r.dRange = chunk.DRange r.zeroes = zeroesReader(r.dRange.Size()) r.decompressor = &r.zeroes @@ -399,26 +381,7 @@ func (r *Reader) nextChunk() error { return r.err } - rctx, err := codecReader.MakeReaderContext(r.chunkReader.readSeeker, chunk) - if err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - r.err = err - return r.err - } - - if _, err := r.chunkReader.readSeeker.Seek(chunk.CPrimary[0], io.SeekStart); err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - r.err = err - return r.err - } - r.currChunk.N = chunk.CPrimary.Size() - r.dRange = chunk.DRange - - decompressor, err := codecReader.MakeDecompressor(&r.currChunk, rctx) + decompressor, err := codecReader.MakeDecompressor(r.chunkReader.readSeeker, chunk) if err != nil { if err == io.EOF { err = io.ErrUnexpectedEOF @@ -427,6 +390,7 @@ func (r *Reader) nextChunk() error { return r.err } r.decompressor = decompressor + r.dRange = chunk.DRange return nil } diff --git a/lib/raczlib/make_cgo.go b/lib/raczlib/make_cgo.go index 16543b30..e908147e 100644 --- a/lib/raczlib/make_cgo.go +++ b/lib/raczlib/make_cgo.go @@ -20,15 +20,13 @@ import ( "io" "github.com/google/wuffs/lib/cgozlib" - "github.com/google/wuffs/lib/rac" ) -// MakeDecompressor implements rac.CodecReader. -func (r *CodecReader) MakeDecompressor(compressed io.Reader, rctx rac.ReaderContext) (io.Reader, error) { +func (r *CodecReader) makeDecompressor(compressed io.Reader, dict []byte) (io.Reader, error) { if r.cachedReader == nil { r.cachedReader = &cgozlib.Reader{} } - if err := r.cachedReader.Reset(compressed, rctx.Secondary); err != nil { + if err := r.cachedReader.Reset(compressed, dict); err != nil { return nil, err } return r.cachedReader, nil diff --git a/lib/raczlib/make_notcgo.go b/lib/raczlib/make_notcgo.go index b2de0a5e..d4eaa5a3 100644 --- a/lib/raczlib/make_notcgo.go +++ b/lib/raczlib/make_notcgo.go @@ -21,19 +21,17 @@ import ( "io" "github.com/google/wuffs/lib/compression" - "github.com/google/wuffs/lib/rac" ) -// MakeDecompressor implements rac.CodecReader. -func (r *CodecReader) MakeDecompressor(compressed io.Reader, rctx rac.ReaderContext) (io.Reader, error) { +func (r *CodecReader) makeDecompressor(compressed io.Reader, dict []byte) (io.Reader, error) { if r.cachedReader != nil { - if err := r.cachedReader.Reset(compressed, rctx.Secondary); err != nil { + if err := r.cachedReader.Reset(compressed, dict); err != nil { return nil, err } return r.cachedReader, nil } - zlibReader, err := zlib.NewReaderDict(compressed, rctx.Secondary) + zlibReader, err := zlib.NewReaderDict(compressed, dict) if err != nil { return nil, err } diff --git a/lib/raczlib/raczlib.go b/lib/raczlib/raczlib.go index 62ce352f..df85c6b5 100644 --- a/lib/raczlib/raczlib.go +++ b/lib/raczlib/raczlib.go @@ -61,6 +61,9 @@ type CodecReader struct { // decompressing multiple chunks. cachedReader compression.Reader + // lim provides a limited view of a RAC file. + lim io.LimitedReader + // These fields contain the most recently used shared dictionary. cachedDictionary []byte cachedDictionaryCRange rac.Range @@ -84,42 +87,55 @@ func (r *CodecReader) Clone() rac.CodecReader { return &CodecReader{} } -// MakeReaderContext implements rac.CodecReader. -func (r *CodecReader) MakeReaderContext(rs io.ReadSeeker, chunk rac.Chunk) (rac.ReaderContext, error) { +// MakeDecompressor implements rac.CodecReader. +func (r *CodecReader) MakeDecompressor(racFile io.ReadSeeker, chunk rac.Chunk) (io.Reader, error) { + dict, err := r.loadDictionary(racFile, chunk) + if err != nil { + return nil, err + } + if _, err := racFile.Seek(chunk.CPrimary[0], io.SeekStart); err != nil { + return nil, err + } + r.lim.R = racFile + r.lim.N = chunk.CPrimary.Size() + return r.makeDecompressor(&r.lim, dict) +} + +func (r *CodecReader) loadDictionary(rs io.ReadSeeker, chunk rac.Chunk) ([]byte, error) { // For a description of the RAC+Zlib secondary-data format, see // https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib if !chunk.CTertiary.Empty() { - return rac.ReaderContext{}, errInvalidDictionary + return nil, errInvalidDictionary } if chunk.CSecondary.Empty() { - return rac.ReaderContext{}, nil + return nil, nil } cRange := chunk.CSecondary // Load from the MRU cache, if it was loaded from the same cRange. if (cRange == r.cachedDictionaryCRange) && !cRange.Empty() { - return rac.ReaderContext{Secondary: r.cachedDictionary}, nil + return r.cachedDictionary, nil } // Check the cRange size and the tTag. if (cRange.Size() < 6) || (chunk.TTag != 0xFF) { - return rac.ReaderContext{}, errInvalidDictionary + return nil, errInvalidDictionary } // Read the dictionary size. if _, err := rs.Seek(cRange[0], io.SeekStart); err != nil { - return rac.ReaderContext{}, errInvalidDictionary + return nil, errInvalidDictionary } if _, err := io.ReadFull(rs, r.buf[:2]); err != nil { - return rac.ReaderContext{}, errInvalidDictionary + return nil, errInvalidDictionary } dictSize := int64(r.buf[0]) | (int64(r.buf[1]) << 8) // Check the size. The +6 is for the 2 byte prefix (dictionary size) and // the 4 byte suffix (checksum). if (dictSize + 6) > cRange.Size() { - return rac.ReaderContext{}, errInvalidDictionary + return nil, errInvalidDictionary } // Allocate or re-use the cachedDictionary buffer. @@ -134,19 +150,19 @@ func (r *CodecReader) MakeReaderContext(rs io.ReadSeeker, chunk rac.Chunk) (rac. // Read the dictionary and checksum. if _, err := io.ReadFull(rs, buffer); err != nil { - return rac.ReaderContext{}, errInvalidDictionary + return nil, errInvalidDictionary } // Verify the checksum. dict, checksum := buffer[:dictSize], buffer[dictSize:] if u32BE(checksum) != adler32.Checksum(dict) { - return rac.ReaderContext{}, errInvalidDictionary + return nil, errInvalidDictionary } // Save to the MRU cache and return. r.cachedDictionary = dict r.cachedDictionaryCRange = cRange - return rac.ReaderContext{Secondary: dict}, nil + return dict, nil } // CodecWriter specializes a rac.Writer to encode Zlib-compressed chunks. diff --git a/lib/raczstd/raczstd.go b/lib/raczstd/raczstd.go index 28c029d1..e9a2e90e 100644 --- a/lib/raczstd/raczstd.go +++ b/lib/raczstd/raczstd.go @@ -40,6 +40,7 @@ var ( type CodecReader struct { cachedReader compression.Reader recycler cgozstd.ReaderRecycler + lim io.LimitedReader } // Close implements rac.CodecReader. @@ -58,23 +59,24 @@ func (r *CodecReader) Clone() rac.CodecReader { } // MakeDecompressor implements rac.CodecReader. -func (r *CodecReader) MakeDecompressor(compressed io.Reader, rctx rac.ReaderContext) (io.Reader, error) { +func (r *CodecReader) MakeDecompressor(racFile io.ReadSeeker, chunk rac.Chunk) (io.Reader, error) { + if _, err := racFile.Seek(chunk.CPrimary[0], io.SeekStart); err != nil { + return nil, err + } + r.lim.R = racFile + r.lim.N = chunk.CPrimary.Size() + if r.cachedReader == nil { zr := &cgozstd.Reader{} r.cachedReader = zr r.recycler.Bind(zr) } - if err := r.cachedReader.Reset(compressed, rctx.Secondary); err != nil { + if err := r.cachedReader.Reset(&r.lim, nil); err != nil { return nil, err } return r.cachedReader, nil } -// MakeReaderContext implements rac.CodecReader. -func (r *CodecReader) MakeReaderContext(rs io.ReadSeeker, chunk rac.Chunk) (rac.ReaderContext, error) { - return rac.ReaderContext{}, nil -} - // CodecWriter specializes a rac.Writer to encode Zstd-compressed chunks. type CodecWriter struct { compressed bytes.Buffer