Remove the rac.ReaderContext type

This commit is contained in:
Nigel Tao committed 2019-09-15 10:16:59 +10:00
1 parent 65e7b2b6c9
commit cde91019d9
5 files changed
+47 -69

No files matched your search

+5 -41
View File
@@ -37,14 +37,6 @@ func (z *zeroesReader) Read(p []byte) (int, error) {
return len(p), nil
}
// ReaderContext contains the decoded Codec-specific metadata (non-primary
// data) associated with a RAC chunk.
type ReaderContext struct {
Secondary []byte
Tertiary []byte
Extra interface{}
}
// CodecReader specializes a Reader to decode a specific compression codec.
//
// Instances are not required to support concurrent use.
@@ -62,10 +54,7 @@ type CodecReader interface {
//
// The returned io.Reader may optionally implement the io.Closer interface,
// in which case this Reader will call Close when has finished the chunk.
MakeDecompressor(r io.Reader, rctx ReaderContext) (io.Reader, error)
// MakeReaderContext returns the Codec-specific ReaderContext for a chunk.
MakeReaderContext(r io.ReadSeeker, c Chunk) (ReaderContext, error)
MakeDecompressor(racFile io.ReadSeeker, c Chunk) (io.Reader, error)
}
// Reader reads a RAC file.
@@ -121,7 +110,7 @@ type Reader struct {
// - "State A" (both fields are zero): no RAC chunk is loaded.
//
// - "State B" (decompressor is non-zero, inImplicitZeroes is zero): a RAC
// chunk is loaded, but not fully exhausted: decompressing the zlib
// chunk is loaded, but not fully exhausted: decompressing the e.g. zlib
// stream has not hit io.EOF yet.
//
// - "State C" (decompressor is zero, inImplicitZeroes is non-zero): a RAC
@@ -141,11 +130,6 @@ type Reader struct {
// closed is whether this Reader is closed.
closed bool
// currChunk is an io.Reader for the current chunk, used while in "State
// B". It serves zlib-compressed data, which the (non-nil) decompressor
// turns into decompressed data.
currChunk io.LimitedReader
// pos is the current position, in DSpace. It is the base value when Seek
// is called with io.SeekCurrent.
pos int64
@@ -195,7 +179,6 @@ func (r *Reader) initialize() error {
r.err = err
return r.err
}
r.currChunk.R = r.chunkReader.readSeeker
r.posLimit = r.chunkReader.decompressedSize
r.concReader.initialize(r)
return nil
@@ -268,7 +251,7 @@ func (r *Reader) Read(p []byte) (int, error) {
}
}
// readExplicitData serves the zlib-compressed data in a chunk.
// readExplicitData serves the compressed data in a chunk.
func (r *Reader) readExplicitData(p []byte) (int, error) {
// If the chunk started before r.pos, discard the opening bytes of the
// chunk's decompressed data.
@@ -375,7 +358,6 @@ func (r *Reader) nextChunk() error {
}
if (chunk.Codec == CodecZeroes) || (chunk.Codec == codecLongZeroes) {
r.currChunk.N = 0
r.dRange = chunk.DRange
r.zeroes = zeroesReader(r.dRange.Size())
r.decompressor = &r.zeroes
@@ -399,26 +381,7 @@ func (r *Reader) nextChunk() error {
return r.err
}
rctx, err := codecReader.MakeReaderContext(r.chunkReader.readSeeker, chunk)
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
r.err = err
return r.err
}
if _, err := r.chunkReader.readSeeker.Seek(chunk.CPrimary[0], io.SeekStart); err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
r.err = err
return r.err
}
r.currChunk.N = chunk.CPrimary.Size()
r.dRange = chunk.DRange
decompressor, err := codecReader.MakeDecompressor(&r.currChunk, rctx)
decompressor, err := codecReader.MakeDecompressor(r.chunkReader.readSeeker, chunk)
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
@@ -427,6 +390,7 @@ func (r *Reader) nextChunk() error {
return r.err
}
r.decompressor = decompressor
r.dRange = chunk.DRange
return nil
}
+2 -4
View File
@@ -20,15 +20,13 @@ import (
"io"
"github.com/google/wuffs/lib/cgozlib"
"github.com/google/wuffs/lib/rac"
)
// MakeDecompressor implements rac.CodecReader.
func (r *CodecReader) MakeDecompressor(compressed io.Reader, rctx rac.ReaderContext) (io.Reader, error) {
func (r *CodecReader) makeDecompressor(compressed io.Reader, dict []byte) (io.Reader, error) {
if r.cachedReader == nil {
r.cachedReader = &cgozlib.Reader{}
}
if err := r.cachedReader.Reset(compressed, rctx.Secondary); err != nil {
if err := r.cachedReader.Reset(compressed, dict); err != nil {
return nil, err
}
return r.cachedReader, nil
+3 -5
View File
@@ -21,19 +21,17 @@ import (
"io"
"github.com/google/wuffs/lib/compression"
"github.com/google/wuffs/lib/rac"
)
// MakeDecompressor implements rac.CodecReader.
func (r *CodecReader) MakeDecompressor(compressed io.Reader, rctx rac.ReaderContext) (io.Reader, error) {
func (r *CodecReader) makeDecompressor(compressed io.Reader, dict []byte) (io.Reader, error) {
if r.cachedReader != nil {
if err := r.cachedReader.Reset(compressed, rctx.Secondary); err != nil {
if err := r.cachedReader.Reset(compressed, dict); err != nil {
return nil, err
}
return r.cachedReader, nil
}
zlibReader, err := zlib.NewReaderDict(compressed, rctx.Secondary)
zlibReader, err := zlib.NewReaderDict(compressed, dict)
if err != nil {
return nil, err
}
+28 -12
View File
@@ -61,6 +61,9 @@ type CodecReader struct {
// decompressing multiple chunks.
cachedReader compression.Reader
// lim provides a limited view of a RAC file.
lim io.LimitedReader
// These fields contain the most recently used shared dictionary.
cachedDictionary []byte
cachedDictionaryCRange rac.Range
@@ -84,42 +87,55 @@ func (r *CodecReader) Clone() rac.CodecReader {
return &CodecReader{}
}
// MakeReaderContext implements rac.CodecReader.
func (r *CodecReader) MakeReaderContext(rs io.ReadSeeker, chunk rac.Chunk) (rac.ReaderContext, error) {
// MakeDecompressor implements rac.CodecReader.
func (r *CodecReader) MakeDecompressor(racFile io.ReadSeeker, chunk rac.Chunk) (io.Reader, error) {
dict, err := r.loadDictionary(racFile, chunk)
if err != nil {
return nil, err
}
if _, err := racFile.Seek(chunk.CPrimary[0], io.SeekStart); err != nil {
return nil, err
}
r.lim.R = racFile
r.lim.N = chunk.CPrimary.Size()
return r.makeDecompressor(&r.lim, dict)
}
func (r *CodecReader) loadDictionary(rs io.ReadSeeker, chunk rac.Chunk) ([]byte, error) {
// For a description of the RAC+Zlib secondary-data format, see
// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib
if !chunk.CTertiary.Empty() {
return rac.ReaderContext{}, errInvalidDictionary
return nil, errInvalidDictionary
}
if chunk.CSecondary.Empty() {
return rac.ReaderContext{}, nil
return nil, nil
}
cRange := chunk.CSecondary
// Load from the MRU cache, if it was loaded from the same cRange.
if (cRange == r.cachedDictionaryCRange) && !cRange.Empty() {
return rac.ReaderContext{Secondary: r.cachedDictionary}, nil
return r.cachedDictionary, nil
}
// Check the cRange size and the tTag.
if (cRange.Size() < 6) || (chunk.TTag != 0xFF) {
return rac.ReaderContext{}, errInvalidDictionary
return nil, errInvalidDictionary
}
// Read the dictionary size.
if _, err := rs.Seek(cRange[0], io.SeekStart); err != nil {
return rac.ReaderContext{}, errInvalidDictionary
return nil, errInvalidDictionary
}
if _, err := io.ReadFull(rs, r.buf[:2]); err != nil {
return rac.ReaderContext{}, errInvalidDictionary
return nil, errInvalidDictionary
}
dictSize := int64(r.buf[0]) | (int64(r.buf[1]) << 8)
// Check the size. The +6 is for the 2 byte prefix (dictionary size) and
// the 4 byte suffix (checksum).
if (dictSize + 6) > cRange.Size() {
return rac.ReaderContext{}, errInvalidDictionary
return nil, errInvalidDictionary
}
// Allocate or re-use the cachedDictionary buffer.
@@ -134,19 +150,19 @@ func (r *CodecReader) MakeReaderContext(rs io.ReadSeeker, chunk rac.Chunk) (rac.
// Read the dictionary and checksum.
if _, err := io.ReadFull(rs, buffer); err != nil {
return rac.ReaderContext{}, errInvalidDictionary
return nil, errInvalidDictionary
}
// Verify the checksum.
dict, checksum := buffer[:dictSize], buffer[dictSize:]
if u32BE(checksum) != adler32.Checksum(dict) {
return rac.ReaderContext{}, errInvalidDictionary
return nil, errInvalidDictionary
}
// Save to the MRU cache and return.
r.cachedDictionary = dict
r.cachedDictionaryCRange = cRange
return rac.ReaderContext{Secondary: dict}, nil
return dict, nil
}
// CodecWriter specializes a rac.Writer to encode Zlib-compressed chunks.
+9 -7
View File
@@ -40,6 +40,7 @@ var (
type CodecReader struct {
cachedReader compression.Reader
recycler cgozstd.ReaderRecycler
lim io.LimitedReader
}
// Close implements rac.CodecReader.
@@ -58,23 +59,24 @@ func (r *CodecReader) Clone() rac.CodecReader {
}
// MakeDecompressor implements rac.CodecReader.
func (r *CodecReader) MakeDecompressor(compressed io.Reader, rctx rac.ReaderContext) (io.Reader, error) {
func (r *CodecReader) MakeDecompressor(racFile io.ReadSeeker, chunk rac.Chunk) (io.Reader, error) {
if _, err := racFile.Seek(chunk.CPrimary[0], io.SeekStart); err != nil {
return nil, err
}
r.lim.R = racFile
r.lim.N = chunk.CPrimary.Size()
if r.cachedReader == nil {
zr := &cgozstd.Reader{}
r.cachedReader = zr
r.recycler.Bind(zr)
}
if err := r.cachedReader.Reset(compressed, rctx.Secondary); err != nil {
if err := r.cachedReader.Reset(&r.lim, nil); err != nil {
return nil, err
}
return r.cachedReader, nil
}
// MakeReaderContext implements rac.CodecReader.
func (r *CodecReader) MakeReaderContext(rs io.ReadSeeker, chunk rac.Chunk) (rac.ReaderContext, error) {
return rac.ReaderContext{}, nil
}
// CodecWriter specializes a rac.Writer to encode Zstd-compressed chunks.
type CodecWriter struct {
compressed bytes.Buffer