From 9394364a47622dd32be239cb13953932ab11b913 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sat, 24 Aug 2019 07:44:07 +1000 Subject: [PATCH] Have rac favor io.ReaderAt over io.ReadSeeker --- lib/rac/chunk_reader.go | 100 ++++++++++++++++++++---------------- lib/rac/reader.go | 48 ++++++++--------- lib/raczlib/raczlib.go | 31 +++-------- lib/raczlib/raczlib_test.go | 50 ++++++++++++++++++ lib/readerat/readerat.go | 95 ++++++++++++++++++++++++++++++++++ 5 files changed, 228 insertions(+), 96 deletions(-) create mode 100644 lib/readerat/readerat.go diff --git a/lib/rac/chunk_reader.go b/lib/rac/chunk_reader.go index b19dd87a..a3116a8e 100644 --- a/lib/rac/chunk_reader.go +++ b/lib/rac/chunk_reader.go @@ -17,6 +17,8 @@ package rac import ( "hash/crc32" "io" + + "github.com/google/wuffs/lib/readerat" ) func u48LE(b []byte) int64 { @@ -26,24 +28,6 @@ func u48LE(b []byte) int64 { return int64(u & 0xFFFFFFFFFFFF) } -// readAt calls ReadAt if it is available, otherwise it falls back to calling -// Seek and then ReadFull. Calling ReadAt is presumably slightly more -// efficient, e.g. one syscall instead of two. -func readAt(r io.ReadSeeker, p []byte, offset int64) error { - if a, ok := r.(io.ReaderAt); ok { - n, err := a.ReadAt(p, offset) - if (n == len(p)) && (err == io.EOF) { - err = nil - } - return err - } - if _, err := r.Seek(offset, io.SeekStart); err != nil { - return err - } - _, err := io.ReadFull(r, p) - return err -} - // bitwiseSubset returns whether every 1-bit in inner is also set in outer. func bitwiseSubset(outer Codec, inner Codec) bool { return outer == (outer | inner) @@ -237,14 +221,18 @@ func (b *rNode) valid() bool { type ChunkReader struct { // ReadSeeker is where the RAC-encoded data is read from. // - // It may also implement io.ReaderAt, in which case its ReadAt method will - // be preferred over combining Read and Seek, as the former is presumably - // more efficient. This is optional: io.ReaderAt is a stronger contract - // than io.ReadSeeker, as multiple concurrent ReadAt calls must not - // interfere with each other. + // It may optionally implement io.ReaderAt, in which case its ReadAt method + // will be preferred and its Read and Seek methods will never be called, + // other than Seek'ing to the end of the data to determine the compressed + // size. The ReadAt method is safe to use concurrently, so that multiple + // rac.Reader's can concurrently use the same source provided that the + // source (this field, nominally an io.ReadSeeker) implements io.ReaderAt. // - // For example, this type itself only implements io.ReadSeeker, not - // io.ReaderAt, as it is not safe for concurrent use. + // In particular, if the source is a bytes.Reader or an os.File, multiple + // rac.ChunkReader's can work in parallel, which can speed up decoding. + // + // This type itself only implements io.ReadSeeker, not io.ReaderAt, as it + // is not safe for concurrent use. // // Nil is an invalid value. ReadSeeker io.ReadSeeker @@ -259,6 +247,11 @@ type ChunkReader struct { // seekPosition. needToResolveSeekPosition bool + // readSeeker is either the same as the ReadSeeker field value, or it is + // that field value wrapped by a readerat.ReadSeeker to be safe to use + // concurrently. + readSeeker io.ReadSeeker + // err is the first error encountered. It is sticky: once a non-nil error // occurs, all public methods will return that error. err error @@ -311,23 +304,26 @@ func (r *ChunkReader) initialize() error { return err } - if r.compressedSize == 0 { - if size, err := r.ReadSeeker.Seek(0, io.SeekEnd); err != nil { - r.err = err - return r.err - } else { - r.compressedSize = size - } - if _, err := r.ReadSeeker.Seek(0, io.SeekStart); err != nil { - r.err = err - return r.err - } + if size, err := r.ReadSeeker.Seek(0, io.SeekEnd); err != nil { + r.err = err + return r.err + } else { + r.compressedSize = size } if r.compressedSize < 32 { r.err = errInvalidCompressedSize return r.err } + if ra, ok := r.ReadSeeker.(io.ReaderAt); ok { + r.readSeeker = &readerat.ReadSeeker{ + ReaderAt: ra, + Size: r.compressedSize, + } + } else { + r.readSeeker = r.ReadSeeker + } + if err := r.findRootNode(); err != nil { return err } @@ -340,9 +336,13 @@ func (r *ChunkReader) initialize() error { func (r *ChunkReader) findRootNode() error { // Look at the start of the compressed file. - if err := readAt(r.ReadSeeker, r.currNode[:4], 0); err != nil { + if _, err := r.readSeeker.Seek(0, io.SeekStart); err != nil { r.err = err - return r.err + return err + } + if _, err := io.ReadFull(r.readSeeker, r.currNode[:4]); err != nil { + r.err = err + return err } if (r.currNode[0] != magic[0]) || (r.currNode[1] != magic[1]) || @@ -357,9 +357,13 @@ func (r *ChunkReader) findRootNode() error { } // Look at the end of the compressed file. - if err := readAt(r.ReadSeeker, r.currNode[:1], r.compressedSize-1); err != nil { + if _, err := r.readSeeker.Seek(r.compressedSize-1, io.SeekStart); err != nil { r.err = err - return r.err + return err + } + if _, err := io.ReadFull(r.readSeeker, r.currNode[:1]); err != nil { + r.err = err + return err } if found, err := r.tryRootNode(r.currNode[0], true); err != nil { return err @@ -407,9 +411,13 @@ func (r *ChunkReader) load(cOffset int64, arity uint8) error { return r.err } size := nodeSize(arity) - if err := readAt(r.ReadSeeker, r.currNode[:size], cOffset); err != nil { + if _, err := r.readSeeker.Seek(cOffset, io.SeekStart); err != nil { r.err = err - return r.err + return err + } + if _, err := io.ReadFull(r.readSeeker, r.currNode[:size]); err != nil { + r.err = err + return err } return nil } @@ -422,9 +430,13 @@ func (r *ChunkReader) loadAndValidate(cOffset int64, r.err = errInvalidIndexNode return r.err } - if err := readAt(r.ReadSeeker, r.currNode[:4], cOffset); err != nil { + if _, err := r.readSeeker.Seek(cOffset, io.SeekStart); err != nil { r.err = err - return r.err + return err + } + if _, err := io.ReadFull(r.readSeeker, r.currNode[:4]); err != nil { + r.err = err + return err } arity := r.currNode[3] if arity == 0 { diff --git a/lib/rac/reader.go b/lib/rac/reader.go index c9ef3b9d..9723e46f 100644 --- a/lib/rac/reader.go +++ b/lib/rac/reader.go @@ -57,7 +57,7 @@ type CodecReader interface { MakeDecompressor(r io.Reader, rctx ReaderContext) (io.Reader, error) // MakeReaderContext returns the Codec-specific ReaderContext for a chunk. - MakeReaderContext(r io.ReadSeeker, compressedSize int64, c Chunk) (ReaderContext, error) + MakeReaderContext(r io.ReadSeeker, c Chunk) (ReaderContext, error) } // Reader reads a RAC file. @@ -68,14 +68,18 @@ type CodecReader interface { type Reader struct { // ReadSeeker is where the RAC-encoded data is read from. // - // It may also implement io.ReaderAt, in which case its ReadAt method will - // be preferred over combining Read and Seek, as the former is presumably - // more efficient. This is optional: io.ReaderAt is a stronger contract - // than io.ReadSeeker, as multiple concurrent ReadAt calls must not - // interfere with each other. + // It may optionally implement io.ReaderAt, in which case its ReadAt method + // will be preferred and its Read and Seek methods will never be called, + // other than Seek'ing to the end of the data to determine the compressed + // size. The ReadAt method is safe to use concurrently, so that multiple + // rac.Reader's can concurrently use the same source provided that the + // source (this field, nominally an io.ReadSeeker) implements io.ReaderAt. // - // For example, this type itself only implements io.ReadSeeker, not - // io.ReaderAt, as it is not safe for concurrent use. + // In particular, if the source is a bytes.Reader or an os.File, multiple + // rac.Reader's can work in parallel, which can speed up decoding. + // + // This type itself only implements io.ReadSeeker, not io.ReaderAt, as it + // is not safe for concurrent use. // // Nil is an invalid value. ReadSeeker io.ReadSeeker @@ -152,27 +156,15 @@ func (r *Reader) initialize() error { if r.err != nil { return r.err } - if r.chunkReader.ReadSeeker != nil { - // We're already initialized. + if r.chunkReader.initialized { return nil } - if r.ReadSeeker == nil { - r.err = errInvalidReadSeeker - return r.err - } - if size, err := r.ReadSeeker.Seek(0, io.SeekEnd); err != nil { - r.err = err - return r.err - } else { - r.chunkReader.compressedSize = size - } - if _, err := r.ReadSeeker.Seek(0, io.SeekStart); err != nil { - r.err = err - return r.err - } - r.chunkReader.ReadSeeker = r.ReadSeeker - r.currChunk.R = r.ReadSeeker + if err := r.chunkReader.initialize(); err != nil { + r.err = err + return r.err + } + r.currChunk.R = r.chunkReader.readSeeker return nil } @@ -349,7 +341,7 @@ func (r *Reader) nextChunk() error { return r.err } - rctx, err := codecReader.MakeReaderContext(r.ReadSeeker, r.chunkReader.compressedSize, chunk) + rctx, err := codecReader.MakeReaderContext(r.chunkReader.readSeeker, chunk) if err != nil { if err == io.EOF { err = io.ErrUnexpectedEOF @@ -358,7 +350,7 @@ func (r *Reader) nextChunk() error { return r.err } - if _, err := r.ReadSeeker.Seek(chunk.CPrimary[0], io.SeekStart); err != nil { + if _, err := r.chunkReader.readSeeker.Seek(chunk.CPrimary[0], io.SeekStart); err != nil { if err == io.EOF { err = io.ErrUnexpectedEOF } diff --git a/lib/raczlib/raczlib.go b/lib/raczlib/raczlib.go index 309512aa..ead51f30 100644 --- a/lib/raczlib/raczlib.go +++ b/lib/raczlib/raczlib.go @@ -42,24 +42,6 @@ func u32BE(b []byte) uint32 { return (uint32(b[0]) << 24) | (uint32(b[1]) << 16) | (uint32(b[2]) << 8) | (uint32(b[3])) } -// readAt calls ReadAt if it is available, otherwise it falls back to calling -// Seek and then ReadFull. Calling ReadAt is presumably slightly more -// efficient, e.g. one syscall instead of two. -func readAt(r io.ReadSeeker, p []byte, offset int64) error { - if a, ok := r.(io.ReaderAt); ok { - n, err := a.ReadAt(p, offset) - if (n == len(p)) && (err == io.EOF) { - err = nil - } - return err - } - if _, err := r.Seek(offset, io.SeekStart); err != nil { - return err - } - _, err := io.ReadFull(r, p) - return err -} - // suffix32K returns the last 32 KiB of b, or if b is shorter than that, it // returns just b. // @@ -111,9 +93,7 @@ func (r *CodecReader) MakeDecompressor( } // MakeReaderContext implements rac.CodecReader. -func (r *CodecReader) MakeReaderContext( - rs io.ReadSeeker, compressedSize int64, chunk rac.Chunk) (rac.ReaderContext, error) { - +func (r *CodecReader) MakeReaderContext(rs io.ReadSeeker, chunk rac.Chunk) (rac.ReaderContext, error) { // For a description of the RAC+Zlib secondary-data format, see // https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib @@ -131,12 +111,15 @@ func (r *CodecReader) MakeReaderContext( } // Check the cRange size and the tTag. - if (cRange.Size() < 6) || (cRange[1] > compressedSize) || (chunk.TTag != 0xFF) { + if (cRange.Size() < 6) || (chunk.TTag != 0xFF) { return rac.ReaderContext{}, errInvalidDictionary } // Read the dictionary size. - if err := readAt(rs, r.buf[:2], cRange[0]); err != nil { + if _, err := rs.Seek(cRange[0], io.SeekStart); err != nil { + return rac.ReaderContext{}, errInvalidDictionary + } + if _, err := io.ReadFull(rs, r.buf[:2]); err != nil { return rac.ReaderContext{}, errInvalidDictionary } dictSize := int64(r.buf[0]) | (int64(r.buf[1]) << 8) @@ -158,7 +141,7 @@ func (r *CodecReader) MakeReaderContext( } // Read the dictionary and checksum. - if err := readAt(rs, buffer, cRange[0]+2); err != nil { + if _, err := io.ReadFull(rs, buffer); err != nil { return rac.ReaderContext{}, errInvalidDictionary } diff --git a/lib/raczlib/raczlib_test.go b/lib/raczlib/raczlib_test.go index e62b21b4..45f5b343 100644 --- a/lib/raczlib/raczlib_test.go +++ b/lib/raczlib/raczlib_test.go @@ -20,6 +20,7 @@ import ( "fmt" "hash/crc32" "io" + "strings" "testing" "github.com/google/wuffs/lib/rac" @@ -273,3 +274,52 @@ func TestSharedDictionary(t *testing.T) { t.Fatalf("ratio: got %dx, want at least 4x", ratio) } } + +// rsSansReadAt wraps a strings.Reader to have only Read and Seek methods. +type rsSansReadAt struct { + r *strings.Reader +} + +func (r *rsSansReadAt) Read(p []byte) (int, error) { return r.r.Read(p) } +func (r *rsSansReadAt) Seek(o int64, w int) (int64, error) { return r.r.Seek(o, w) } + +// rsWithReadAt wraps a strings.Reader to have Read, Seek and ReadAt methods. +// Technically, it satisfies the io.ReadSeeker interface, but calling Read or +// Seek will panic, other than Seek'ing to the end of the data. +type rsWithReadAt struct { + r *strings.Reader +} + +func (r *rsWithReadAt) Read(p []byte) (int, error) { panic("unimplemented") } +func (r *rsWithReadAt) Seek(o int64, w int) (int64, error) { + if (o == 0) && (w == io.SeekEnd) { + return r.r.Seek(o, w) + } + panic("unimplemented") +} +func (r *rsWithReadAt) ReadAt(p []byte, o int64) (int, error) { return r.r.ReadAt(p, o) } + +// testReadSeeker tests that decoding from rs works, regardless of whether rs +// implements the optional ReadAt method. If rs does implement io.ReaderAt then +// its Read method should never be called. +func testReadSeeker(t *testing.T, rs io.ReadSeeker) { + buf := &bytes.Buffer{} + r := &rac.Reader{ + ReadSeeker: rs, + CodecReaders: []rac.CodecReader{&CodecReader{}}, + } + if _, err := io.Copy(buf, r); err != nil { + t.Fatalf("io.Copy: %v", err) + } + if got, want := buf.String(), decodedSheep; got != want { + t.Fatalf("got:\n%s\nwant:\n%s", got, want) + } +} + +func TestReadSeekerSansReadAt(t *testing.T) { + testReadSeeker(t, &rsSansReadAt{strings.NewReader(encodedSheep)}) +} + +func TestReadSeekerWithReadAt(t *testing.T) { + testReadSeeker(t, &rsWithReadAt{strings.NewReader(encodedSheep)}) +} diff --git a/lib/readerat/readerat.go b/lib/readerat/readerat.go new file mode 100644 index 00000000..ffd96cc7 --- /dev/null +++ b/lib/readerat/readerat.go @@ -0,0 +1,95 @@ +// Copyright 2019 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ---------------- + +// Package readerat provides utilities for the io.ReaderAt type. +package readerat + +import ( + "errors" + "io" +) + +var ( + errInvalidSize = errors.New("readerat: invalid size") + errSeekToInvalidWhence = errors.New("readerat: seek to invalid whence") + errSeekToNegativePosition = errors.New("readerat: seek to negative position") +) + +// ReadSeeker is an io.ReadSeeker implementation based on an io.ReaderAt (and +// an int64 size). +// +// For example, an os.File is both an io.ReaderAt and an io.ReadSeeker, but its +// io.ReadSeeker methods are not safe to use concurrently. In comparison, +// multiple readerat.ReadSeeker values (using the same os.File as their +// io.ReaderAt) are safe to use concurrently. Each can Read and Seek +// independently. +// +// A single readerat.ReadSeeker is not safe to use concurrently. +// +// Do not modify its exported fields after calling any of its methods. +type ReadSeeker struct { + ReaderAt io.ReaderAt + Size int64 + offset int64 +} + +// Read implements io.Reader. +func (r *ReadSeeker) Read(p []byte) (int, error) { + if r.Size < 0 { + return 0, errInvalidSize + } + if r.Size <= r.offset { + return 0, io.EOF + } + length := r.Size - r.offset + if int64(len(p)) > length { + p = p[:length] + } + if len(p) == 0 { + return 0, nil + } + + actual, err := r.ReaderAt.ReadAt(p, r.offset) + r.offset += int64(actual) + if (err == nil) && (r.offset == r.Size) { + err = io.EOF + } + return actual, err +} + +// Seek implements io.Seeker. +func (r *ReadSeeker) Seek(offset int64, whence int) (int64, error) { + if r.Size < 0 { + return 0, errInvalidSize + } + + switch whence { + case io.SeekStart: + // No-op. + case io.SeekCurrent: + offset += r.offset + case io.SeekEnd: + offset += r.Size + default: + return 0, errSeekToInvalidWhence + } + + if r.offset < 0 { + return 0, errSeekToNegativePosition + } + r.offset = offset + return r.offset, nil +}