Have rac favor io.ReaderAt over io.ReadSeeker

This commit is contained in:
Nigel Tao committed 2019-08-24 07:56:34 +10:00
1 parent 62211056f8
commit 9394364a47
5 files changed
+228 -96

No files matched your search

+56 -44
View File
@@ -17,6 +17,8 @@ package rac
import (
"hash/crc32"
"io"
"github.com/google/wuffs/lib/readerat"
)
func u48LE(b []byte) int64 {
@@ -26,24 +28,6 @@ func u48LE(b []byte) int64 {
return int64(u & 0xFFFFFFFFFFFF)
}
// readAt calls ReadAt if it is available, otherwise it falls back to calling
// Seek and then ReadFull. Calling ReadAt is presumably slightly more
// efficient, e.g. one syscall instead of two.
func readAt(r io.ReadSeeker, p []byte, offset int64) error {
if a, ok := r.(io.ReaderAt); ok {
n, err := a.ReadAt(p, offset)
if (n == len(p)) && (err == io.EOF) {
err = nil
}
return err
}
if _, err := r.Seek(offset, io.SeekStart); err != nil {
return err
}
_, err := io.ReadFull(r, p)
return err
}
// bitwiseSubset returns whether every 1-bit in inner is also set in outer.
func bitwiseSubset(outer Codec, inner Codec) bool {
return outer == (outer | inner)
@@ -237,14 +221,18 @@ func (b *rNode) valid() bool {
type ChunkReader struct {
// ReadSeeker is where the RAC-encoded data is read from.
//
// It may also implement io.ReaderAt, in which case its ReadAt method will
// be preferred over combining Read and Seek, as the former is presumably
// more efficient. This is optional: io.ReaderAt is a stronger contract
// than io.ReadSeeker, as multiple concurrent ReadAt calls must not
// interfere with each other.
// It may optionally implement io.ReaderAt, in which case its ReadAt method
// will be preferred and its Read and Seek methods will never be called,
// other than Seek'ing to the end of the data to determine the compressed
// size. The ReadAt method is safe to use concurrently, so that multiple
// rac.Reader's can concurrently use the same source provided that the
// source (this field, nominally an io.ReadSeeker) implements io.ReaderAt.
//
// For example, this type itself only implements io.ReadSeeker, not
// io.ReaderAt, as it is not safe for concurrent use.
// In particular, if the source is a bytes.Reader or an os.File, multiple
// rac.ChunkReader's can work in parallel, which can speed up decoding.
//
// This type itself only implements io.ReadSeeker, not io.ReaderAt, as it
// is not safe for concurrent use.
//
// Nil is an invalid value.
ReadSeeker io.ReadSeeker
@@ -259,6 +247,11 @@ type ChunkReader struct {
// seekPosition.
needToResolveSeekPosition bool
// readSeeker is either the same as the ReadSeeker field value, or it is
// that field value wrapped by a readerat.ReadSeeker to be safe to use
// concurrently.
readSeeker io.ReadSeeker
// err is the first error encountered. It is sticky: once a non-nil error
// occurs, all public methods will return that error.
err error
@@ -311,23 +304,26 @@ func (r *ChunkReader) initialize() error {
return err
}
if r.compressedSize == 0 {
if size, err := r.ReadSeeker.Seek(0, io.SeekEnd); err != nil {
r.err = err
return r.err
} else {
r.compressedSize = size
}
if _, err := r.ReadSeeker.Seek(0, io.SeekStart); err != nil {
r.err = err
return r.err
}
if size, err := r.ReadSeeker.Seek(0, io.SeekEnd); err != nil {
r.err = err
return r.err
} else {
r.compressedSize = size
}
if r.compressedSize < 32 {
r.err = errInvalidCompressedSize
return r.err
}
if ra, ok := r.ReadSeeker.(io.ReaderAt); ok {
r.readSeeker = &readerat.ReadSeeker{
ReaderAt: ra,
Size: r.compressedSize,
}
} else {
r.readSeeker = r.ReadSeeker
}
if err := r.findRootNode(); err != nil {
return err
}
@@ -340,9 +336,13 @@ func (r *ChunkReader) initialize() error {
func (r *ChunkReader) findRootNode() error {
// Look at the start of the compressed file.
if err := readAt(r.ReadSeeker, r.currNode[:4], 0); err != nil {
if _, err := r.readSeeker.Seek(0, io.SeekStart); err != nil {
r.err = err
return r.err
return err
}
if _, err := io.ReadFull(r.readSeeker, r.currNode[:4]); err != nil {
r.err = err
return err
}
if (r.currNode[0] != magic[0]) ||
(r.currNode[1] != magic[1]) ||
@@ -357,9 +357,13 @@ func (r *ChunkReader) findRootNode() error {
}
// Look at the end of the compressed file.
if err := readAt(r.ReadSeeker, r.currNode[:1], r.compressedSize-1); err != nil {
if _, err := r.readSeeker.Seek(r.compressedSize-1, io.SeekStart); err != nil {
r.err = err
return r.err
return err
}
if _, err := io.ReadFull(r.readSeeker, r.currNode[:1]); err != nil {
r.err = err
return err
}
if found, err := r.tryRootNode(r.currNode[0], true); err != nil {
return err
@@ -407,9 +411,13 @@ func (r *ChunkReader) load(cOffset int64, arity uint8) error {
return r.err
}
size := nodeSize(arity)
if err := readAt(r.ReadSeeker, r.currNode[:size], cOffset); err != nil {
if _, err := r.readSeeker.Seek(cOffset, io.SeekStart); err != nil {
r.err = err
return r.err
return err
}
if _, err := io.ReadFull(r.readSeeker, r.currNode[:size]); err != nil {
r.err = err
return err
}
return nil
}
@@ -422,9 +430,13 @@ func (r *ChunkReader) loadAndValidate(cOffset int64,
r.err = errInvalidIndexNode
return r.err
}
if err := readAt(r.ReadSeeker, r.currNode[:4], cOffset); err != nil {
if _, err := r.readSeeker.Seek(cOffset, io.SeekStart); err != nil {
r.err = err
return r.err
return err
}
if _, err := io.ReadFull(r.readSeeker, r.currNode[:4]); err != nil {
r.err = err
return err
}
arity := r.currNode[3]
if arity == 0 {
+20 -28
View File
@@ -57,7 +57,7 @@ type CodecReader interface {
MakeDecompressor(r io.Reader, rctx ReaderContext) (io.Reader, error)
// MakeReaderContext returns the Codec-specific ReaderContext for a chunk.
MakeReaderContext(r io.ReadSeeker, compressedSize int64, c Chunk) (ReaderContext, error)
MakeReaderContext(r io.ReadSeeker, c Chunk) (ReaderContext, error)
}
// Reader reads a RAC file.
@@ -68,14 +68,18 @@ type CodecReader interface {
type Reader struct {
// ReadSeeker is where the RAC-encoded data is read from.
//
// It may also implement io.ReaderAt, in which case its ReadAt method will
// be preferred over combining Read and Seek, as the former is presumably
// more efficient. This is optional: io.ReaderAt is a stronger contract
// than io.ReadSeeker, as multiple concurrent ReadAt calls must not
// interfere with each other.
// It may optionally implement io.ReaderAt, in which case its ReadAt method
// will be preferred and its Read and Seek methods will never be called,
// other than Seek'ing to the end of the data to determine the compressed
// size. The ReadAt method is safe to use concurrently, so that multiple
// rac.Reader's can concurrently use the same source provided that the
// source (this field, nominally an io.ReadSeeker) implements io.ReaderAt.
//
// For example, this type itself only implements io.ReadSeeker, not
// io.ReaderAt, as it is not safe for concurrent use.
// In particular, if the source is a bytes.Reader or an os.File, multiple
// rac.Reader's can work in parallel, which can speed up decoding.
//
// This type itself only implements io.ReadSeeker, not io.ReaderAt, as it
// is not safe for concurrent use.
//
// Nil is an invalid value.
ReadSeeker io.ReadSeeker
@@ -152,27 +156,15 @@ func (r *Reader) initialize() error {
if r.err != nil {
return r.err
}
if r.chunkReader.ReadSeeker != nil {
// We're already initialized.
if r.chunkReader.initialized {
return nil
}
if r.ReadSeeker == nil {
r.err = errInvalidReadSeeker
return r.err
}
if size, err := r.ReadSeeker.Seek(0, io.SeekEnd); err != nil {
r.err = err
return r.err
} else {
r.chunkReader.compressedSize = size
}
if _, err := r.ReadSeeker.Seek(0, io.SeekStart); err != nil {
r.err = err
return r.err
}
r.chunkReader.ReadSeeker = r.ReadSeeker
r.currChunk.R = r.ReadSeeker
if err := r.chunkReader.initialize(); err != nil {
r.err = err
return r.err
}
r.currChunk.R = r.chunkReader.readSeeker
return nil
}
@@ -349,7 +341,7 @@ func (r *Reader) nextChunk() error {
return r.err
}
rctx, err := codecReader.MakeReaderContext(r.ReadSeeker, r.chunkReader.compressedSize, chunk)
rctx, err := codecReader.MakeReaderContext(r.chunkReader.readSeeker, chunk)
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
@@ -358,7 +350,7 @@ func (r *Reader) nextChunk() error {
return r.err
}
if _, err := r.ReadSeeker.Seek(chunk.CPrimary[0], io.SeekStart); err != nil {
if _, err := r.chunkReader.readSeeker.Seek(chunk.CPrimary[0], io.SeekStart); err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
+7 -24
View File
@@ -42,24 +42,6 @@ func u32BE(b []byte) uint32 {
return (uint32(b[0]) << 24) | (uint32(b[1]) << 16) | (uint32(b[2]) << 8) | (uint32(b[3]))
}
// readAt calls ReadAt if it is available, otherwise it falls back to calling
// Seek and then ReadFull. Calling ReadAt is presumably slightly more
// efficient, e.g. one syscall instead of two.
func readAt(r io.ReadSeeker, p []byte, offset int64) error {
if a, ok := r.(io.ReaderAt); ok {
n, err := a.ReadAt(p, offset)
if (n == len(p)) && (err == io.EOF) {
err = nil
}
return err
}
if _, err := r.Seek(offset, io.SeekStart); err != nil {
return err
}
_, err := io.ReadFull(r, p)
return err
}
// suffix32K returns the last 32 KiB of b, or if b is shorter than that, it
// returns just b.
//
@@ -111,9 +93,7 @@ func (r *CodecReader) MakeDecompressor(
}
// MakeReaderContext implements rac.CodecReader.
func (r *CodecReader) MakeReaderContext(
rs io.ReadSeeker, compressedSize int64, chunk rac.Chunk) (rac.ReaderContext, error) {
func (r *CodecReader) MakeReaderContext(rs io.ReadSeeker, chunk rac.Chunk) (rac.ReaderContext, error) {
// For a description of the RAC+Zlib secondary-data format, see
// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib
@@ -131,12 +111,15 @@ func (r *CodecReader) MakeReaderContext(
}
// Check the cRange size and the tTag.
if (cRange.Size() < 6) || (cRange[1] > compressedSize) || (chunk.TTag != 0xFF) {
if (cRange.Size() < 6) || (chunk.TTag != 0xFF) {
return rac.ReaderContext{}, errInvalidDictionary
}
// Read the dictionary size.
if err := readAt(rs, r.buf[:2], cRange[0]); err != nil {
if _, err := rs.Seek(cRange[0], io.SeekStart); err != nil {
return rac.ReaderContext{}, errInvalidDictionary
}
if _, err := io.ReadFull(rs, r.buf[:2]); err != nil {
return rac.ReaderContext{}, errInvalidDictionary
}
dictSize := int64(r.buf[0]) | (int64(r.buf[1]) << 8)
@@ -158,7 +141,7 @@ func (r *CodecReader) MakeReaderContext(
}
// Read the dictionary and checksum.
if err := readAt(rs, buffer, cRange[0]+2); err != nil {
if _, err := io.ReadFull(rs, buffer); err != nil {
return rac.ReaderContext{}, errInvalidDictionary
}
+50
View File
@@ -20,6 +20,7 @@ import (
"fmt"
"hash/crc32"
"io"
"strings"
"testing"
"github.com/google/wuffs/lib/rac"
@@ -273,3 +274,52 @@ func TestSharedDictionary(t *testing.T) {
t.Fatalf("ratio: got %dx, want at least 4x", ratio)
}
}
// rsSansReadAt wraps a strings.Reader to have only Read and Seek methods.
type rsSansReadAt struct {
r *strings.Reader
}
func (r *rsSansReadAt) Read(p []byte) (int, error) { return r.r.Read(p) }
func (r *rsSansReadAt) Seek(o int64, w int) (int64, error) { return r.r.Seek(o, w) }
// rsWithReadAt wraps a strings.Reader to have Read, Seek and ReadAt methods.
// Technically, it satisfies the io.ReadSeeker interface, but calling Read or
// Seek will panic, other than Seek'ing to the end of the data.
type rsWithReadAt struct {
r *strings.Reader
}
func (r *rsWithReadAt) Read(p []byte) (int, error) { panic("unimplemented") }
func (r *rsWithReadAt) Seek(o int64, w int) (int64, error) {
if (o == 0) && (w == io.SeekEnd) {
return r.r.Seek(o, w)
}
panic("unimplemented")
}
func (r *rsWithReadAt) ReadAt(p []byte, o int64) (int, error) { return r.r.ReadAt(p, o) }
// testReadSeeker tests that decoding from rs works, regardless of whether rs
// implements the optional ReadAt method. If rs does implement io.ReaderAt then
// its Read method should never be called.
func testReadSeeker(t *testing.T, rs io.ReadSeeker) {
buf := &bytes.Buffer{}
r := &rac.Reader{
ReadSeeker: rs,
CodecReaders: []rac.CodecReader{&CodecReader{}},
}
if _, err := io.Copy(buf, r); err != nil {
t.Fatalf("io.Copy: %v", err)
}
if got, want := buf.String(), decodedSheep; got != want {
t.Fatalf("got:\n%s\nwant:\n%s", got, want)
}
}
func TestReadSeekerSansReadAt(t *testing.T) {
testReadSeeker(t, &rsSansReadAt{strings.NewReader(encodedSheep)})
}
func TestReadSeekerWithReadAt(t *testing.T) {
testReadSeeker(t, &rsWithReadAt{strings.NewReader(encodedSheep)})
}
+95
View File
@@ -0,0 +1,95 @@
// Copyright 2019 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------
// Package readerat provides utilities for the io.ReaderAt type.
package readerat
import (
"errors"
"io"
)
var (
errInvalidSize = errors.New("readerat: invalid size")
errSeekToInvalidWhence = errors.New("readerat: seek to invalid whence")
errSeekToNegativePosition = errors.New("readerat: seek to negative position")
)
// ReadSeeker is an io.ReadSeeker implementation based on an io.ReaderAt (and
// an int64 size).
//
// For example, an os.File is both an io.ReaderAt and an io.ReadSeeker, but its
// io.ReadSeeker methods are not safe to use concurrently. In comparison,
// multiple readerat.ReadSeeker values (using the same os.File as their
// io.ReaderAt) are safe to use concurrently. Each can Read and Seek
// independently.
//
// A single readerat.ReadSeeker is not safe to use concurrently.
//
// Do not modify its exported fields after calling any of its methods.
type ReadSeeker struct {
ReaderAt io.ReaderAt
Size int64
offset int64
}
// Read implements io.Reader.
func (r *ReadSeeker) Read(p []byte) (int, error) {
if r.Size < 0 {
return 0, errInvalidSize
}
if r.Size <= r.offset {
return 0, io.EOF
}
length := r.Size - r.offset
if int64(len(p)) > length {
p = p[:length]
}
if len(p) == 0 {
return 0, nil
}
actual, err := r.ReaderAt.ReadAt(p, r.offset)
r.offset += int64(actual)
if (err == nil) && (r.offset == r.Size) {
err = io.EOF
}
return actual, err
}
// Seek implements io.Seeker.
func (r *ReadSeeker) Seek(offset int64, whence int) (int64, error) {
if r.Size < 0 {
return 0, errInvalidSize
}
switch whence {
case io.SeekStart:
// No-op.
case io.SeekCurrent:
offset += r.offset
case io.SeekEnd:
offset += r.Size
default:
return 0, errSeekToInvalidWhence
}
if r.offset < 0 {
return 0, errSeekToNegativePosition
}
r.offset = offset
return r.offset, nil
}