mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 16:32:11 +01:00
Have rac favor io.ReaderAt over io.ReadSeeker
This commit is contained in:
1 parent
62211056f8
commit
9394364a47
5 files changed
+228
-96
No files matched your search
+56
-44
@@ -17,6 +17,8 @@ package rac
|
||||
import (
|
||||
"hash/crc32"
|
||||
"io"
|
||||
|
||||
"github.com/google/wuffs/lib/readerat"
|
||||
)
|
||||
|
||||
func u48LE(b []byte) int64 {
|
||||
@@ -26,24 +28,6 @@ func u48LE(b []byte) int64 {
|
||||
return int64(u & 0xFFFFFFFFFFFF)
|
||||
}
|
||||
|
||||
// readAt calls ReadAt if it is available, otherwise it falls back to calling
|
||||
// Seek and then ReadFull. Calling ReadAt is presumably slightly more
|
||||
// efficient, e.g. one syscall instead of two.
|
||||
func readAt(r io.ReadSeeker, p []byte, offset int64) error {
|
||||
if a, ok := r.(io.ReaderAt); ok {
|
||||
n, err := a.ReadAt(p, offset)
|
||||
if (n == len(p)) && (err == io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if _, err := r.Seek(offset, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := io.ReadFull(r, p)
|
||||
return err
|
||||
}
|
||||
|
||||
// bitwiseSubset returns whether every 1-bit in inner is also set in outer.
|
||||
func bitwiseSubset(outer Codec, inner Codec) bool {
|
||||
return outer == (outer | inner)
|
||||
@@ -237,14 +221,18 @@ func (b *rNode) valid() bool {
|
||||
type ChunkReader struct {
|
||||
// ReadSeeker is where the RAC-encoded data is read from.
|
||||
//
|
||||
// It may also implement io.ReaderAt, in which case its ReadAt method will
|
||||
// be preferred over combining Read and Seek, as the former is presumably
|
||||
// more efficient. This is optional: io.ReaderAt is a stronger contract
|
||||
// than io.ReadSeeker, as multiple concurrent ReadAt calls must not
|
||||
// interfere with each other.
|
||||
// It may optionally implement io.ReaderAt, in which case its ReadAt method
|
||||
// will be preferred and its Read and Seek methods will never be called,
|
||||
// other than Seek'ing to the end of the data to determine the compressed
|
||||
// size. The ReadAt method is safe to use concurrently, so that multiple
|
||||
// rac.Reader's can concurrently use the same source provided that the
|
||||
// source (this field, nominally an io.ReadSeeker) implements io.ReaderAt.
|
||||
//
|
||||
// For example, this type itself only implements io.ReadSeeker, not
|
||||
// io.ReaderAt, as it is not safe for concurrent use.
|
||||
// In particular, if the source is a bytes.Reader or an os.File, multiple
|
||||
// rac.ChunkReader's can work in parallel, which can speed up decoding.
|
||||
//
|
||||
// This type itself only implements io.ReadSeeker, not io.ReaderAt, as it
|
||||
// is not safe for concurrent use.
|
||||
//
|
||||
// Nil is an invalid value.
|
||||
ReadSeeker io.ReadSeeker
|
||||
@@ -259,6 +247,11 @@ type ChunkReader struct {
|
||||
// seekPosition.
|
||||
needToResolveSeekPosition bool
|
||||
|
||||
// readSeeker is either the same as the ReadSeeker field value, or it is
|
||||
// that field value wrapped by a readerat.ReadSeeker to be safe to use
|
||||
// concurrently.
|
||||
readSeeker io.ReadSeeker
|
||||
|
||||
// err is the first error encountered. It is sticky: once a non-nil error
|
||||
// occurs, all public methods will return that error.
|
||||
err error
|
||||
@@ -311,23 +304,26 @@ func (r *ChunkReader) initialize() error {
|
||||
return err
|
||||
}
|
||||
|
||||
if r.compressedSize == 0 {
|
||||
if size, err := r.ReadSeeker.Seek(0, io.SeekEnd); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
} else {
|
||||
r.compressedSize = size
|
||||
}
|
||||
if _, err := r.ReadSeeker.Seek(0, io.SeekStart); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
}
|
||||
if size, err := r.ReadSeeker.Seek(0, io.SeekEnd); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
} else {
|
||||
r.compressedSize = size
|
||||
}
|
||||
if r.compressedSize < 32 {
|
||||
r.err = errInvalidCompressedSize
|
||||
return r.err
|
||||
}
|
||||
|
||||
if ra, ok := r.ReadSeeker.(io.ReaderAt); ok {
|
||||
r.readSeeker = &readerat.ReadSeeker{
|
||||
ReaderAt: ra,
|
||||
Size: r.compressedSize,
|
||||
}
|
||||
} else {
|
||||
r.readSeeker = r.ReadSeeker
|
||||
}
|
||||
|
||||
if err := r.findRootNode(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -340,9 +336,13 @@ func (r *ChunkReader) initialize() error {
|
||||
|
||||
func (r *ChunkReader) findRootNode() error {
|
||||
// Look at the start of the compressed file.
|
||||
if err := readAt(r.ReadSeeker, r.currNode[:4], 0); err != nil {
|
||||
if _, err := r.readSeeker.Seek(0, io.SeekStart); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
return err
|
||||
}
|
||||
if _, err := io.ReadFull(r.readSeeker, r.currNode[:4]); err != nil {
|
||||
r.err = err
|
||||
return err
|
||||
}
|
||||
if (r.currNode[0] != magic[0]) ||
|
||||
(r.currNode[1] != magic[1]) ||
|
||||
@@ -357,9 +357,13 @@ func (r *ChunkReader) findRootNode() error {
|
||||
}
|
||||
|
||||
// Look at the end of the compressed file.
|
||||
if err := readAt(r.ReadSeeker, r.currNode[:1], r.compressedSize-1); err != nil {
|
||||
if _, err := r.readSeeker.Seek(r.compressedSize-1, io.SeekStart); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
return err
|
||||
}
|
||||
if _, err := io.ReadFull(r.readSeeker, r.currNode[:1]); err != nil {
|
||||
r.err = err
|
||||
return err
|
||||
}
|
||||
if found, err := r.tryRootNode(r.currNode[0], true); err != nil {
|
||||
return err
|
||||
@@ -407,9 +411,13 @@ func (r *ChunkReader) load(cOffset int64, arity uint8) error {
|
||||
return r.err
|
||||
}
|
||||
size := nodeSize(arity)
|
||||
if err := readAt(r.ReadSeeker, r.currNode[:size], cOffset); err != nil {
|
||||
if _, err := r.readSeeker.Seek(cOffset, io.SeekStart); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
return err
|
||||
}
|
||||
if _, err := io.ReadFull(r.readSeeker, r.currNode[:size]); err != nil {
|
||||
r.err = err
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -422,9 +430,13 @@ func (r *ChunkReader) loadAndValidate(cOffset int64,
|
||||
r.err = errInvalidIndexNode
|
||||
return r.err
|
||||
}
|
||||
if err := readAt(r.ReadSeeker, r.currNode[:4], cOffset); err != nil {
|
||||
if _, err := r.readSeeker.Seek(cOffset, io.SeekStart); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
return err
|
||||
}
|
||||
if _, err := io.ReadFull(r.readSeeker, r.currNode[:4]); err != nil {
|
||||
r.err = err
|
||||
return err
|
||||
}
|
||||
arity := r.currNode[3]
|
||||
if arity == 0 {
|
||||
|
||||
+20
-28
@@ -57,7 +57,7 @@ type CodecReader interface {
|
||||
MakeDecompressor(r io.Reader, rctx ReaderContext) (io.Reader, error)
|
||||
|
||||
// MakeReaderContext returns the Codec-specific ReaderContext for a chunk.
|
||||
MakeReaderContext(r io.ReadSeeker, compressedSize int64, c Chunk) (ReaderContext, error)
|
||||
MakeReaderContext(r io.ReadSeeker, c Chunk) (ReaderContext, error)
|
||||
}
|
||||
|
||||
// Reader reads a RAC file.
|
||||
@@ -68,14 +68,18 @@ type CodecReader interface {
|
||||
type Reader struct {
|
||||
// ReadSeeker is where the RAC-encoded data is read from.
|
||||
//
|
||||
// It may also implement io.ReaderAt, in which case its ReadAt method will
|
||||
// be preferred over combining Read and Seek, as the former is presumably
|
||||
// more efficient. This is optional: io.ReaderAt is a stronger contract
|
||||
// than io.ReadSeeker, as multiple concurrent ReadAt calls must not
|
||||
// interfere with each other.
|
||||
// It may optionally implement io.ReaderAt, in which case its ReadAt method
|
||||
// will be preferred and its Read and Seek methods will never be called,
|
||||
// other than Seek'ing to the end of the data to determine the compressed
|
||||
// size. The ReadAt method is safe to use concurrently, so that multiple
|
||||
// rac.Reader's can concurrently use the same source provided that the
|
||||
// source (this field, nominally an io.ReadSeeker) implements io.ReaderAt.
|
||||
//
|
||||
// For example, this type itself only implements io.ReadSeeker, not
|
||||
// io.ReaderAt, as it is not safe for concurrent use.
|
||||
// In particular, if the source is a bytes.Reader or an os.File, multiple
|
||||
// rac.Reader's can work in parallel, which can speed up decoding.
|
||||
//
|
||||
// This type itself only implements io.ReadSeeker, not io.ReaderAt, as it
|
||||
// is not safe for concurrent use.
|
||||
//
|
||||
// Nil is an invalid value.
|
||||
ReadSeeker io.ReadSeeker
|
||||
@@ -152,27 +156,15 @@ func (r *Reader) initialize() error {
|
||||
if r.err != nil {
|
||||
return r.err
|
||||
}
|
||||
if r.chunkReader.ReadSeeker != nil {
|
||||
// We're already initialized.
|
||||
if r.chunkReader.initialized {
|
||||
return nil
|
||||
}
|
||||
if r.ReadSeeker == nil {
|
||||
r.err = errInvalidReadSeeker
|
||||
return r.err
|
||||
}
|
||||
if size, err := r.ReadSeeker.Seek(0, io.SeekEnd); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
} else {
|
||||
r.chunkReader.compressedSize = size
|
||||
}
|
||||
if _, err := r.ReadSeeker.Seek(0, io.SeekStart); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
}
|
||||
|
||||
r.chunkReader.ReadSeeker = r.ReadSeeker
|
||||
r.currChunk.R = r.ReadSeeker
|
||||
if err := r.chunkReader.initialize(); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
}
|
||||
r.currChunk.R = r.chunkReader.readSeeker
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -349,7 +341,7 @@ func (r *Reader) nextChunk() error {
|
||||
return r.err
|
||||
}
|
||||
|
||||
rctx, err := codecReader.MakeReaderContext(r.ReadSeeker, r.chunkReader.compressedSize, chunk)
|
||||
rctx, err := codecReader.MakeReaderContext(r.chunkReader.readSeeker, chunk)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
@@ -358,7 +350,7 @@ func (r *Reader) nextChunk() error {
|
||||
return r.err
|
||||
}
|
||||
|
||||
if _, err := r.ReadSeeker.Seek(chunk.CPrimary[0], io.SeekStart); err != nil {
|
||||
if _, err := r.chunkReader.readSeeker.Seek(chunk.CPrimary[0], io.SeekStart); err != nil {
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
+7
-24
@@ -42,24 +42,6 @@ func u32BE(b []byte) uint32 {
|
||||
return (uint32(b[0]) << 24) | (uint32(b[1]) << 16) | (uint32(b[2]) << 8) | (uint32(b[3]))
|
||||
}
|
||||
|
||||
// readAt calls ReadAt if it is available, otherwise it falls back to calling
|
||||
// Seek and then ReadFull. Calling ReadAt is presumably slightly more
|
||||
// efficient, e.g. one syscall instead of two.
|
||||
func readAt(r io.ReadSeeker, p []byte, offset int64) error {
|
||||
if a, ok := r.(io.ReaderAt); ok {
|
||||
n, err := a.ReadAt(p, offset)
|
||||
if (n == len(p)) && (err == io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if _, err := r.Seek(offset, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := io.ReadFull(r, p)
|
||||
return err
|
||||
}
|
||||
|
||||
// suffix32K returns the last 32 KiB of b, or if b is shorter than that, it
|
||||
// returns just b.
|
||||
//
|
||||
@@ -111,9 +93,7 @@ func (r *CodecReader) MakeDecompressor(
|
||||
}
|
||||
|
||||
// MakeReaderContext implements rac.CodecReader.
|
||||
func (r *CodecReader) MakeReaderContext(
|
||||
rs io.ReadSeeker, compressedSize int64, chunk rac.Chunk) (rac.ReaderContext, error) {
|
||||
|
||||
func (r *CodecReader) MakeReaderContext(rs io.ReadSeeker, chunk rac.Chunk) (rac.ReaderContext, error) {
|
||||
// For a description of the RAC+Zlib secondary-data format, see
|
||||
// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib
|
||||
|
||||
@@ -131,12 +111,15 @@ func (r *CodecReader) MakeReaderContext(
|
||||
}
|
||||
|
||||
// Check the cRange size and the tTag.
|
||||
if (cRange.Size() < 6) || (cRange[1] > compressedSize) || (chunk.TTag != 0xFF) {
|
||||
if (cRange.Size() < 6) || (chunk.TTag != 0xFF) {
|
||||
return rac.ReaderContext{}, errInvalidDictionary
|
||||
}
|
||||
|
||||
// Read the dictionary size.
|
||||
if err := readAt(rs, r.buf[:2], cRange[0]); err != nil {
|
||||
if _, err := rs.Seek(cRange[0], io.SeekStart); err != nil {
|
||||
return rac.ReaderContext{}, errInvalidDictionary
|
||||
}
|
||||
if _, err := io.ReadFull(rs, r.buf[:2]); err != nil {
|
||||
return rac.ReaderContext{}, errInvalidDictionary
|
||||
}
|
||||
dictSize := int64(r.buf[0]) | (int64(r.buf[1]) << 8)
|
||||
@@ -158,7 +141,7 @@ func (r *CodecReader) MakeReaderContext(
|
||||
}
|
||||
|
||||
// Read the dictionary and checksum.
|
||||
if err := readAt(rs, buffer, cRange[0]+2); err != nil {
|
||||
if _, err := io.ReadFull(rs, buffer); err != nil {
|
||||
return rac.ReaderContext{}, errInvalidDictionary
|
||||
}
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/wuffs/lib/rac"
|
||||
@@ -273,3 +274,52 @@ func TestSharedDictionary(t *testing.T) {
|
||||
t.Fatalf("ratio: got %dx, want at least 4x", ratio)
|
||||
}
|
||||
}
|
||||
|
||||
// rsSansReadAt wraps a strings.Reader to have only Read and Seek methods.
|
||||
type rsSansReadAt struct {
|
||||
r *strings.Reader
|
||||
}
|
||||
|
||||
func (r *rsSansReadAt) Read(p []byte) (int, error) { return r.r.Read(p) }
|
||||
func (r *rsSansReadAt) Seek(o int64, w int) (int64, error) { return r.r.Seek(o, w) }
|
||||
|
||||
// rsWithReadAt wraps a strings.Reader to have Read, Seek and ReadAt methods.
|
||||
// Technically, it satisfies the io.ReadSeeker interface, but calling Read or
|
||||
// Seek will panic, other than Seek'ing to the end of the data.
|
||||
type rsWithReadAt struct {
|
||||
r *strings.Reader
|
||||
}
|
||||
|
||||
func (r *rsWithReadAt) Read(p []byte) (int, error) { panic("unimplemented") }
|
||||
func (r *rsWithReadAt) Seek(o int64, w int) (int64, error) {
|
||||
if (o == 0) && (w == io.SeekEnd) {
|
||||
return r.r.Seek(o, w)
|
||||
}
|
||||
panic("unimplemented")
|
||||
}
|
||||
func (r *rsWithReadAt) ReadAt(p []byte, o int64) (int, error) { return r.r.ReadAt(p, o) }
|
||||
|
||||
// testReadSeeker tests that decoding from rs works, regardless of whether rs
|
||||
// implements the optional ReadAt method. If rs does implement io.ReaderAt then
|
||||
// its Read method should never be called.
|
||||
func testReadSeeker(t *testing.T, rs io.ReadSeeker) {
|
||||
buf := &bytes.Buffer{}
|
||||
r := &rac.Reader{
|
||||
ReadSeeker: rs,
|
||||
CodecReaders: []rac.CodecReader{&CodecReader{}},
|
||||
}
|
||||
if _, err := io.Copy(buf, r); err != nil {
|
||||
t.Fatalf("io.Copy: %v", err)
|
||||
}
|
||||
if got, want := buf.String(), decodedSheep; got != want {
|
||||
t.Fatalf("got:\n%s\nwant:\n%s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadSeekerSansReadAt(t *testing.T) {
|
||||
testReadSeeker(t, &rsSansReadAt{strings.NewReader(encodedSheep)})
|
||||
}
|
||||
|
||||
func TestReadSeekerWithReadAt(t *testing.T) {
|
||||
testReadSeeker(t, &rsWithReadAt{strings.NewReader(encodedSheep)})
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
// Copyright 2019 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// ----------------
|
||||
|
||||
// Package readerat provides utilities for the io.ReaderAt type.
|
||||
package readerat
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
var (
|
||||
errInvalidSize = errors.New("readerat: invalid size")
|
||||
errSeekToInvalidWhence = errors.New("readerat: seek to invalid whence")
|
||||
errSeekToNegativePosition = errors.New("readerat: seek to negative position")
|
||||
)
|
||||
|
||||
// ReadSeeker is an io.ReadSeeker implementation based on an io.ReaderAt (and
|
||||
// an int64 size).
|
||||
//
|
||||
// For example, an os.File is both an io.ReaderAt and an io.ReadSeeker, but its
|
||||
// io.ReadSeeker methods are not safe to use concurrently. In comparison,
|
||||
// multiple readerat.ReadSeeker values (using the same os.File as their
|
||||
// io.ReaderAt) are safe to use concurrently. Each can Read and Seek
|
||||
// independently.
|
||||
//
|
||||
// A single readerat.ReadSeeker is not safe to use concurrently.
|
||||
//
|
||||
// Do not modify its exported fields after calling any of its methods.
|
||||
type ReadSeeker struct {
|
||||
ReaderAt io.ReaderAt
|
||||
Size int64
|
||||
offset int64
|
||||
}
|
||||
|
||||
// Read implements io.Reader.
|
||||
func (r *ReadSeeker) Read(p []byte) (int, error) {
|
||||
if r.Size < 0 {
|
||||
return 0, errInvalidSize
|
||||
}
|
||||
if r.Size <= r.offset {
|
||||
return 0, io.EOF
|
||||
}
|
||||
length := r.Size - r.offset
|
||||
if int64(len(p)) > length {
|
||||
p = p[:length]
|
||||
}
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
actual, err := r.ReaderAt.ReadAt(p, r.offset)
|
||||
r.offset += int64(actual)
|
||||
if (err == nil) && (r.offset == r.Size) {
|
||||
err = io.EOF
|
||||
}
|
||||
return actual, err
|
||||
}
|
||||
|
||||
// Seek implements io.Seeker.
|
||||
func (r *ReadSeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
if r.Size < 0 {
|
||||
return 0, errInvalidSize
|
||||
}
|
||||
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
// No-op.
|
||||
case io.SeekCurrent:
|
||||
offset += r.offset
|
||||
case io.SeekEnd:
|
||||
offset += r.Size
|
||||
default:
|
||||
return 0, errSeekToInvalidWhence
|
||||
}
|
||||
|
||||
if r.offset < 0 {
|
||||
return 0, errSeekToNegativePosition
|
||||
}
|
||||
r.offset = offset
|
||||
return r.offset, nil
|
||||
}
|
||||
Reference in new issue
Block a user