Restore rac's CompressedSize to avoid Seek calls

This commit is contained in:
Nigel Tao committed 2019-08-24 09:34:12 +10:00
1 parent f53bee37ba
commit ca4a6c8846
8 files changed
+68 -54

No files matched your search

+6 -1
View File
@@ -304,8 +304,13 @@ func decode(r io.Reader, usingStdin bool) error {
}
}
compressedSize, err := rs.Seek(0, io.SeekEnd)
if err != nil {
return err
}
racReader := &rac.Reader{
ReadSeeker: rs,
ReadSeeker: rs,
CompressedSize: compressedSize,
}
switch *codecFlag {
case "zlib":
+19 -24
View File
@@ -222,9 +222,8 @@ type ChunkReader struct {
// ReadSeeker is where the RAC-encoded data is read from.
//
// It may optionally implement io.ReaderAt, in which case its ReadAt method
// will be preferred and its Read and Seek methods will never be called,
// other than Seek'ing to the end of the data to determine the compressed
// size. The ReadAt method is safe to use concurrently, so that multiple
// will be preferred and its Read and Seek methods will never be called.
// The ReadAt method is safe to use concurrently, so that multiple
// rac.Reader's can concurrently use the same source provided that the
// source (this field, nominally an io.ReadSeeker) implements io.ReaderAt.
//
@@ -237,6 +236,11 @@ type ChunkReader struct {
// Nil is an invalid value.
ReadSeeker io.ReadSeeker
// CompressedSize is the size of the RAC file in CSpace.
//
// Zero is an invalid value. The smallest valid RAC file is 32 bytes long.
CompressedSize int64
// initialized is set true after the first call on this ChunkReader.
initialized bool
@@ -256,9 +260,6 @@ type ChunkReader struct {
// occurs, all public methods will return that error.
err error
// compressedSize is the size of the RAC file in CSpace.
compressedSize int64
// decompressedSize is the size of the RAC file in DSpace.
decompressedSize int64
@@ -288,6 +289,11 @@ func (r *ChunkReader) checkParameters() error {
r.err = errInvalidReadSeeker
return r.err
}
// The smallest valid RAC file is 32 bytes long.
if r.CompressedSize < 32 {
r.err = errInvalidCompressedSize
return r.err
}
return nil
}
@@ -304,21 +310,10 @@ func (r *ChunkReader) initialize() error {
return err
}
if size, err := r.ReadSeeker.Seek(0, io.SeekEnd); err != nil {
r.err = err
return r.err
} else {
r.compressedSize = size
}
if r.compressedSize < 32 {
r.err = errInvalidCompressedSize
return r.err
}
if ra, ok := r.ReadSeeker.(io.ReaderAt); ok {
r.readSeeker = &readerat.ReadSeeker{
ReaderAt: ra,
Size: r.compressedSize,
Size: r.CompressedSize,
}
} else {
r.readSeeker = r.ReadSeeker
@@ -357,7 +352,7 @@ func (r *ChunkReader) findRootNode() error {
}
// Look at the end of the compressed file.
if _, err := r.readSeeker.Seek(r.compressedSize-1, io.SeekStart); err != nil {
if _, err := r.readSeeker.Seek(r.CompressedSize-1, io.SeekStart); err != nil {
r.err = err
return err
}
@@ -379,12 +374,12 @@ func (r *ChunkReader) tryRootNode(arity uint8, fromEnd bool) (found bool, ioErr
return false, nil
}
size := int64(nodeSize(arity))
if r.compressedSize < size {
if r.CompressedSize < size {
return false, nil
}
cOffset := int64(0)
if fromEnd {
cOffset = r.compressedSize - size
cOffset = r.CompressedSize - size
}
if err := r.load(cOffset, arity); err != nil {
return false, err
@@ -392,7 +387,7 @@ func (r *ChunkReader) tryRootNode(arity uint8, fromEnd bool) (found bool, ioErr
if !r.currNode.valid() {
return false, nil
}
if r.currNode.cPtrMax() != r.compressedSize {
if r.currNode.cPtrMax() != r.CompressedSize {
return false, nil
}
r.needToResolveSeekPosition = true
@@ -426,7 +421,7 @@ func (r *ChunkReader) loadAndValidate(cOffset int64,
parentCodec Codec, parentVersion uint8, parentCOffMax int64,
childCBias int64, childDSize int64) error {
if (cOffset < 0) || ((r.compressedSize - 4) < cOffset) {
if (cOffset < 0) || ((r.CompressedSize - 4) < cOffset) {
r.err = errInvalidIndexNode
return r.err
}
@@ -444,7 +439,7 @@ func (r *ChunkReader) loadAndValidate(cOffset int64,
return r.err
}
size := int64(nodeSize(arity))
if (r.compressedSize < size) || ((r.compressedSize - size) < cOffset) {
if (r.CompressedSize < size) || ((r.CompressedSize - size) < cOffset) {
r.err = errInvalidIndexNode
return r.err
}
+2 -1
View File
@@ -160,7 +160,8 @@ func Example_indexLocationAtStart() {
fmt.Printf("Decoded:\n")
r := &rac.ChunkReader{
ReadSeeker: bytes.NewReader(encoded),
ReadSeeker: bytes.NewReader(encoded),
CompressedSize: int64(len(encoded)),
}
zr := io.ReadCloser(nil)
for {
+12 -5
View File
@@ -350,7 +350,8 @@ func TestMultiLevelIndex(t *testing.T) {
}
r := &ChunkReader{
ReadSeeker: bytes.NewReader(encoded),
ReadSeeker: bytes.NewReader(encoded),
CompressedSize: int64(len(encoded)),
}
gotPrimaries := []byte(nil)
for {
@@ -402,7 +403,8 @@ loop:
encoded := buf.Bytes()
r := &ChunkReader{
ReadSeeker: bytes.NewReader(encoded),
ReadSeeker: bytes.NewReader(encoded),
CompressedSize: int64(len(encoded)),
}
for n := 0; ; n++ {
if _, err := r.NextChunk(); err == io.EOF {
@@ -469,7 +471,8 @@ loop:
}
r := &ChunkReader{
ReadSeeker: bytes.NewReader(tc.compressed),
ReadSeeker: bytes.NewReader(tc.compressed),
CompressedSize: int64(len(tc.compressed)),
}
if gotDecompressedSize, err := r.DecompressedSize(); err != nil {
@@ -545,8 +548,10 @@ loop:
}
func TestReaderEmpty(t *testing.T) {
encoded := undoHexDump(writerWantEmpty)
got, err := ioutil.ReadAll(&Reader{
ReadSeeker: bytes.NewReader(undoHexDump(writerWantEmpty)),
ReadSeeker: bytes.NewReader(encoded),
CompressedSize: int64(len(encoded)),
})
if err != nil {
t.Fatalf("ReadAll: %v", err)
@@ -568,8 +573,10 @@ func TestReaderZeroes(t *testing.T) {
if err := w.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
encoded := buf.Bytes()
got, err := ioutil.ReadAll(&Reader{
ReadSeeker: bytes.NewReader(buf.Bytes()),
ReadSeeker: bytes.NewReader(encoded),
CompressedSize: int64(len(encoded)),
})
if err != nil {
t.Fatalf("ReadAll: %v", err)
+8 -3
View File
@@ -69,9 +69,8 @@ type Reader struct {
// ReadSeeker is where the RAC-encoded data is read from.
//
// It may optionally implement io.ReaderAt, in which case its ReadAt method
// will be preferred and its Read and Seek methods will never be called,
// other than Seek'ing to the end of the data to determine the compressed
// size. The ReadAt method is safe to use concurrently, so that multiple
// will be preferred and its Read and Seek methods will never be called.
// The ReadAt method is safe to use concurrently, so that multiple
// rac.Reader's can concurrently use the same source provided that the
// source (this field, nominally an io.ReadSeeker) implements io.ReaderAt.
//
@@ -84,6 +83,11 @@ type Reader struct {
// Nil is an invalid value.
ReadSeeker io.ReadSeeker
// CompressedSize is the size of the RAC file in CSpace.
//
// Zero is an invalid value. The smallest valid RAC file is 32 bytes long.
CompressedSize int64
// CodecReaders are the compression codecs that this Reader can decompress.
//
// For example, use a raczlib.CodecReader from the sibilng "raczlib"
@@ -160,6 +164,7 @@ func (r *Reader) initialize() error {
return nil
}
r.chunkReader.ReadSeeker = r.ReadSeeker
r.chunkReader.CompressedSize = r.CompressedSize
if err := r.chunkReader.initialize(); err != nil {
r.err = err
return r.err
+3 -2
View File
@@ -71,8 +71,9 @@ func Example_roundTrip() {
// Prepare to decompress.
r := &rac.Reader{
ReadSeeker: bytes.NewReader(compressed),
CodecReaders: []rac.CodecReader{&raczlib.CodecReader{}},
ReadSeeker: bytes.NewReader(compressed),
CompressedSize: int64(len(compressed)),
CodecReaders: []rac.CodecReader{&raczlib.CodecReader{}},
}
// Read the whole file.
+13 -15
View File
@@ -78,8 +78,9 @@ func racCompress(original []byte, cChunkSize uint64, dChunkSize uint64, resource
func racDecompress(compressed []byte) ([]byte, error) {
buf := &bytes.Buffer{}
r := &rac.Reader{
ReadSeeker: bytes.NewReader(compressed),
CodecReaders: []rac.CodecReader{&CodecReader{}},
ReadSeeker: bytes.NewReader(compressed),
CompressedSize: int64(len(compressed)),
CodecReaders: []rac.CodecReader{&CodecReader{}},
}
if _, err := io.Copy(buf, r); err != nil {
return nil, err
@@ -197,8 +198,9 @@ func TestZeroedBytes(t *testing.T) {
}
r := &rac.Reader{
ReadSeeker: bytes.NewReader(compressed),
CodecReaders: []rac.CodecReader{&CodecReader{}},
ReadSeeker: bytes.NewReader(compressed),
CompressedSize: int64(len(compressed)),
CodecReaders: []rac.CodecReader{&CodecReader{}},
}
for j := 0; j <= len(original); j++ {
want := original[j:]
@@ -285,28 +287,24 @@ func (r *rsSansReadAt) Seek(o int64, w int) (int64, error) { return r.r.Seek(o,
// rsWithReadAt wraps a strings.Reader to have Read, Seek and ReadAt methods.
// Technically, it satisfies the io.ReadSeeker interface, but calling Read or
// Seek will panic, other than Seek'ing to the end of the data.
// Seek will panic.
type rsWithReadAt struct {
r *strings.Reader
}
func (r *rsWithReadAt) Read(p []byte) (int, error) { panic("unimplemented") }
func (r *rsWithReadAt) Seek(o int64, w int) (int64, error) {
if (o == 0) && (w == io.SeekEnd) {
return r.r.Seek(o, w)
}
panic("unimplemented")
}
func (r *rsWithReadAt) Read(p []byte) (int, error) { panic("unimplemented") }
func (r *rsWithReadAt) Seek(o int64, w int) (int64, error) { panic("unimplemented") }
func (r *rsWithReadAt) ReadAt(p []byte, o int64) (int, error) { return r.r.ReadAt(p, o) }
// testReadSeeker tests that decoding from rs works, regardless of whether rs
// implements the optional ReadAt method. If rs does implement io.ReaderAt then
// its Read method should never be called.
// its Read and Seek methods should never be called.
func testReadSeeker(t *testing.T, rs io.ReadSeeker) {
buf := &bytes.Buffer{}
r := &rac.Reader{
ReadSeeker: rs,
CodecReaders: []rac.CodecReader{&CodecReader{}},
ReadSeeker: rs,
CompressedSize: int64(len(encodedSheep)),
CodecReaders: []rac.CodecReader{&CodecReader{}},
}
if _, err := io.Copy(buf, r); err != nil {
t.Fatalf("io.Copy: %v", err)
+5 -3
View File
@@ -60,8 +60,9 @@ func main1() error {
}
compressed := buf.Bytes()
r := &rac.Reader{
ReadSeeker: bytes.NewReader(compressed),
CodecReaders: []rac.CodecReader{&raczlib.CodecReader{}},
ReadSeeker: bytes.NewReader(compressed),
CompressedSize: int64(buf.Len()),
CodecReaders: []rac.CodecReader{&raczlib.CodecReader{}},
}
numChunks, err := countRACChunks(compressed)
if err != nil {
@@ -110,7 +111,8 @@ func main1() error {
func countRACChunks(compressed []byte) (int, error) {
r := &rac.ChunkReader{
ReadSeeker: bytes.NewReader(compressed),
ReadSeeker: bytes.NewReader(compressed),
CompressedSize: int64(len(compressed)),
}
for n := 0; ; n++ {
if _, err := r.NextChunk(); err == io.EOF {