Restore rac's CompressedSize to avoid Seek calls

This commit is contained in:
Nigel Tao committed 2019-08-24 09:34:12 +10:00
1 parent f53bee37ba
commit ca4a6c8846
8 files changed
+68 -54

No files matched your search

+19 -24
View File
@@ -222,9 +222,8 @@ type ChunkReader struct {
// ReadSeeker is where the RAC-encoded data is read from.
//
// It may optionally implement io.ReaderAt, in which case its ReadAt method
// will be preferred and its Read and Seek methods will never be called,
// other than Seek'ing to the end of the data to determine the compressed
// size. The ReadAt method is safe to use concurrently, so that multiple
// will be preferred and its Read and Seek methods will never be called.
// The ReadAt method is safe to use concurrently, so that multiple
// rac.Reader's can concurrently use the same source provided that the
// source (this field, nominally an io.ReadSeeker) implements io.ReaderAt.
//
@@ -237,6 +236,11 @@ type ChunkReader struct {
// Nil is an invalid value.
ReadSeeker io.ReadSeeker
// CompressedSize is the size of the RAC file in CSpace.
//
// Zero is an invalid value. The smallest valid RAC file is 32 bytes long.
CompressedSize int64
// initialized is set true after the first call on this ChunkReader.
initialized bool
@@ -256,9 +260,6 @@ type ChunkReader struct {
// occurs, all public methods will return that error.
err error
// compressedSize is the size of the RAC file in CSpace.
compressedSize int64
// decompressedSize is the size of the RAC file in DSpace.
decompressedSize int64
@@ -288,6 +289,11 @@ func (r *ChunkReader) checkParameters() error {
r.err = errInvalidReadSeeker
return r.err
}
// The smallest valid RAC file is 32 bytes long.
if r.CompressedSize < 32 {
r.err = errInvalidCompressedSize
return r.err
}
return nil
}
@@ -304,21 +310,10 @@ func (r *ChunkReader) initialize() error {
return err
}
if size, err := r.ReadSeeker.Seek(0, io.SeekEnd); err != nil {
r.err = err
return r.err
} else {
r.compressedSize = size
}
if r.compressedSize < 32 {
r.err = errInvalidCompressedSize
return r.err
}
if ra, ok := r.ReadSeeker.(io.ReaderAt); ok {
r.readSeeker = &readerat.ReadSeeker{
ReaderAt: ra,
Size: r.compressedSize,
Size: r.CompressedSize,
}
} else {
r.readSeeker = r.ReadSeeker
@@ -357,7 +352,7 @@ func (r *ChunkReader) findRootNode() error {
}
// Look at the end of the compressed file.
if _, err := r.readSeeker.Seek(r.compressedSize-1, io.SeekStart); err != nil {
if _, err := r.readSeeker.Seek(r.CompressedSize-1, io.SeekStart); err != nil {
r.err = err
return err
}
@@ -379,12 +374,12 @@ func (r *ChunkReader) tryRootNode(arity uint8, fromEnd bool) (found bool, ioErr
return false, nil
}
size := int64(nodeSize(arity))
if r.compressedSize < size {
if r.CompressedSize < size {
return false, nil
}
cOffset := int64(0)
if fromEnd {
cOffset = r.compressedSize - size
cOffset = r.CompressedSize - size
}
if err := r.load(cOffset, arity); err != nil {
return false, err
@@ -392,7 +387,7 @@ func (r *ChunkReader) tryRootNode(arity uint8, fromEnd bool) (found bool, ioErr
if !r.currNode.valid() {
return false, nil
}
if r.currNode.cPtrMax() != r.compressedSize {
if r.currNode.cPtrMax() != r.CompressedSize {
return false, nil
}
r.needToResolveSeekPosition = true
@@ -426,7 +421,7 @@ func (r *ChunkReader) loadAndValidate(cOffset int64,
parentCodec Codec, parentVersion uint8, parentCOffMax int64,
childCBias int64, childDSize int64) error {
if (cOffset < 0) || ((r.compressedSize - 4) < cOffset) {
if (cOffset < 0) || ((r.CompressedSize - 4) < cOffset) {
r.err = errInvalidIndexNode
return r.err
}
@@ -444,7 +439,7 @@ func (r *ChunkReader) loadAndValidate(cOffset int64,
return r.err
}
size := int64(nodeSize(arity))
if (r.compressedSize < size) || ((r.compressedSize - size) < cOffset) {
if (r.CompressedSize < size) || ((r.CompressedSize - size) < cOffset) {
r.err = errInvalidIndexNode
return r.err
}
+2 -1
View File
@@ -160,7 +160,8 @@ func Example_indexLocationAtStart() {
fmt.Printf("Decoded:\n")
r := &rac.ChunkReader{
ReadSeeker: bytes.NewReader(encoded),
ReadSeeker: bytes.NewReader(encoded),
CompressedSize: int64(len(encoded)),
}
zr := io.ReadCloser(nil)
for {
+12 -5
View File
@@ -350,7 +350,8 @@ func TestMultiLevelIndex(t *testing.T) {
}
r := &ChunkReader{
ReadSeeker: bytes.NewReader(encoded),
ReadSeeker: bytes.NewReader(encoded),
CompressedSize: int64(len(encoded)),
}
gotPrimaries := []byte(nil)
for {
@@ -402,7 +403,8 @@ loop:
encoded := buf.Bytes()
r := &ChunkReader{
ReadSeeker: bytes.NewReader(encoded),
ReadSeeker: bytes.NewReader(encoded),
CompressedSize: int64(len(encoded)),
}
for n := 0; ; n++ {
if _, err := r.NextChunk(); err == io.EOF {
@@ -469,7 +471,8 @@ loop:
}
r := &ChunkReader{
ReadSeeker: bytes.NewReader(tc.compressed),
ReadSeeker: bytes.NewReader(tc.compressed),
CompressedSize: int64(len(tc.compressed)),
}
if gotDecompressedSize, err := r.DecompressedSize(); err != nil {
@@ -545,8 +548,10 @@ loop:
}
func TestReaderEmpty(t *testing.T) {
encoded := undoHexDump(writerWantEmpty)
got, err := ioutil.ReadAll(&Reader{
ReadSeeker: bytes.NewReader(undoHexDump(writerWantEmpty)),
ReadSeeker: bytes.NewReader(encoded),
CompressedSize: int64(len(encoded)),
})
if err != nil {
t.Fatalf("ReadAll: %v", err)
@@ -568,8 +573,10 @@ func TestReaderZeroes(t *testing.T) {
if err := w.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
encoded := buf.Bytes()
got, err := ioutil.ReadAll(&Reader{
ReadSeeker: bytes.NewReader(buf.Bytes()),
ReadSeeker: bytes.NewReader(encoded),
CompressedSize: int64(len(encoded)),
})
if err != nil {
t.Fatalf("ReadAll: %v", err)
+8 -3
View File
@@ -69,9 +69,8 @@ type Reader struct {
// ReadSeeker is where the RAC-encoded data is read from.
//
// It may optionally implement io.ReaderAt, in which case its ReadAt method
// will be preferred and its Read and Seek methods will never be called,
// other than Seek'ing to the end of the data to determine the compressed
// size. The ReadAt method is safe to use concurrently, so that multiple
// will be preferred and its Read and Seek methods will never be called.
// The ReadAt method is safe to use concurrently, so that multiple
// rac.Reader's can concurrently use the same source provided that the
// source (this field, nominally an io.ReadSeeker) implements io.ReaderAt.
//
@@ -84,6 +83,11 @@ type Reader struct {
// Nil is an invalid value.
ReadSeeker io.ReadSeeker
// CompressedSize is the size of the RAC file in CSpace.
//
// Zero is an invalid value. The smallest valid RAC file is 32 bytes long.
CompressedSize int64
// CodecReaders are the compression codecs that this Reader can decompress.
//
// For example, use a raczlib.CodecReader from the sibilng "raczlib"
@@ -160,6 +164,7 @@ func (r *Reader) initialize() error {
return nil
}
r.chunkReader.ReadSeeker = r.ReadSeeker
r.chunkReader.CompressedSize = r.CompressedSize
if err := r.chunkReader.initialize(); err != nil {
r.err = err
return r.err