diff --git a/cmd/ractool/main.go b/cmd/ractool/main.go index 2977e95c..c702b5a5 100644 --- a/cmd/ractool/main.go +++ b/cmd/ractool/main.go @@ -304,8 +304,13 @@ func decode(r io.Reader, usingStdin bool) error { } } + compressedSize, err := rs.Seek(0, io.SeekEnd) + if err != nil { + return err + } racReader := &rac.Reader{ - ReadSeeker: rs, + ReadSeeker: rs, + CompressedSize: compressedSize, } switch *codecFlag { case "zlib": diff --git a/lib/rac/chunk_reader.go b/lib/rac/chunk_reader.go index a3116a8e..4190379d 100644 --- a/lib/rac/chunk_reader.go +++ b/lib/rac/chunk_reader.go @@ -222,9 +222,8 @@ type ChunkReader struct { // ReadSeeker is where the RAC-encoded data is read from. // // It may optionally implement io.ReaderAt, in which case its ReadAt method - // will be preferred and its Read and Seek methods will never be called, - // other than Seek'ing to the end of the data to determine the compressed - // size. The ReadAt method is safe to use concurrently, so that multiple + // will be preferred and its Read and Seek methods will never be called. + // The ReadAt method is safe to use concurrently, so that multiple // rac.Reader's can concurrently use the same source provided that the // source (this field, nominally an io.ReadSeeker) implements io.ReaderAt. // @@ -237,6 +236,11 @@ type ChunkReader struct { // Nil is an invalid value. ReadSeeker io.ReadSeeker + // CompressedSize is the size of the RAC file in CSpace. + // + // Zero is an invalid value. The smallest valid RAC file is 32 bytes long. + CompressedSize int64 + // initialized is set true after the first call on this ChunkReader. initialized bool @@ -256,9 +260,6 @@ type ChunkReader struct { // occurs, all public methods will return that error. err error - // compressedSize is the size of the RAC file in CSpace. - compressedSize int64 - // decompressedSize is the size of the RAC file in DSpace. decompressedSize int64 @@ -288,6 +289,11 @@ func (r *ChunkReader) checkParameters() error { r.err = errInvalidReadSeeker return r.err } + // The smallest valid RAC file is 32 bytes long. + if r.CompressedSize < 32 { + r.err = errInvalidCompressedSize + return r.err + } return nil } @@ -304,21 +310,10 @@ func (r *ChunkReader) initialize() error { return err } - if size, err := r.ReadSeeker.Seek(0, io.SeekEnd); err != nil { - r.err = err - return r.err - } else { - r.compressedSize = size - } - if r.compressedSize < 32 { - r.err = errInvalidCompressedSize - return r.err - } - if ra, ok := r.ReadSeeker.(io.ReaderAt); ok { r.readSeeker = &readerat.ReadSeeker{ ReaderAt: ra, - Size: r.compressedSize, + Size: r.CompressedSize, } } else { r.readSeeker = r.ReadSeeker @@ -357,7 +352,7 @@ func (r *ChunkReader) findRootNode() error { } // Look at the end of the compressed file. - if _, err := r.readSeeker.Seek(r.compressedSize-1, io.SeekStart); err != nil { + if _, err := r.readSeeker.Seek(r.CompressedSize-1, io.SeekStart); err != nil { r.err = err return err } @@ -379,12 +374,12 @@ func (r *ChunkReader) tryRootNode(arity uint8, fromEnd bool) (found bool, ioErr return false, nil } size := int64(nodeSize(arity)) - if r.compressedSize < size { + if r.CompressedSize < size { return false, nil } cOffset := int64(0) if fromEnd { - cOffset = r.compressedSize - size + cOffset = r.CompressedSize - size } if err := r.load(cOffset, arity); err != nil { return false, err @@ -392,7 +387,7 @@ func (r *ChunkReader) tryRootNode(arity uint8, fromEnd bool) (found bool, ioErr if !r.currNode.valid() { return false, nil } - if r.currNode.cPtrMax() != r.compressedSize { + if r.currNode.cPtrMax() != r.CompressedSize { return false, nil } r.needToResolveSeekPosition = true @@ -426,7 +421,7 @@ func (r *ChunkReader) loadAndValidate(cOffset int64, parentCodec Codec, parentVersion uint8, parentCOffMax int64, childCBias int64, childDSize int64) error { - if (cOffset < 0) || ((r.compressedSize - 4) < cOffset) { + if (cOffset < 0) || ((r.CompressedSize - 4) < cOffset) { r.err = errInvalidIndexNode return r.err } @@ -444,7 +439,7 @@ func (r *ChunkReader) loadAndValidate(cOffset int64, return r.err } size := int64(nodeSize(arity)) - if (r.compressedSize < size) || ((r.compressedSize - size) < cOffset) { + if (r.CompressedSize < size) || ((r.CompressedSize - size) < cOffset) { r.err = errInvalidIndexNode return r.err } diff --git a/lib/rac/example_test.go b/lib/rac/example_test.go index 71fc3289..5bc4bde6 100644 --- a/lib/rac/example_test.go +++ b/lib/rac/example_test.go @@ -160,7 +160,8 @@ func Example_indexLocationAtStart() { fmt.Printf("Decoded:\n") r := &rac.ChunkReader{ - ReadSeeker: bytes.NewReader(encoded), + ReadSeeker: bytes.NewReader(encoded), + CompressedSize: int64(len(encoded)), } zr := io.ReadCloser(nil) for { diff --git a/lib/rac/rac_test.go b/lib/rac/rac_test.go index e13b354d..5fa2d112 100644 --- a/lib/rac/rac_test.go +++ b/lib/rac/rac_test.go @@ -350,7 +350,8 @@ func TestMultiLevelIndex(t *testing.T) { } r := &ChunkReader{ - ReadSeeker: bytes.NewReader(encoded), + ReadSeeker: bytes.NewReader(encoded), + CompressedSize: int64(len(encoded)), } gotPrimaries := []byte(nil) for { @@ -402,7 +403,8 @@ loop: encoded := buf.Bytes() r := &ChunkReader{ - ReadSeeker: bytes.NewReader(encoded), + ReadSeeker: bytes.NewReader(encoded), + CompressedSize: int64(len(encoded)), } for n := 0; ; n++ { if _, err := r.NextChunk(); err == io.EOF { @@ -469,7 +471,8 @@ loop: } r := &ChunkReader{ - ReadSeeker: bytes.NewReader(tc.compressed), + ReadSeeker: bytes.NewReader(tc.compressed), + CompressedSize: int64(len(tc.compressed)), } if gotDecompressedSize, err := r.DecompressedSize(); err != nil { @@ -545,8 +548,10 @@ loop: } func TestReaderEmpty(t *testing.T) { + encoded := undoHexDump(writerWantEmpty) got, err := ioutil.ReadAll(&Reader{ - ReadSeeker: bytes.NewReader(undoHexDump(writerWantEmpty)), + ReadSeeker: bytes.NewReader(encoded), + CompressedSize: int64(len(encoded)), }) if err != nil { t.Fatalf("ReadAll: %v", err) @@ -568,8 +573,10 @@ func TestReaderZeroes(t *testing.T) { if err := w.Close(); err != nil { t.Fatalf("Close: %v", err) } + encoded := buf.Bytes() got, err := ioutil.ReadAll(&Reader{ - ReadSeeker: bytes.NewReader(buf.Bytes()), + ReadSeeker: bytes.NewReader(encoded), + CompressedSize: int64(len(encoded)), }) if err != nil { t.Fatalf("ReadAll: %v", err) diff --git a/lib/rac/reader.go b/lib/rac/reader.go index 9723e46f..1672d5eb 100644 --- a/lib/rac/reader.go +++ b/lib/rac/reader.go @@ -69,9 +69,8 @@ type Reader struct { // ReadSeeker is where the RAC-encoded data is read from. // // It may optionally implement io.ReaderAt, in which case its ReadAt method - // will be preferred and its Read and Seek methods will never be called, - // other than Seek'ing to the end of the data to determine the compressed - // size. The ReadAt method is safe to use concurrently, so that multiple + // will be preferred and its Read and Seek methods will never be called. + // The ReadAt method is safe to use concurrently, so that multiple // rac.Reader's can concurrently use the same source provided that the // source (this field, nominally an io.ReadSeeker) implements io.ReaderAt. // @@ -84,6 +83,11 @@ type Reader struct { // Nil is an invalid value. ReadSeeker io.ReadSeeker + // CompressedSize is the size of the RAC file in CSpace. + // + // Zero is an invalid value. The smallest valid RAC file is 32 bytes long. + CompressedSize int64 + // CodecReaders are the compression codecs that this Reader can decompress. // // For example, use a raczlib.CodecReader from the sibilng "raczlib" @@ -160,6 +164,7 @@ func (r *Reader) initialize() error { return nil } r.chunkReader.ReadSeeker = r.ReadSeeker + r.chunkReader.CompressedSize = r.CompressedSize if err := r.chunkReader.initialize(); err != nil { r.err = err return r.err diff --git a/lib/raczlib/example_test.go b/lib/raczlib/example_test.go index 6b8f88d6..c33fc1f5 100644 --- a/lib/raczlib/example_test.go +++ b/lib/raczlib/example_test.go @@ -71,8 +71,9 @@ func Example_roundTrip() { // Prepare to decompress. r := &rac.Reader{ - ReadSeeker: bytes.NewReader(compressed), - CodecReaders: []rac.CodecReader{&raczlib.CodecReader{}}, + ReadSeeker: bytes.NewReader(compressed), + CompressedSize: int64(len(compressed)), + CodecReaders: []rac.CodecReader{&raczlib.CodecReader{}}, } // Read the whole file. diff --git a/lib/raczlib/raczlib_test.go b/lib/raczlib/raczlib_test.go index 45f5b343..c88def44 100644 --- a/lib/raczlib/raczlib_test.go +++ b/lib/raczlib/raczlib_test.go @@ -78,8 +78,9 @@ func racCompress(original []byte, cChunkSize uint64, dChunkSize uint64, resource func racDecompress(compressed []byte) ([]byte, error) { buf := &bytes.Buffer{} r := &rac.Reader{ - ReadSeeker: bytes.NewReader(compressed), - CodecReaders: []rac.CodecReader{&CodecReader{}}, + ReadSeeker: bytes.NewReader(compressed), + CompressedSize: int64(len(compressed)), + CodecReaders: []rac.CodecReader{&CodecReader{}}, } if _, err := io.Copy(buf, r); err != nil { return nil, err @@ -197,8 +198,9 @@ func TestZeroedBytes(t *testing.T) { } r := &rac.Reader{ - ReadSeeker: bytes.NewReader(compressed), - CodecReaders: []rac.CodecReader{&CodecReader{}}, + ReadSeeker: bytes.NewReader(compressed), + CompressedSize: int64(len(compressed)), + CodecReaders: []rac.CodecReader{&CodecReader{}}, } for j := 0; j <= len(original); j++ { want := original[j:] @@ -285,28 +287,24 @@ func (r *rsSansReadAt) Seek(o int64, w int) (int64, error) { return r.r.Seek(o, // rsWithReadAt wraps a strings.Reader to have Read, Seek and ReadAt methods. // Technically, it satisfies the io.ReadSeeker interface, but calling Read or -// Seek will panic, other than Seek'ing to the end of the data. +// Seek will panic. type rsWithReadAt struct { r *strings.Reader } -func (r *rsWithReadAt) Read(p []byte) (int, error) { panic("unimplemented") } -func (r *rsWithReadAt) Seek(o int64, w int) (int64, error) { - if (o == 0) && (w == io.SeekEnd) { - return r.r.Seek(o, w) - } - panic("unimplemented") -} +func (r *rsWithReadAt) Read(p []byte) (int, error) { panic("unimplemented") } +func (r *rsWithReadAt) Seek(o int64, w int) (int64, error) { panic("unimplemented") } func (r *rsWithReadAt) ReadAt(p []byte, o int64) (int, error) { return r.r.ReadAt(p, o) } // testReadSeeker tests that decoding from rs works, regardless of whether rs // implements the optional ReadAt method. If rs does implement io.ReaderAt then -// its Read method should never be called. +// its Read and Seek methods should never be called. func testReadSeeker(t *testing.T, rs io.ReadSeeker) { buf := &bytes.Buffer{} r := &rac.Reader{ - ReadSeeker: rs, - CodecReaders: []rac.CodecReader{&CodecReader{}}, + ReadSeeker: rs, + CompressedSize: int64(len(encodedSheep)), + CodecReaders: []rac.CodecReader{&CodecReader{}}, } if _, err := io.Copy(buf, r); err != nil { t.Fatalf("io.Copy: %v", err) diff --git a/script/rac-random-seek-test.go b/script/rac-random-seek-test.go index 7d0dfda2..c5696e5c 100644 --- a/script/rac-random-seek-test.go +++ b/script/rac-random-seek-test.go @@ -60,8 +60,9 @@ func main1() error { } compressed := buf.Bytes() r := &rac.Reader{ - ReadSeeker: bytes.NewReader(compressed), - CodecReaders: []rac.CodecReader{&raczlib.CodecReader{}}, + ReadSeeker: bytes.NewReader(compressed), + CompressedSize: int64(buf.Len()), + CodecReaders: []rac.CodecReader{&raczlib.CodecReader{}}, } numChunks, err := countRACChunks(compressed) if err != nil { @@ -110,7 +111,8 @@ func main1() error { func countRACChunks(compressed []byte) (int, error) { r := &rac.ChunkReader{ - ReadSeeker: bytes.NewReader(compressed), + ReadSeeker: bytes.NewReader(compressed), + CompressedSize: int64(len(compressed)), } for n := 0; ; n++ { if _, err := r.NextChunk(); err == io.EOF {