From b8cd560594060844ed78c106ead19f3b3056e7cc Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sat, 31 Aug 2019 22:19:54 +1000 Subject: [PATCH] Add a rac.Reader.SeekRange method --- cmd/ractool/main.go | 7 ++--- lib/rac/conc_reader.go | 37 ++++++++++++++++---------- lib/rac/rac.go | 1 + lib/rac/reader.go | 59 +++++++++++++++++++++++++++++++++++++++--- 4 files changed, 81 insertions(+), 23 deletions(-) diff --git a/cmd/ractool/main.go b/cmd/ractool/main.go index 3f7bea11..9c47647e 100644 --- a/cmd/ractool/main.go +++ b/cmd/ractool/main.go @@ -380,13 +380,10 @@ func decode(inFile *os.File) error { } r.Concurrency = n } - if _, err := r.Seek(i, io.SeekStart); err != nil { + if err := r.SeekRange(i, j); err != nil { return err } - _, err = io.Copy(os.Stdout, &io.LimitedReader{ - R: r, - N: j - i, - }) + _, err = io.Copy(os.Stdout, r) return err } diff --git a/lib/rac/conc_reader.go b/lib/rac/conc_reader.go index 9750547d..21b95af4 100644 --- a/lib/rac/conc_reader.go +++ b/lib/rac/conc_reader.go @@ -35,6 +35,7 @@ type stopWork struct { keepWorking bool } +// concReader co-ordinates multiple goroutines serving a Reader. type concReader struct { // Channels between the concReader, the Manager and multiple Workers. // @@ -88,6 +89,11 @@ type concReader struct { // is called with io.SeekCurrent. pos int64 + // posLimit is an upper limit on pos. pos can go higher than it (e.g. + // seeking past the end of the file in DSpace), but after doing so, Read + // will always return (0, io.EOF). + posLimit int64 + // decompressedSize is the size of the RAC file in DSpace. decompressedSize int64 } @@ -103,6 +109,7 @@ func (c *concReader) initialize(racReader *Reader) { // Set up other state. c.completedWorks = map[int64]rWork{} + c.posLimit = racReader.chunkReader.decompressedSize c.decompressedSize = racReader.chunkReader.decompressedSize // Set up re-usable buffers to hold decoded bytes. The number of buffers is @@ -152,7 +159,7 @@ func (c *concReader) CloseWithoutWaiting() error { return nil } -func (c *concReader) Seek(offset int64, whence int) (int64, error) { +func (c *concReader) seek(offset int64, whence int, limit int64) (int64, error) { pos := c.pos switch whence { case io.SeekStart: @@ -172,11 +179,17 @@ func (c *concReader) Seek(offset int64, whence int) (int64, error) { c.pos = pos c.seekResolved = false } + + if limit > c.decompressedSize { + limit = c.decompressedSize + } + c.posLimit = limit + return pos, nil } func (c *concReader) Read(p []byte) (int, error) { - if c.pos >= c.decompressedSize { + if c.pos >= c.posLimit { return 0, io.EOF } @@ -186,11 +199,11 @@ func (c *concReader) Read(p []byte) (int, error) { c.stopAnyWorkInProgress(true) } c.seenRead = true - c.roic <- Range{c.pos, c.decompressedSize} + c.roic <- Range{c.pos, c.posLimit} } for numRead := 0; ; { - if c.pos >= c.decompressedSize { + if c.pos >= c.posLimit { return numRead, io.EOF } if len(p) == 0 { @@ -293,7 +306,6 @@ func (c *concReader) recycleBuffers() { func runRWorker(stopc <-chan stopWork, resc chan<- rWork, reqc <-chan rWork, racReader *Reader) { input, output := reqc, (chan<- rWork)(nil) work := rWork{} - limitedReader := io.LimitedReader{} loop: for { @@ -312,14 +324,10 @@ loop: case work = <-input: input, output = nil, resc if work.err == nil { - _, work.err = racReader.Seek(work.dRange[0], io.SeekStart) + work.err = racReader.SeekRange(work.dRange[0], work.dRange[1]) } if work.err == nil { - limitedReader = io.LimitedReader{ - R: racReader, - N: work.dRange.Size(), - } - _, work.err = io.Copy(work.buf, &limitedReader) + _, work.err = io.Copy(work.buf, racReader) } if work.err == io.EOF { work.err = io.ErrUnexpectedEOF @@ -380,11 +388,12 @@ loop: continue loop } - dr := chunk.DRange.Intersect(roi) - if dr[0] >= roi[1] { + if chunk.DRange[0] >= roi[1] { input, output = roic, nil continue loop - } else if dr.Empty() { + } + dr := chunk.DRange.Intersect(roi) + if dr.Empty() { continue } diff --git a/lib/rac/rac.go b/lib/rac/rac.go index 481cf147..f5817e9b 100644 --- a/lib/rac/rac.go +++ b/lib/rac/rac.go @@ -83,6 +83,7 @@ var ( errInvalidWriter = errors.New("rac: invalid Writer") errSeekToInvalidWhence = errors.New("rac: seek to invalid whence") errSeekToNegativePosition = errors.New("rac: seek to negative position") + errSeekToNegativeRange = errors.New("rac: seek to negative range") errTooManyChunks = errors.New("rac: too many chunks") errTooManyResources = errors.New("rac: too many resources") errTooMuchInput = errors.New("rac: too much input") diff --git a/lib/rac/reader.go b/lib/rac/reader.go index f1b20dbe..665894be 100644 --- a/lib/rac/reader.go +++ b/lib/rac/reader.go @@ -144,6 +144,11 @@ type Reader struct { // is called with io.SeekCurrent. pos int64 + // posLimit is an upper limit on pos. pos can go higher than it (e.g. + // seeking past the end of the file in DSpace), but after doing so, Read + // will always return (0, io.EOF). + posLimit int64 + // dRange is, in "State B" and "State C", what part (in DSpace) of the // current chunk has not yet been passed on (via this type's Read method). // @@ -185,6 +190,7 @@ func (r *Reader) initialize() error { return r.err } r.currChunk.R = r.chunkReader.readSeeker + r.posLimit = r.chunkReader.decompressedSize r.concReader.initialize(r) return nil } @@ -212,9 +218,21 @@ func (r *Reader) Read(p []byte) (int, error) { r.err = err return n, err } - numRead := 0 - for len(p) > 0 { + if r.pos >= r.posLimit { + return 0, io.EOF + } + if n := r.posLimit - r.pos; int64(len(p)) > n { + p = p[:n] + } + + for numRead := 0; ; { + if r.pos >= r.posLimit { + return numRead, io.EOF + } + if len(p) == 0 { + return numRead, nil + } if (r.pos < r.dRange[0]) || (r.dRange[1] < r.pos) { r.err = errInternalInconsistentPosition return numRead, r.err @@ -242,7 +260,6 @@ func (r *Reader) Read(p []byte) (int, error) { return numRead, err } } - return numRead, nil } // readExplicitData serves the zlib-compressed data in a chunk. @@ -416,8 +433,36 @@ func (r *Reader) Seek(offset int64, whence int) (int64, error) { if err := r.initialize(); err != nil { return 0, err } + const maxInt64 = (1 << 63) - 1 + return r.seek(offset, whence, maxInt64) +} + +// SeekRange restricts r to the half-open range [low, high). +// +// It is more efficient than but conceptually equivalent to calling Seek(low, +// io.SeekStart) and wrapping r in an io.LimitedReader whose N is (high - low). +// +// Multiple SeekRange calls apply the most recent high limit, not the minimum +// of the high limits. +// +// Any Seek call, such as Seek(0, io.SeekCurrent), will remove the high limit. +// +// It returns an error if low > high. +func (r *Reader) SeekRange(low int64, high int64) error { + if err := r.initialize(); err != nil { + return err + } + if low > high { + r.err = errSeekToNegativeRange + return r.err + } + _, err := r.seek(low, io.SeekStart, high) + return err +} + +func (r *Reader) seek(offset int64, whence int, limit int64) (int64, error) { if r.concReader.ready() { - n, err := r.concReader.Seek(offset, whence) + n, err := r.concReader.seek(offset, whence, limit) r.err = r.err return n, err } @@ -453,6 +498,12 @@ func (r *Reader) Seek(offset int64, whence int) (int64, error) { r.decompressor = nil r.inImplicitZeroes = false } + + if limit > r.chunkReader.decompressedSize { + limit = r.chunkReader.decompressedSize + } + r.posLimit = limit + return r.pos, nil }