Add a rac.Reader.SeekRange method

This commit is contained in:
Nigel Tao committed 2019-08-31 22:19:54 +10:00
1 parent 449bbaf68e
commit b8cd560594
4 files changed
+81 -23

No files matched your search

+2 -5
View File
@@ -380,13 +380,10 @@ func decode(inFile *os.File) error {
}
r.Concurrency = n
}
if _, err := r.Seek(i, io.SeekStart); err != nil {
if err := r.SeekRange(i, j); err != nil {
return err
}
_, err = io.Copy(os.Stdout, &io.LimitedReader{
R: r,
N: j - i,
})
_, err = io.Copy(os.Stdout, r)
return err
}
+23 -14
View File
@@ -35,6 +35,7 @@ type stopWork struct {
keepWorking bool
}
// concReader co-ordinates multiple goroutines serving a Reader.
type concReader struct {
// Channels between the concReader, the Manager and multiple Workers.
//
@@ -88,6 +89,11 @@ type concReader struct {
// is called with io.SeekCurrent.
pos int64
// posLimit is an upper limit on pos. pos can go higher than it (e.g.
// seeking past the end of the file in DSpace), but after doing so, Read
// will always return (0, io.EOF).
posLimit int64
// decompressedSize is the size of the RAC file in DSpace.
decompressedSize int64
}
@@ -103,6 +109,7 @@ func (c *concReader) initialize(racReader *Reader) {
// Set up other state.
c.completedWorks = map[int64]rWork{}
c.posLimit = racReader.chunkReader.decompressedSize
c.decompressedSize = racReader.chunkReader.decompressedSize
// Set up re-usable buffers to hold decoded bytes. The number of buffers is
@@ -152,7 +159,7 @@ func (c *concReader) CloseWithoutWaiting() error {
return nil
}
func (c *concReader) Seek(offset int64, whence int) (int64, error) {
func (c *concReader) seek(offset int64, whence int, limit int64) (int64, error) {
pos := c.pos
switch whence {
case io.SeekStart:
@@ -172,11 +179,17 @@ func (c *concReader) Seek(offset int64, whence int) (int64, error) {
c.pos = pos
c.seekResolved = false
}
if limit > c.decompressedSize {
limit = c.decompressedSize
}
c.posLimit = limit
return pos, nil
}
func (c *concReader) Read(p []byte) (int, error) {
if c.pos >= c.decompressedSize {
if c.pos >= c.posLimit {
return 0, io.EOF
}
@@ -186,11 +199,11 @@ func (c *concReader) Read(p []byte) (int, error) {
c.stopAnyWorkInProgress(true)
}
c.seenRead = true
c.roic <- Range{c.pos, c.decompressedSize}
c.roic <- Range{c.pos, c.posLimit}
}
for numRead := 0; ; {
if c.pos >= c.decompressedSize {
if c.pos >= c.posLimit {
return numRead, io.EOF
}
if len(p) == 0 {
@@ -293,7 +306,6 @@ func (c *concReader) recycleBuffers() {
func runRWorker(stopc <-chan stopWork, resc chan<- rWork, reqc <-chan rWork, racReader *Reader) {
input, output := reqc, (chan<- rWork)(nil)
work := rWork{}
limitedReader := io.LimitedReader{}
loop:
for {
@@ -312,14 +324,10 @@ loop:
case work = <-input:
input, output = nil, resc
if work.err == nil {
_, work.err = racReader.Seek(work.dRange[0], io.SeekStart)
work.err = racReader.SeekRange(work.dRange[0], work.dRange[1])
}
if work.err == nil {
limitedReader = io.LimitedReader{
R: racReader,
N: work.dRange.Size(),
}
_, work.err = io.Copy(work.buf, &limitedReader)
_, work.err = io.Copy(work.buf, racReader)
}
if work.err == io.EOF {
work.err = io.ErrUnexpectedEOF
@@ -380,11 +388,12 @@ loop:
continue loop
}
dr := chunk.DRange.Intersect(roi)
if dr[0] >= roi[1] {
if chunk.DRange[0] >= roi[1] {
input, output = roic, nil
continue loop
} else if dr.Empty() {
}
dr := chunk.DRange.Intersect(roi)
if dr.Empty() {
continue
}
+1
View File
@@ -83,6 +83,7 @@ var (
errInvalidWriter = errors.New("rac: invalid Writer")
errSeekToInvalidWhence = errors.New("rac: seek to invalid whence")
errSeekToNegativePosition = errors.New("rac: seek to negative position")
errSeekToNegativeRange = errors.New("rac: seek to negative range")
errTooManyChunks = errors.New("rac: too many chunks")
errTooManyResources = errors.New("rac: too many resources")
errTooMuchInput = errors.New("rac: too much input")
+55 -4
View File
@@ -144,6 +144,11 @@ type Reader struct {
// is called with io.SeekCurrent.
pos int64
// posLimit is an upper limit on pos. pos can go higher than it (e.g.
// seeking past the end of the file in DSpace), but after doing so, Read
// will always return (0, io.EOF).
posLimit int64
// dRange is, in "State B" and "State C", what part (in DSpace) of the
// current chunk has not yet been passed on (via this type's Read method).
//
@@ -185,6 +190,7 @@ func (r *Reader) initialize() error {
return r.err
}
r.currChunk.R = r.chunkReader.readSeeker
r.posLimit = r.chunkReader.decompressedSize
r.concReader.initialize(r)
return nil
}
@@ -212,9 +218,21 @@ func (r *Reader) Read(p []byte) (int, error) {
r.err = err
return n, err
}
numRead := 0
for len(p) > 0 {
if r.pos >= r.posLimit {
return 0, io.EOF
}
if n := r.posLimit - r.pos; int64(len(p)) > n {
p = p[:n]
}
for numRead := 0; ; {
if r.pos >= r.posLimit {
return numRead, io.EOF
}
if len(p) == 0 {
return numRead, nil
}
if (r.pos < r.dRange[0]) || (r.dRange[1] < r.pos) {
r.err = errInternalInconsistentPosition
return numRead, r.err
@@ -242,7 +260,6 @@ func (r *Reader) Read(p []byte) (int, error) {
return numRead, err
}
}
return numRead, nil
}
// readExplicitData serves the zlib-compressed data in a chunk.
@@ -416,8 +433,36 @@ func (r *Reader) Seek(offset int64, whence int) (int64, error) {
if err := r.initialize(); err != nil {
return 0, err
}
const maxInt64 = (1 << 63) - 1
return r.seek(offset, whence, maxInt64)
}
// SeekRange restricts r to the half-open range [low, high).
//
// It is more efficient than but conceptually equivalent to calling Seek(low,
// io.SeekStart) and wrapping r in an io.LimitedReader whose N is (high - low).
//
// Multiple SeekRange calls apply the most recent high limit, not the minimum
// of the high limits.
//
// Any Seek call, such as Seek(0, io.SeekCurrent), will remove the high limit.
//
// It returns an error if low > high.
func (r *Reader) SeekRange(low int64, high int64) error {
if err := r.initialize(); err != nil {
return err
}
if low > high {
r.err = errSeekToNegativeRange
return r.err
}
_, err := r.seek(low, io.SeekStart, high)
return err
}
func (r *Reader) seek(offset int64, whence int, limit int64) (int64, error) {
if r.concReader.ready() {
n, err := r.concReader.Seek(offset, whence)
n, err := r.concReader.seek(offset, whence, limit)
r.err = r.err
return n, err
}
@@ -453,6 +498,12 @@ func (r *Reader) Seek(offset int64, whence int) (int64, error) {
r.decompressor = nil
r.inImplicitZeroes = false
}
if limit > r.chunkReader.decompressedSize {
limit = r.chunkReader.decompressedSize
}
r.posLimit = limit
return r.pos, nil
}