mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 16:32:11 +01:00
Move RAC concurrent decode from program to library
This commit is contained in:
1 parent
e0d9749f4e
commit
7753d147b7
7 files changed
+518
-164
No files matched your search
+28
-154
@@ -315,12 +315,6 @@ func parseRange(s string) (i int64, j int64, ok bool) {
|
||||
}
|
||||
|
||||
func decode(inFile *os.File) error {
|
||||
switch *codecFlag {
|
||||
case "zlib":
|
||||
// No-op.
|
||||
default:
|
||||
return errors.New("unsupported -codec")
|
||||
}
|
||||
i, j, ok := parseRange(*drangeFlag)
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid -drange")
|
||||
@@ -357,152 +351,6 @@ func decode(inFile *os.File) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if *singlethreadedFlag {
|
||||
racReader := mustMakeRACReader(rs, compressedSize)
|
||||
return decodeSingleThreaded(os.Stdout, rac.Range{i, j}, racReader)
|
||||
}
|
||||
return decodeMultiThreaded(os.Stdout, rac.Range{i, j}, rs, compressedSize)
|
||||
}
|
||||
|
||||
func decodeMultiThreaded(dst io.Writer, overallDRange rac.Range, rs io.ReadSeeker, compressedSize int64) error {
|
||||
numWorkers := runtime.NumCPU()
|
||||
// After 16 workers, we seem to hit diminishing returns.
|
||||
if numWorkers > 16 {
|
||||
numWorkers = 16
|
||||
}
|
||||
|
||||
// Set up re-usable buffers to hold decoded bytes. The number of buffers is
|
||||
// arbitrary, but is larger than the number of workers because individual
|
||||
// pieces of work (the decodeWork type) may complete in a different order
|
||||
// than they need to be written to dst. Having spare buffers means fewer
|
||||
// idle workers.
|
||||
bufc := make(chan *bytes.Buffer, 2*numWorkers)
|
||||
for i := 0; i < cap(bufc); i++ {
|
||||
bufc <- &bytes.Buffer{}
|
||||
}
|
||||
|
||||
// Set up the workers, fed incomplete decodeWork's by issueDecodeWork.
|
||||
reqc := make(chan decodeWork, numWorkers)
|
||||
resc := make(chan decodeWork, numWorkers)
|
||||
go issueDecodeWork(reqc, bufc, overallDRange, rs, compressedSize)
|
||||
for i := 0; i < numWorkers; i++ {
|
||||
go decodeWorker(resc, reqc, mustMakeRACReader(rs, compressedSize))
|
||||
}
|
||||
|
||||
// m holds completed decodeWork's, which may arrive out of order. The next
|
||||
// (in ascending DRange order) decodeWork starts at nextDRange0.
|
||||
m := map[int64]*bytes.Buffer{}
|
||||
nextDRange0 := overallDRange[0]
|
||||
|
||||
// Handle completed decodeWork's.
|
||||
for work := range resc {
|
||||
// Add the next arrival to m (if non-empty), then pop all of m's
|
||||
// entries that are next in ascending DRange order, returning any used
|
||||
// buffers to bufc to be recycled.
|
||||
if work.buf == nil {
|
||||
// No-op.
|
||||
} else if work.buf.Len() == 0 {
|
||||
bufc <- work.buf
|
||||
} else {
|
||||
m[work.dRange[0]] = work.buf
|
||||
for {
|
||||
buf, ok := m[nextDRange0]
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
delete(m, nextDRange0)
|
||||
n, err := dst.Write(buf.Bytes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nextDRange0 += int64(n)
|
||||
bufc <- buf
|
||||
}
|
||||
}
|
||||
|
||||
if work.err == errEndOfWork {
|
||||
numWorkers--
|
||||
if numWorkers != 0 {
|
||||
continue
|
||||
}
|
||||
return nil
|
||||
} else if work.err != nil {
|
||||
return work.err
|
||||
}
|
||||
|
||||
// This shouldn't happen, but if it does, print a specific error
|
||||
// message instead of the more general "deadlock" message.
|
||||
if len(m) == cap(bufc) {
|
||||
return errors.New("internal error: all workers idle but the next chunk is not found")
|
||||
}
|
||||
}
|
||||
panic("unreachable")
|
||||
}
|
||||
|
||||
var errEndOfWork = errors.New("end of work")
|
||||
|
||||
// decodeWork is a unit of work for multi-threaded decoding. Workers receive
|
||||
// empty buffers and send filled buffers.
|
||||
type decodeWork struct {
|
||||
dRange rac.Range
|
||||
buf *bytes.Buffer
|
||||
err error
|
||||
}
|
||||
|
||||
func decodeWorker(resc chan<- decodeWork, reqc <-chan decodeWork, racReader *rac.Reader) {
|
||||
for work := range reqc {
|
||||
if work.err == nil {
|
||||
work.err = decodeSingleThreaded(work.buf, work.dRange, racReader)
|
||||
}
|
||||
resc <- work
|
||||
if work.err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
resc <- decodeWork{err: errEndOfWork}
|
||||
}
|
||||
|
||||
func issueDecodeWork(reqc chan<- decodeWork, bufc <-chan *bytes.Buffer, overallDRange rac.Range, rs io.ReadSeeker, compressedSize int64) {
|
||||
defer close(reqc)
|
||||
chunkReader := &rac.ChunkReader{
|
||||
ReadSeeker: rs,
|
||||
CompressedSize: compressedSize,
|
||||
}
|
||||
if err := chunkReader.SeekToChunkContaining(overallDRange[0]); err != nil {
|
||||
reqc <- decodeWork{err: err}
|
||||
return
|
||||
}
|
||||
for {
|
||||
chunk, err := chunkReader.NextChunk()
|
||||
if err == io.EOF {
|
||||
return
|
||||
} else if err != nil {
|
||||
reqc <- decodeWork{err: err}
|
||||
return
|
||||
}
|
||||
if dr := chunk.DRange.Intersect(overallDRange); !dr.Empty() {
|
||||
buf := <-bufc
|
||||
buf.Reset()
|
||||
reqc <- decodeWork{dRange: dr, buf: buf}
|
||||
}
|
||||
if chunk.DRange[1] >= overallDRange[1] {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func decodeSingleThreaded(dst io.Writer, dRange rac.Range, racReader *rac.Reader) error {
|
||||
if _, err := racReader.Seek(dRange[0], io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := io.Copy(dst, &io.LimitedReader{
|
||||
R: racReader,
|
||||
N: dRange.Size(),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func mustMakeRACReader(rs io.ReadSeeker, compressedSize int64) *rac.Reader {
|
||||
r := &rac.Reader{
|
||||
ReadSeeker: rs,
|
||||
CompressedSize: compressedSize,
|
||||
@@ -511,9 +359,35 @@ func mustMakeRACReader(rs io.ReadSeeker, compressedSize int64) *rac.Reader {
|
||||
case "zlib":
|
||||
r.CodecReaders = []rac.CodecReader{&raczlib.CodecReader{}}
|
||||
default:
|
||||
panic("unreachable")
|
||||
return errors.New("unsupported -codec")
|
||||
}
|
||||
return r
|
||||
|
||||
// The r.Close method might need to wait for its goroutines to shut down
|
||||
// cleanly, to guarantee that the underlying io.ReadSeeker won't be used
|
||||
// after r.Close returns.
|
||||
//
|
||||
// But here, we're a program ("package main"), not a library. After this
|
||||
// function (decode) returns, we'll exit the program. There's no need to
|
||||
// hold that up, so we call CloseWithoutWaiting instead of Close.
|
||||
defer r.CloseWithoutWaiting()
|
||||
|
||||
if !*singlethreadedFlag {
|
||||
n := runtime.NumCPU()
|
||||
// After 16 workers, we seem to hit diminishing speed returns, but
|
||||
// still face increasing memory costs.
|
||||
if n > 16 {
|
||||
n = 16
|
||||
}
|
||||
r.Concurrency = n
|
||||
}
|
||||
if _, err := r.Seek(i, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(os.Stdout, &io.LimitedReader{
|
||||
R: r,
|
||||
N: j - i,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func encode(r io.Reader) error {
|
||||
|
||||
@@ -479,7 +479,7 @@ func (w *ChunkWriter) Close() error {
|
||||
}
|
||||
}
|
||||
|
||||
w.err = errWriterIsClosed
|
||||
w.err = errAlreadyClosed
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,410 @@
|
||||
// Copyright 2019 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package rac
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
)
|
||||
|
||||
// rWork is a unit of work for concurrent reading. The Manager sends empty
|
||||
// buffers. Workers receive empty buffers and send filled buffers.
|
||||
type rWork struct {
|
||||
dRange Range
|
||||
buf *bytes.Buffer
|
||||
err error
|
||||
}
|
||||
|
||||
// stopWork is a cancel (non-permanent) or close (permanent) notice to the
|
||||
// Manager and Workers. The recipient needs to acknowledge the request by
|
||||
// receiving from ackc (if non-nil).
|
||||
type stopWork struct {
|
||||
ackc <-chan struct{}
|
||||
keepWorking bool
|
||||
}
|
||||
|
||||
type concReader struct {
|
||||
// Channels between the concReader, the Manager and multiple Workers.
|
||||
//
|
||||
// The concReader sends roic and recvs resc.
|
||||
// The Manager recvs roic and sends reqc.
|
||||
// Each Worker recvs reqc and sends resc.
|
||||
roic chan Range // Region of Interest channel.
|
||||
reqc chan rWork // Work-Request channel.
|
||||
resc chan rWork // Work-Response channel.
|
||||
|
||||
// bufc holds re-usable buffers.
|
||||
bufc chan *bytes.Buffer
|
||||
|
||||
// stopc and ackc are used to synchronize the concReader, Manager and
|
||||
// Workers, either canceling work-in-progress or closing everything down.
|
||||
//
|
||||
// Importantly, these are unbuffered channels. Sending and receiving will
|
||||
// wait for the other end to synchronize.
|
||||
stopc chan stopWork
|
||||
ackc chan struct{}
|
||||
|
||||
// currWork holds the unit of work currently being processed by Read. It
|
||||
// will be recycled after Read is done with it, or if a Seek causes us to
|
||||
// cancel the work-in-progress.
|
||||
//
|
||||
// currBytes is currWork's buffer's bytes. currBytes[:currIndex] has
|
||||
// already been sent out via Read. currBytes[currIndex:] has not.
|
||||
currWork rWork
|
||||
currBytes []byte
|
||||
currIndex int
|
||||
|
||||
// completedWorks hold completed units of work that are not the next unit
|
||||
// to be sent out via Read. Works may arrive out of order.
|
||||
//
|
||||
// The map is keyed by an rWork's dRange[0].
|
||||
completedWorks map[int64]rWork
|
||||
|
||||
// numWorkers is the number of concurrent Workers.
|
||||
numWorkers int
|
||||
|
||||
// seekResolved means that Read does not have to seek to pos.
|
||||
//
|
||||
// Each Seek call is relatively cheap, only changing the pos field. The
|
||||
// bulk of the work happens in the first Read call following a Seek call.
|
||||
seekResolved bool
|
||||
|
||||
// seenRead means that we've already seen at least one Read call.
|
||||
seenRead bool
|
||||
|
||||
// pos is the current position, in DSpace. It is the base value when Seek
|
||||
// is called with io.SeekCurrent.
|
||||
pos int64
|
||||
|
||||
// decompressedSize is the size of the RAC file in DSpace.
|
||||
decompressedSize int64
|
||||
}
|
||||
|
||||
func (c *concReader) initialize(racReader *Reader) {
|
||||
if racReader.Concurrency <= 1 {
|
||||
return
|
||||
}
|
||||
c.numWorkers = racReader.Concurrency
|
||||
if c.numWorkers > 65536 {
|
||||
c.numWorkers = 65536
|
||||
}
|
||||
|
||||
// Set up other state.
|
||||
c.completedWorks = map[int64]rWork{}
|
||||
c.decompressedSize = racReader.chunkReader.decompressedSize
|
||||
|
||||
// Set up re-usable buffers to hold decoded bytes. The number of buffers is
|
||||
// arbitrary, but is larger than the number of Workers because individual
|
||||
// pieces of work (the rWork type) may complete in a different order than
|
||||
// they need to be written to dst. Having spare buffers means fewer idle
|
||||
// Workers.
|
||||
c.bufc = make(chan *bytes.Buffer, 2*c.numWorkers)
|
||||
for i := 0; i < cap(c.bufc); i++ {
|
||||
c.bufc <- &bytes.Buffer{}
|
||||
}
|
||||
|
||||
// Set up the Manager and the Workers.
|
||||
c.roic = make(chan Range, 1)
|
||||
c.reqc = make(chan rWork, c.numWorkers)
|
||||
c.resc = make(chan rWork, c.numWorkers)
|
||||
c.ackc = make(chan struct{})
|
||||
c.stopc = make(chan stopWork)
|
||||
for i := 0; i < c.numWorkers; i++ {
|
||||
rr := racReader.clone()
|
||||
rr.Concurrency = 0
|
||||
go runRWorker(c.stopc, c.resc, c.reqc, rr)
|
||||
}
|
||||
go runRManager(c.stopc, c.roic, c.bufc, c.reqc, &racReader.chunkReader)
|
||||
}
|
||||
|
||||
func (c *concReader) ready() bool {
|
||||
return c.stopc != nil
|
||||
}
|
||||
|
||||
func (c *concReader) Close() error {
|
||||
if c.stopc != nil {
|
||||
c.stopAnyWorkInProgress(false)
|
||||
c.stopc = nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *concReader) CloseWithoutWaiting() error {
|
||||
if c.stopc != nil {
|
||||
// Just close the c.stopc channel, which should eventually shut down
|
||||
// the Manager and Worker goroutines. Everything else can be garbage
|
||||
// collected.
|
||||
close(c.stopc)
|
||||
c.stopc = nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *concReader) Seek(offset int64, whence int) (int64, error) {
|
||||
pos := c.pos
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
pos = offset
|
||||
case io.SeekCurrent:
|
||||
pos += offset
|
||||
case io.SeekEnd:
|
||||
pos = c.decompressedSize + offset
|
||||
default:
|
||||
return 0, errSeekToInvalidWhence
|
||||
}
|
||||
|
||||
if c.pos != pos {
|
||||
if pos < 0 {
|
||||
return 0, errSeekToNegativePosition
|
||||
}
|
||||
c.pos = pos
|
||||
c.seekResolved = false
|
||||
}
|
||||
return pos, nil
|
||||
}
|
||||
|
||||
func (c *concReader) Read(p []byte) (int, error) {
|
||||
if c.pos >= c.decompressedSize {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
if !c.seekResolved {
|
||||
c.seekResolved = true
|
||||
if c.seenRead {
|
||||
c.stopAnyWorkInProgress(true)
|
||||
}
|
||||
c.seenRead = true
|
||||
c.roic <- Range{c.pos, c.decompressedSize}
|
||||
}
|
||||
|
||||
for numRead := 0; ; {
|
||||
if c.pos >= c.decompressedSize {
|
||||
return numRead, io.EOF
|
||||
}
|
||||
if len(p) == 0 {
|
||||
return numRead, nil
|
||||
}
|
||||
|
||||
// Fill p from c.currWork.
|
||||
if c.currWork.buf != nil {
|
||||
n := copy(p, c.currBytes[c.currIndex:])
|
||||
p = p[n:]
|
||||
numRead += n
|
||||
c.pos += int64(n)
|
||||
c.currIndex += n
|
||||
err := c.currWork.err
|
||||
|
||||
// Recycle c.currWork if we're done with it.
|
||||
if c.currIndex == len(c.currBytes) {
|
||||
c.bufc <- c.currWork.buf
|
||||
c.currWork = rWork{}
|
||||
c.currBytes = nil
|
||||
c.currIndex = 0
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return numRead, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
work, err := c.nextWork()
|
||||
if err != nil {
|
||||
return numRead, err
|
||||
}
|
||||
c.currWork = work
|
||||
c.currBytes = work.buf.Bytes()
|
||||
c.currIndex = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (c *concReader) nextWork() (rWork, error) {
|
||||
for {
|
||||
if work, ok := c.completedWorks[c.pos]; ok {
|
||||
delete(c.completedWorks, c.pos)
|
||||
return work, nil
|
||||
}
|
||||
|
||||
// This shouldn't happen, but if it does, print a specific error
|
||||
// message instead of the more general "deadlock" message.
|
||||
if len(c.completedWorks) == cap(c.bufc) {
|
||||
return rWork{}, errInternalAllWorkersIdle
|
||||
}
|
||||
|
||||
work := <-c.resc
|
||||
c.completedWorks[work.dRange[0]] = work
|
||||
}
|
||||
}
|
||||
|
||||
// stopAnyWorkInProgress winds up any Manager and Worker work-in-progress.
|
||||
// keepWorking is whether those goroutines should stick around to do future
|
||||
// work. It should be false for closes and true otherwise.
|
||||
func (c *concReader) stopAnyWorkInProgress(keepWorking bool) {
|
||||
// Synchronize the Manager and Workers on stopc (an unbuffered channel).
|
||||
for i, n := 0, 1+c.numWorkers; i < n; i++ {
|
||||
c.stopc <- stopWork{c.ackc, keepWorking}
|
||||
}
|
||||
|
||||
if keepWorking {
|
||||
c.recycleBuffers()
|
||||
}
|
||||
|
||||
// Synchronize the Manager and Workers on ackc (an unbuffered channel).
|
||||
for i, n := 0, 1+c.numWorkers; i < n; i++ {
|
||||
c.ackc <- struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *concReader) recycleBuffers() {
|
||||
if c.currWork.buf != nil {
|
||||
c.bufc <- c.currWork.buf
|
||||
c.currWork = rWork{}
|
||||
c.currBytes = nil
|
||||
c.currIndex = 0
|
||||
}
|
||||
|
||||
for k, work := range c.completedWorks {
|
||||
c.bufc <- work.buf
|
||||
delete(c.completedWorks, k)
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case work := <-c.resc:
|
||||
c.bufc <- work.buf
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runRWorker(stopc <-chan stopWork, resc chan<- rWork, reqc <-chan rWork, racReader *Reader) {
|
||||
input, output := reqc, (chan<- rWork)(nil)
|
||||
work := rWork{}
|
||||
limitedReader := io.LimitedReader{}
|
||||
|
||||
loop:
|
||||
for {
|
||||
select {
|
||||
case stop := <-stopc:
|
||||
if stop.ackc != nil {
|
||||
<-stop.ackc
|
||||
} else {
|
||||
// No need to ack. This is CloseWithoutWaiting.
|
||||
}
|
||||
if !stop.keepWorking {
|
||||
return
|
||||
}
|
||||
continue loop
|
||||
|
||||
case work = <-input:
|
||||
input, output = nil, resc
|
||||
if work.err == nil {
|
||||
_, work.err = racReader.Seek(work.dRange[0], io.SeekStart)
|
||||
}
|
||||
if work.err == nil {
|
||||
limitedReader = io.LimitedReader{
|
||||
R: racReader,
|
||||
N: work.dRange.Size(),
|
||||
}
|
||||
_, work.err = io.Copy(work.buf, &limitedReader)
|
||||
}
|
||||
if work.err == io.EOF {
|
||||
work.err = io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
case output <- work:
|
||||
input, output = reqc, nil
|
||||
work = rWork{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runRManager(stopc <-chan stopWork, roic <-chan Range, bufc <-chan *bytes.Buffer, reqc chan<- rWork, chunkReader *ChunkReader) {
|
||||
input, output := roic, (chan<- rWork)(nil)
|
||||
roi := Range{}
|
||||
work := rWork{}
|
||||
|
||||
loop:
|
||||
for {
|
||||
select {
|
||||
case stop := <-stopc:
|
||||
if stop.ackc != nil {
|
||||
<-stop.ackc
|
||||
} else {
|
||||
// No need to ack. This is CloseWithoutWaiting.
|
||||
}
|
||||
if !stop.keepWorking {
|
||||
return
|
||||
}
|
||||
continue loop
|
||||
|
||||
case roi = <-input:
|
||||
input, output = nil, reqc
|
||||
if err := chunkReader.SeekToChunkContaining(roi[0]); err != nil {
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
work = rWork{err: err}
|
||||
continue loop
|
||||
}
|
||||
|
||||
case output <- work:
|
||||
err := work.err
|
||||
work = rWork{}
|
||||
if err != nil {
|
||||
input, output = roic, nil
|
||||
continue loop
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
chunk, err := chunkReader.NextChunk()
|
||||
if err == io.EOF {
|
||||
input, output = roic, nil
|
||||
continue loop
|
||||
} else if err != nil {
|
||||
work = rWork{err: err}
|
||||
continue loop
|
||||
}
|
||||
|
||||
dr := chunk.DRange.Intersect(roi)
|
||||
if dr[0] >= roi[1] {
|
||||
input, output = roic, nil
|
||||
continue loop
|
||||
} else if dr.Empty() {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case stop := <-stopc:
|
||||
if stop.ackc != nil {
|
||||
<-stop.ackc
|
||||
} else {
|
||||
// No need to ack. This is CloseWithoutWaiting.
|
||||
}
|
||||
if !stop.keepWorking {
|
||||
return
|
||||
}
|
||||
continue loop
|
||||
|
||||
case buf := <-bufc:
|
||||
buf.Reset()
|
||||
work = rWork{dRange: dr, buf: buf}
|
||||
continue loop
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -64,6 +64,7 @@ const (
|
||||
var indexLocationAtEndMagic = []byte("\x72\xC3\x63\x00")
|
||||
|
||||
var (
|
||||
errAlreadyClosed = errors.New("rac: already closed")
|
||||
errCChunkSizeIsTooSmall = errors.New("rac: CChunkSize is too small")
|
||||
errILAEndTempFile = errors.New("rac: IndexLocationAtEnd requires a nil TempFile")
|
||||
errILAStartTempFile = errors.New("rac: IndexLocationAtStart requires a non-nil TempFile")
|
||||
@@ -86,8 +87,8 @@ var (
|
||||
errTooManyResources = errors.New("rac: too many resources")
|
||||
errTooMuchInput = errors.New("rac: too much input")
|
||||
errUnsupportedRACFileVersion = errors.New("rac: unsupported RAC file version")
|
||||
errWriterIsClosed = errors.New("rac: Writer is closed")
|
||||
|
||||
errInternalAllWorkersIdle = errors.New("rac: internal error: all workers idle")
|
||||
errInternalArityIsTooLarge = errors.New("rac: internal error: arity is too large")
|
||||
errInternalInconsistentArity = errors.New("rac: internal error: inconsistent arity")
|
||||
errInternalInconsistentPosition = errors.New("rac: internal error: inconsistent position")
|
||||
|
||||
+61
-7
@@ -46,10 +46,15 @@ type ReaderContext struct {
|
||||
}
|
||||
|
||||
// CodecReader specializes a Reader to decode a specific compression codec.
|
||||
//
|
||||
// Instances are not required to support concurrent use.
|
||||
type CodecReader interface {
|
||||
// Accepts returns whether this CodecReader can decode a Codec.
|
||||
Accepts(c Codec) bool
|
||||
|
||||
// Clone duplicates this. The clone and the original can run concurrently.
|
||||
Clone() CodecReader
|
||||
|
||||
// MakeDecompressor returns the Codec-specific io.Reader for a chunk.
|
||||
//
|
||||
// The returned io.Reader may optionally implement the io.Closer interface,
|
||||
@@ -94,6 +99,13 @@ type Reader struct {
|
||||
// package.
|
||||
CodecReaders []CodecReader
|
||||
|
||||
// Concurrency is how many worker goroutines are used to decode RAC chunks.
|
||||
// Bigger values often lead to faster throughput, up to a
|
||||
// hardware-dependent point, but also larger memory requirements.
|
||||
//
|
||||
// Zero means a non-concurrent (single-goroutine) reader.
|
||||
Concurrency int
|
||||
|
||||
// err is the first error encountered. It is sticky: once a non-nil error
|
||||
// occurs, all public methods will return that error.
|
||||
err error
|
||||
@@ -154,6 +166,9 @@ type Reader struct {
|
||||
|
||||
// zeroes serves the Zeroes Codec.
|
||||
zeroes zeroesReader
|
||||
|
||||
// concReader decodes the RAC-compressed data concurrently.
|
||||
concReader concReader
|
||||
}
|
||||
|
||||
func (r *Reader) initialize() error {
|
||||
@@ -170,14 +185,33 @@ func (r *Reader) initialize() error {
|
||||
return r.err
|
||||
}
|
||||
r.currChunk.R = r.chunkReader.readSeeker
|
||||
r.concReader.initialize(r)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) clone() *Reader {
|
||||
c := &Reader{
|
||||
ReadSeeker: r.ReadSeeker,
|
||||
CompressedSize: r.CompressedSize,
|
||||
CodecReaders: make([]CodecReader, len(r.CodecReaders)),
|
||||
Concurrency: r.Concurrency,
|
||||
}
|
||||
for i := range c.CodecReaders {
|
||||
c.CodecReaders[i] = r.CodecReaders[i].Clone()
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// Read implements io.Reader.
|
||||
func (r *Reader) Read(p []byte) (int, error) {
|
||||
if err := r.initialize(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if r.concReader.ready() {
|
||||
n, err := r.concReader.Read(p)
|
||||
r.err = err
|
||||
return n, err
|
||||
}
|
||||
numRead := 0
|
||||
|
||||
for len(p) > 0 {
|
||||
@@ -382,6 +416,11 @@ func (r *Reader) Seek(offset int64, whence int) (int64, error) {
|
||||
if err := r.initialize(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if r.concReader.ready() {
|
||||
n, err := r.concReader.Seek(offset, whence)
|
||||
r.err = r.err
|
||||
return n, err
|
||||
}
|
||||
|
||||
pos := r.pos
|
||||
switch whence {
|
||||
@@ -390,12 +429,7 @@ func (r *Reader) Seek(offset int64, whence int) (int64, error) {
|
||||
case io.SeekCurrent:
|
||||
pos += offset
|
||||
case io.SeekEnd:
|
||||
end, err := r.chunkReader.DecompressedSize()
|
||||
if err != nil {
|
||||
r.err = err
|
||||
return 0, r.err
|
||||
}
|
||||
pos = end + offset
|
||||
pos = r.chunkReader.decompressedSize + offset
|
||||
default:
|
||||
return 0, errSeekToInvalidWhence
|
||||
}
|
||||
@@ -423,6 +457,26 @@ func (r *Reader) Seek(offset int64, whence int) (int64, error) {
|
||||
}
|
||||
|
||||
// Close implements io.Closer.
|
||||
//
|
||||
// r.ReadSeeker will not be accessed after Close returns. If r.Concurrency is
|
||||
// non-zero, this may involve waiting for various goroutines to shut down,
|
||||
// which may take some time. If the caller does not care about waiting until it
|
||||
// is safe to close or otherwise release the r.ReadSeeker's resources, call
|
||||
// CloseWithoutWaiting instead.
|
||||
func (r *Reader) Close() error {
|
||||
return nil
|
||||
if err := r.initialize(); err != nil {
|
||||
r.concReader.Close()
|
||||
return err
|
||||
}
|
||||
r.err = errAlreadyClosed
|
||||
return r.concReader.Close()
|
||||
}
|
||||
|
||||
func (r *Reader) CloseWithoutWaiting() error {
|
||||
if err := r.initialize(); err != nil {
|
||||
r.concReader.CloseWithoutWaiting()
|
||||
return err
|
||||
}
|
||||
r.err = errAlreadyClosed
|
||||
return r.concReader.CloseWithoutWaiting()
|
||||
}
|
||||
+6
-1
@@ -153,7 +153,12 @@ func (b *writeBuffer) compact() {
|
||||
const NoResourceUsed int = -1
|
||||
|
||||
// CodecWriter specializes a Writer to encode a specific compression codec.
|
||||
//
|
||||
// Instances are not required to support concurrent use.
|
||||
type CodecWriter interface {
|
||||
// Clone duplicates this. The clone and the original can run concurrently.
|
||||
Clone() CodecWriter
|
||||
|
||||
// Compress returns the codec-specific compressed form of the concatenation
|
||||
// of p and q.
|
||||
//
|
||||
@@ -572,6 +577,6 @@ func (w *Writer) Close() error {
|
||||
w.err = err
|
||||
return err
|
||||
}
|
||||
w.err = errWriterIsClosed
|
||||
w.err = errAlreadyClosed
|
||||
return nil
|
||||
}
|
||||
@@ -73,6 +73,11 @@ func (r *CodecReader) Accepts(c rac.Codec) bool {
|
||||
return c == rac.CodecZlib
|
||||
}
|
||||
|
||||
// Clone implements rac.CodecReader.
|
||||
func (r *CodecReader) Clone() rac.CodecReader {
|
||||
return &CodecReader{}
|
||||
}
|
||||
|
||||
// MakeDecompressor implements rac.CodecReader.
|
||||
func (r *CodecReader) MakeDecompressor(
|
||||
compressed io.Reader, rctx rac.ReaderContext) (io.Reader, error) {
|
||||
@@ -166,6 +171,11 @@ type CodecWriter struct {
|
||||
stash []byte
|
||||
}
|
||||
|
||||
// Clone implements rac.CodecWriter.
|
||||
func (r *CodecWriter) Clone() rac.CodecWriter {
|
||||
return &CodecWriter{}
|
||||
}
|
||||
|
||||
// Compress implements rac.CodecWriter.
|
||||
func (w *CodecWriter) Compress(p []byte, q []byte, resourcesData [][]byte) (
|
||||
codec rac.Codec, compressed []byte, secondaryResource int, tertiaryResource int, retErr error) {
|
||||
|
||||
Reference in new issue
Block a user