From db407e37c6ce327225d4c37d72a2d513ff9aab0d Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Tue, 6 Aug 2019 18:38:50 +1000 Subject: [PATCH] Fix rac IndexLocationAtEnd for deep index trees --- lib/rac/rac_test.go | 56 +++++++++++++++++++++++++++++++++------------ lib/rac/writer.go | 46 ++++++++++++++++++++++++++++--------- 2 files changed, 77 insertions(+), 25 deletions(-) diff --git a/lib/rac/rac_test.go b/lib/rac/rac_test.go index 7d59a72d..d5feece7 100644 --- a/lib/rac/rac_test.go +++ b/lib/rac/rac_test.go @@ -378,21 +378,49 @@ func TestMultiLevelIndex(t *testing.T) { } func TestWriter1000Chunks(t *testing.T) { - w := &Writer{ - Writer: ioutil.Discard, - Codec: fakeCodec, - } - data := make([]byte, 1) - res, _ := w.AddResource(data) - for i := 0; i < 1000; i++ { - if i == 2*255 { - _ = w.AddChunk(1, data, res, 0) - } else { - _ = w.AddChunk(1, data, 0, 0) +loop: + for i := 0; i < 2; i++ { + buf := &bytes.Buffer{} + w := &Writer{ + Writer: buf, + Codec: fakeCodec, + } + if i > 0 { + w.IndexLocation = IndexLocationAtStart + w.TempFile = &bytes.Buffer{} + } + + data := make([]byte, 1) + res, _ := w.AddResource(data) + for i := 0; i < 1000; i++ { + if i == 2*255 { + _ = w.AddChunk(1, data, res, 0) + } else { + _ = w.AddChunk(1, data, 0, 0) + } + } + if err := w.Close(); err != nil { + t.Errorf("i=%d: Close: %v", i, err) + continue loop + } + + encoded := buf.Bytes() + p := &Parser{ + ReadSeeker: bytes.NewReader(encoded), + CompressedSize: int64(len(encoded)), + } + for n := 0; ; n++ { + if _, err := p.NextChunk(); err == io.EOF { + if n != 1000 { + t.Errorf("i=%d: number of chunks: got %d, want %d", i, n, 1000) + continue loop + } + break + } else if err != nil { + t.Errorf("i=%d: NextChunk: %v", i, err) + continue loop + } } - } - if err := w.Close(); err != nil { - t.Fatalf("Close: %v", err) } } diff --git a/lib/rac/writer.go b/lib/rac/writer.go index ece169ab..48ad82bf 100644 --- a/lib/rac/writer.go +++ b/lib/rac/writer.go @@ -422,7 +422,7 @@ func (w *Writer) Close() error { return writeEmpty(w.Writer, w.Codec) } rootNode := gather(w.leafNodes) - indexSize := rootNode.calcEncodedSize(0) + indexSize := rootNode.calcEncodedSize(0, w.IndexLocation == IndexLocationAtEnd) nw := &nodeWriter{ w: w.Writer, @@ -451,7 +451,7 @@ func (w *Writer) Close() error { } // Write the index. The compressed data has already been written. - if err := nw.writeIndex(&rootNode); err != nil { + if err := nw.writeIndex(&rootNode, true); err != nil { w.err = err return err } @@ -460,7 +460,7 @@ func (w *Writer) Close() error { expectedTempFileSize := w.dataSize // Write the index. - if err := nw.writeIndex(&rootNode); err != nil { + if err := nw.writeIndex(&rootNode, false); err != nil { w.err = err return err } @@ -508,18 +508,29 @@ type wNode struct { // traversing the nodes in depth-first order. // // As a side effect, it also sets n.cOffsetCLength for branch nodes. -func (n *wNode) calcEncodedSize(accumulator uint64) (newAccumulator uint64) { +func (n *wNode) calcEncodedSize(accumulator uint64, rootAndIsAtEnd bool) (newAccumulator uint64) { arity := len(n.children) + len(n.resources) if arity == 0 { return accumulator } size := (arity * 16) + 16 cLength := calcCLength(size) + + if rootAndIsAtEnd { + for i := range n.children { + accumulator = n.children[i].calcEncodedSize(accumulator, false) + } + } + n.cOffsetCLength = accumulator | (cLength << 48) accumulator += uint64(size) - for i := range n.children { - accumulator = n.children[i].calcEncodedSize(accumulator) + + if !rootAndIsAtEnd { + for i := range n.children { + accumulator = n.children[i].calcEncodedSize(accumulator, false) + } } + return accumulator } @@ -538,9 +549,19 @@ type nodeWriter struct { buffer [4096]byte } -func (w *nodeWriter) writeIndex(n *wNode) error { +func (w *nodeWriter) writeIndex(n *wNode, rootAndIsAtEnd bool) error { const tagFF = 0xFF << 56 + if rootAndIsAtEnd { + for i, o := range n.children { + if len(o.children) != 0 { + if err := w.writeIndex(&n.children[i], false); err != nil { + return err + } + } + } + } + buf, dPtr := w.buffer[:], uint64(0) arity := uint64(len(n.children) + len(n.resources)) if arity > 0xFF { @@ -609,13 +630,16 @@ func (w *nodeWriter) writeIndex(n *wNode) error { return err } - for i, o := range n.children { - if len(o.children) != 0 { - if err := w.writeIndex(&n.children[i]); err != nil { - return err + if !rootAndIsAtEnd { + for i, o := range n.children { + if len(o.children) != 0 { + if err := w.writeIndex(&n.children[i], false); err != nil { + return err + } } } } + return nil }