Fix rac IndexLocationAtEnd for deep index trees

This commit is contained in:
Nigel Tao committed 2019-08-06 18:38:50 +10:00
1 parent b65807d79d
commit db407e37c6
2 files changed
+77 -25

No files matched your search

+42 -14
View File
@@ -378,21 +378,49 @@ func TestMultiLevelIndex(t *testing.T) {
}
func TestWriter1000Chunks(t *testing.T) {
w := &Writer{
Writer: ioutil.Discard,
Codec: fakeCodec,
}
data := make([]byte, 1)
res, _ := w.AddResource(data)
for i := 0; i < 1000; i++ {
if i == 2*255 {
_ = w.AddChunk(1, data, res, 0)
} else {
_ = w.AddChunk(1, data, 0, 0)
loop:
for i := 0; i < 2; i++ {
buf := &bytes.Buffer{}
w := &Writer{
Writer: buf,
Codec: fakeCodec,
}
if i > 0 {
w.IndexLocation = IndexLocationAtStart
w.TempFile = &bytes.Buffer{}
}
data := make([]byte, 1)
res, _ := w.AddResource(data)
for i := 0; i < 1000; i++ {
if i == 2*255 {
_ = w.AddChunk(1, data, res, 0)
} else {
_ = w.AddChunk(1, data, 0, 0)
}
}
if err := w.Close(); err != nil {
t.Errorf("i=%d: Close: %v", i, err)
continue loop
}
encoded := buf.Bytes()
p := &Parser{
ReadSeeker: bytes.NewReader(encoded),
CompressedSize: int64(len(encoded)),
}
for n := 0; ; n++ {
if _, err := p.NextChunk(); err == io.EOF {
if n != 1000 {
t.Errorf("i=%d: number of chunks: got %d, want %d", i, n, 1000)
continue loop
}
break
} else if err != nil {
t.Errorf("i=%d: NextChunk: %v", i, err)
continue loop
}
}
}
if err := w.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
}
+35 -11
View File
@@ -422,7 +422,7 @@ func (w *Writer) Close() error {
return writeEmpty(w.Writer, w.Codec)
}
rootNode := gather(w.leafNodes)
indexSize := rootNode.calcEncodedSize(0)
indexSize := rootNode.calcEncodedSize(0, w.IndexLocation == IndexLocationAtEnd)
nw := &nodeWriter{
w: w.Writer,
@@ -451,7 +451,7 @@ func (w *Writer) Close() error {
}
// Write the index. The compressed data has already been written.
if err := nw.writeIndex(&rootNode); err != nil {
if err := nw.writeIndex(&rootNode, true); err != nil {
w.err = err
return err
}
@@ -460,7 +460,7 @@ func (w *Writer) Close() error {
expectedTempFileSize := w.dataSize
// Write the index.
if err := nw.writeIndex(&rootNode); err != nil {
if err := nw.writeIndex(&rootNode, false); err != nil {
w.err = err
return err
}
@@ -508,18 +508,29 @@ type wNode struct {
// traversing the nodes in depth-first order.
//
// As a side effect, it also sets n.cOffsetCLength for branch nodes.
func (n *wNode) calcEncodedSize(accumulator uint64) (newAccumulator uint64) {
func (n *wNode) calcEncodedSize(accumulator uint64, rootAndIsAtEnd bool) (newAccumulator uint64) {
arity := len(n.children) + len(n.resources)
if arity == 0 {
return accumulator
}
size := (arity * 16) + 16
cLength := calcCLength(size)
if rootAndIsAtEnd {
for i := range n.children {
accumulator = n.children[i].calcEncodedSize(accumulator, false)
}
}
n.cOffsetCLength = accumulator | (cLength << 48)
accumulator += uint64(size)
for i := range n.children {
accumulator = n.children[i].calcEncodedSize(accumulator)
if !rootAndIsAtEnd {
for i := range n.children {
accumulator = n.children[i].calcEncodedSize(accumulator, false)
}
}
return accumulator
}
@@ -538,9 +549,19 @@ type nodeWriter struct {
buffer [4096]byte
}
func (w *nodeWriter) writeIndex(n *wNode) error {
func (w *nodeWriter) writeIndex(n *wNode, rootAndIsAtEnd bool) error {
const tagFF = 0xFF << 56
if rootAndIsAtEnd {
for i, o := range n.children {
if len(o.children) != 0 {
if err := w.writeIndex(&n.children[i], false); err != nil {
return err
}
}
}
}
buf, dPtr := w.buffer[:], uint64(0)
arity := uint64(len(n.children) + len(n.resources))
if arity > 0xFF {
@@ -609,13 +630,16 @@ func (w *nodeWriter) writeIndex(n *wNode) error {
return err
}
for i, o := range n.children {
if len(o.children) != 0 {
if err := w.writeIndex(&n.children[i]); err != nil {
return err
if !rootAndIsAtEnd {
for i, o := range n.children {
if len(o.children) != 0 {
if err := w.writeIndex(&n.children[i], false); err != nil {
return err
}
}
}
}
return nil
}