diff --git a/lib/rac/writer.go b/lib/rac/writer.go index ab5f0639..aa88cbaa 100644 --- a/lib/rac/writer.go +++ b/lib/rac/writer.go @@ -21,6 +21,10 @@ import ( "sort" ) +func isZeroOrAPowerOf2(x uint64) bool { + return (x & (x - 1)) == 0 +} + func putU64LE(b []byte, v uint64) { _ = b[7] // Early bounds check to guarantee safety of writes below. b[0] = byte(v) @@ -85,6 +89,37 @@ type Writer struct { // responsibility of the rac.Writer caller, not the rac.Writer itself. TempFile io.ReadWriter + // CPageSize guides padding the output to minimize the number of pages that + // each chunk occupies (in what the RAC spec calls CSpace). + // + // It must either be zero (which means no padding is inserted) or a power + // of 2, and no larger than MaxSize. + // + // For example, suppose that CSpace is partitioned into 1024-byte pages, + // that 1000 bytes have already been written to the output, and the next + // chunk is 1500 bytes long. + // + // - With no padding (i.e. with CPageSize set to 0), this chunk will + // occupy the half-open range [1000, 2500) in CSpace, which straddles + // three 1024-byte pages: the page [0, 1024), the page [1024, 2048) and + // the page [2048, 3072). Call those pages p0, p1 and p2. + // + // - With padding (i.e. with CPageSize set to 1024), 24 bytes of zeroes + // are first inserted so that this chunk occupies the half-open range + // [1024, 2524) in CSpace, which straddles only two pages (p1 and p2). + // + // This concept is similar, but not identical, to alignment. Even with a + // non-zero CPageSize, chunk start offsets are not necessarily aligned to + // page boundaries. For example, suppose that the chunk size was only 1040 + // bytes, not 1500 bytes. No padding will be inserted, as both [1000, 2040) + // and [1024, 2064) straddle two pages: either pages p0 and p1, or pages p1 + // and p2. + // + // Nonetheless, if all chunks (or all but the final chunk) have a size + // equal to (or just under) a multiple of the page size, then in practice, + // each chunk's starting offset will be aligned to a page boundary. + CPageSize uint64 + // err is the first error encountered. It is sticky: once a non-nil error // occurs, all public methods will return that error. err error @@ -111,6 +146,32 @@ type Writer struct { // leafNodes are the non-resource leaf nodes of the hierarchical index. leafNodes []node + + // log2CPageSize is the base-2 logarithm of CPageSize, or zero if CPageSize + // is zero. + log2CPageSize uint32 + + // padding is a slice of zeroes to pad output to CPageSize boundaries. + padding []byte +} + +func (w *Writer) checkParameters() error { + if w.Writer == nil { + w.err = errors.New("rac: invalid Writer") + return w.err + } + if w.Codec == 0 { + w.err = errors.New("rac: invalid Codec") + return w.err + } + if !isZeroOrAPowerOf2(w.CPageSize) || (w.CPageSize > MaxSize) { + w.err = errors.New("rac: invalid CPageSize") + return w.err + } else if w.CPageSize > 0 { + for w.log2CPageSize = 1; w.CPageSize != (1 << w.log2CPageSize); w.log2CPageSize++ { + } + } + return nil } func (w *Writer) initialize() error { @@ -122,13 +183,8 @@ func (w *Writer) initialize() error { } w.initialized = true - if w.Writer == nil { - w.err = errors.New("rac: invalid Writer") - return w.err - } - if w.Codec == 0 { - w.err = errors.New("rac: invalid Codec") - return w.err + if err := w.checkParameters(); err != nil { + return err } if s, ok := w.TempFile.(io.Seeker); ok { @@ -167,6 +223,11 @@ func (w *Writer) write(data []byte) error { w.err = errors.New("rac: too much input") return w.err } + if w.CPageSize > 0 { + if err := w.writePadding(ioWriter, uint64(len(data))); err != nil { + return err + } + } if _, err := ioWriter.Write(data); err != nil { w.err = err return err @@ -179,6 +240,65 @@ func (w *Writer) write(data []byte) error { return nil } +func (w *Writer) writePadding(ioWriter io.Writer, lenData uint64) error { + if lenData == 0 { + return nil + } + + offset0 := w.dataSize & (w.CPageSize - 1) + offset1WithPadding := (0 * offset0) + lenData + offset1SansPadding := (1 * offset0) + lenData + numPagesWithPadding := (offset1WithPadding + (w.CPageSize - 1)) >> w.log2CPageSize + numPagesSansPadding := (offset1SansPadding + (w.CPageSize - 1)) >> w.log2CPageSize + + if numPagesSansPadding == numPagesWithPadding { + return nil + } + return w.padToPageSize(ioWriter, w.dataSize) +} + +func (w *Writer) padToPageSize(ioWriter io.Writer, offset uint64) error { + offset &= w.CPageSize - 1 + if (offset == 0) || (w.CPageSize == 0) { + return nil + } + + if w.padding == nil { + n := w.CPageSize + if n > 4096 { + n = 4096 + } + w.padding = make([]byte, n) + } + + for remaining := w.CPageSize - offset; remaining > 0; { + padding := w.padding + if remaining < uint64(len(padding)) { + padding = padding[:remaining] + } + + n, err := ioWriter.Write(padding) + if err != nil { + w.err = err + return err + } + remaining -= uint64(n) + w.dataSize += uint64(n) + if w.dataSize > MaxSize { + w.err = errors.New("rac: too much input") + return w.err + } + } + return nil +} + +func (w *Writer) roundUpToCPageBoundary(x uint64) uint64 { + if w.CPageSize == 0 { + return x + } + return (x + w.CPageSize - 1) &^ (w.CPageSize - 1) +} + // AddResource adds a shared resource to the RAC file. It returns a non-zero // OptResource that identifies the resource's bytes. Future calls to AddChunk // can pass these identifiers to indicate that decompressing a chunk depends on @@ -186,23 +306,25 @@ func (w *Writer) write(data []byte) error { // // The caller may modify resource's contents after this method returns. func (w *Writer) AddResource(resource []byte) (OptResource, error) { - if err := w.initialize(); err != nil { - return 0, err - } - if len(w.resourcesCOffsets) >= (1 << 30) { w.err = errors.New("rac: too many resources") return 0, w.err } + + if err := w.initialize(); err != nil { + return 0, err + } + if err := w.write(resource); err != nil { + return 0, err + } + if len(w.resourcesCOffsets) == 0 { w.resourcesCOffsets = make([]uint64, 1, 8) } id := OptResource(len(w.resourcesCOffsets)) + cOffset := w.dataSize - uint64(len(resource)) cLength := calcCLength(len(resource)) - w.resourcesCOffsets = append(w.resourcesCOffsets, w.dataSize|(cLength<<48)) - if err := w.write(resource); err != nil { - return 0, err - } + w.resourcesCOffsets = append(w.resourcesCOffsets, cOffset|(cLength<<48)) return id, nil } @@ -231,16 +353,20 @@ func (w *Writer) AddChunk(dRangeSize uint64, primary []byte, secondary OptResour if err := w.initialize(); err != nil { return err } + if err := w.write(primary); err != nil { + return err + } + cOffset := w.dataSize - uint64(len(primary)) cLength := calcCLength(len(primary)) w.dFileSize += dRangeSize w.leafNodes = append(w.leafNodes, node{ dRangeSize: dRangeSize, - cOffset: w.dataSize | (cLength << 48), + cOffset: cOffset | (cLength << 48), secondary: secondary, tertiary: tertiary, }) - return w.write(primary) + return nil } // Close writes the RAC index to w.Writer and marks that w accepts no further @@ -261,6 +387,11 @@ func (w *Writer) Close() error { if w.err != nil { return w.err } + if !w.initialized { + if err := w.checkParameters(); err != nil { + return err + } + } rootNode := node{} if len(w.leafNodes) == 0 { @@ -270,33 +401,54 @@ func (w *Writer) Close() error { } indexSize := rootNode.calcEncodedSize(0) - if (indexSize + w.dataSize) > MaxSize { - w.err = errors.New("rac: too much input") - return w.err - } nw := &nodeWriter{ w: w.Writer, resourcesCOffsets: w.resourcesCOffsets, - cFileSize: indexSize + w.dataSize, codec: w.Codec, } if w.IndexLocation == IndexLocationAtEnd { + nw.indexCOffset = w.roundUpToCPageBoundary(w.dataSize) + nw.cFileSize = nw.indexCOffset + indexSize + } else { + nw.dataCOffset = w.roundUpToCPageBoundary(indexSize) + nw.cFileSize = nw.dataCOffset + w.dataSize + } + if nw.cFileSize > MaxSize { + w.err = errors.New("rac: too much input") + return w.err + } + + if w.IndexLocation == IndexLocationAtEnd { + // Write the align-to-CPageSize padding. + if w.CPageSize > 0 { + if err := w.padToPageSize(w.Writer, w.dataSize); err != nil { + return err + } + } + // Write the index. The compressed data has already been written. - nw.indexCOffset = w.dataSize if err := nw.writeIndex(&rootNode); err != nil { w.err = err return err } } else { + expectedTempFileSize := w.dataSize + // Write the index. - nw.dataCOffset = indexSize if err := nw.writeIndex(&rootNode); err != nil { w.err = err return err } + // Write the align-to-CPageSize padding. + if w.CPageSize > 0 { + if err := w.padToPageSize(w.Writer, indexSize); err != nil { + return err + } + } + // Write the compressed data. if s, ok := w.TempFile.(io.Seeker); ok { if _, err := s.Seek(w.tempFileSeekStart, io.SeekStart); err != nil { @@ -307,7 +459,7 @@ func (w *Writer) Close() error { if n, err := io.Copy(w.Writer, w.TempFile); err != nil { w.err = err return err - } else if uint64(n) != w.dataSize { + } else if uint64(n) != expectedTempFileSize { w.err = errors.New("rac: inconsistent compressed data size") return w.err } diff --git a/lib/rac/writer_test.go b/lib/rac/writer_test.go index bd9a0f66..da200d8e 100644 --- a/lib/rac/writer_test.go +++ b/lib/rac/writer_test.go @@ -40,6 +40,16 @@ const writerWantILAEnd = "" + "00000060 00 00 01 ff 0c 00 00 00 00 00 01 00 10 00 00 00 |................|\n" + "00000070 00 00 01 00 7c 00 00 00 00 00 01 05 |....|.......|\n" +const writerWantILAEndCPageSize8 = "" + + "00000000 72 c3 63 00 52 72 72 00 53 73 41 61 61 00 00 00 |r.c.Rrr.SsAaa...|\n" + + "00000010 42 62 62 62 43 63 63 63 63 63 63 63 63 63 31 32 |BbbbCccccccccc12|\n" + + "00000020 72 c3 63 05 90 5e 00 ff 00 00 00 00 00 00 00 ff |r.c..^..........|\n" + + "00000030 00 00 00 00 00 00 00 ff 11 00 00 00 00 00 00 ff |................|\n" + + "00000040 33 00 00 00 00 00 00 01 77 00 00 00 00 00 00 ee |3.......w.......|\n" + + "00000050 04 00 00 00 00 00 01 ff 08 00 00 00 00 00 01 ff |................|\n" + + "00000060 0a 00 00 00 00 00 01 ff 10 00 00 00 00 00 01 00 |................|\n" + + "00000070 14 00 00 00 00 00 01 00 80 00 00 00 00 00 01 05 |................|\n" + const writerWantILAStart = "" + "00000000 72 c3 63 05 bc dc 00 ff 00 00 00 00 00 00 00 ff |r.c.............|\n" + "00000010 00 00 00 00 00 00 00 ff 11 00 00 00 00 00 00 ff |................|\n" + @@ -50,15 +60,37 @@ const writerWantILAStart = "" + "00000060 52 72 72 53 73 41 61 61 42 62 62 62 43 63 63 63 |RrrSsAaaBbbbCccc|\n" + "00000070 63 63 63 63 63 63 31 32 |cccccc12|\n" +const writerWantILAStartCPageSize4 = "" + + "00000000 72 c3 63 05 fc 4c 00 ff 00 00 00 00 00 00 00 ff |r.c..L..........|\n" + + "00000010 00 00 00 00 00 00 00 ff 11 00 00 00 00 00 00 ff |................|\n" + + "00000020 33 00 00 00 00 00 00 01 77 00 00 00 00 00 00 ee |3.......w.......|\n" + + "00000030 60 00 00 00 00 00 01 ff 64 00 00 00 00 00 01 ff |`.......d.......|\n" + + "00000040 68 00 00 00 00 00 01 ff 6c 00 00 00 00 00 01 00 |h.......l.......|\n" + + "00000050 70 00 00 00 00 00 01 00 7c 00 00 00 00 00 01 05 |p.......|.......|\n" + + "00000060 52 72 72 00 53 73 00 00 41 61 61 00 42 62 62 62 |Rrr.Ss..Aaa.Bbbb|\n" + + "00000070 43 63 63 63 63 63 63 63 63 63 31 32 |Cccccccccc12|\n" + +const writerWantILAStartCPageSize128 = "" + + "00000000 72 c3 63 05 d8 df 00 ff 00 00 00 00 00 00 00 ff |r.c.............|\n" + + "00000010 00 00 00 00 00 00 00 ff 11 00 00 00 00 00 00 ff |................|\n" + + "00000020 33 00 00 00 00 00 00 01 77 00 00 00 00 00 00 ee |3.......w.......|\n" + + "00000030 80 00 00 00 00 00 01 ff 83 00 00 00 00 00 01 ff |................|\n" + + "00000040 85 00 00 00 00 00 01 ff 88 00 00 00 00 00 01 00 |................|\n" + + "00000050 8c 00 00 00 00 00 01 00 98 00 00 00 00 00 01 05 |................|\n" + + "00000060 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|\n" + + "00000070 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|\n" + + "00000080 52 72 72 53 73 41 61 61 42 62 62 62 43 63 63 63 |RrrSsAaaBbbbCccc|\n" + + "00000090 63 63 63 63 63 63 31 32 |cccccc12|\n" + func TestWriterILAEndNoTempFile(t *testing.T) { - if err := testWriter(IndexLocationAtEnd, nil); err != nil { + if err := testWriter(IndexLocationAtEnd, nil, 0); err != nil { t.Fatal(err) } } func TestWriterILAEndMemTempFile(t *testing.T) { tempFile := &bytes.Buffer{} - if err := testWriter(IndexLocationAtEnd, tempFile); err == nil { + if err := testWriter(IndexLocationAtEnd, tempFile, 0); err == nil { t.Fatal("err: got nil, want non-nil") } else if !strings.HasPrefix(err.Error(), "rac: IndexLocationAtEnd requires") { t.Fatal(err) @@ -66,7 +98,7 @@ func TestWriterILAEndMemTempFile(t *testing.T) { } func TestWriterILAStartNoTempFile(t *testing.T) { - if err := testWriter(IndexLocationAtStart, nil); err == nil { + if err := testWriter(IndexLocationAtStart, nil, 0); err == nil { t.Fatal("err: got nil, want non-nil") } else if !strings.HasPrefix(err.Error(), "rac: IndexLocationAtStart requires") { t.Fatal(err) @@ -75,7 +107,7 @@ func TestWriterILAStartNoTempFile(t *testing.T) { func TestWriterILAStartMemTempFile(t *testing.T) { tempFile := &bytes.Buffer{} - if err := testWriter(IndexLocationAtStart, tempFile); err != nil { + if err := testWriter(IndexLocationAtStart, tempFile, 0); err != nil { t.Fatal(err) } } @@ -88,12 +120,32 @@ func TestWriterILAStartRealTempFile(t *testing.T) { defer os.Remove(f.Name()) defer f.Close() - if err := testWriter(IndexLocationAtStart, f); err != nil { + if err := testWriter(IndexLocationAtStart, f, 0); err != nil { t.Fatal(err) } } -func testWriter(iloc IndexLocation, tempFile io.ReadWriter) error { +func TestWriterILAEndCPageSize8(t *testing.T) { + if err := testWriter(IndexLocationAtEnd, nil, 8); err != nil { + t.Fatal(err) + } +} + +func TestWriterILAStartCPageSize4(t *testing.T) { + tempFile := &bytes.Buffer{} + if err := testWriter(IndexLocationAtStart, tempFile, 4); err != nil { + t.Fatal(err) + } +} + +func TestWriterILAStartCPageSize128(t *testing.T) { + tempFile := &bytes.Buffer{} + if err := testWriter(IndexLocationAtStart, tempFile, 128); err != nil { + t.Fatal(err) + } +} + +func testWriter(iloc IndexLocation, tempFile io.ReadWriter, cPageSize uint64) error { buf := &bytes.Buffer{} const fakeCodec = Codec(0xEE) w := &Writer{ @@ -101,6 +153,7 @@ func testWriter(iloc IndexLocation, tempFile io.ReadWriter) error { Codec: fakeCodec, IndexLocation: iloc, TempFile: tempFile, + CPageSize: cPageSize, } // We ignore errors (assigning them to _) from the AddXxx calls. Any @@ -115,9 +168,20 @@ func testWriter(iloc IndexLocation, tempFile io.ReadWriter) error { } got := hex.Dump(buf.Bytes()) - want := writerWantILAEnd - if iloc == IndexLocationAtStart { + want := "" + switch { + case (iloc == IndexLocationAtEnd) && (cPageSize == 0): + want = writerWantILAEnd + case (iloc == IndexLocationAtEnd) && (cPageSize == 8): + want = writerWantILAEndCPageSize8 + case (iloc == IndexLocationAtStart) && (cPageSize == 0): want = writerWantILAStart + case (iloc == IndexLocationAtStart) && (cPageSize == 4): + want = writerWantILAStartCPageSize4 + case (iloc == IndexLocationAtStart) && (cPageSize == 128): + want = writerWantILAStartCPageSize128 + default: + return fmt.Errorf("unsupported iloc/cPageSize combination") } if got != want {