From ad8b1540c3dadc434553f10751565302faafed10 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sun, 7 Apr 2019 10:08:42 +1000 Subject: [PATCH] Add CLen fields to the RAC file format --- doc/spec/rac-spec.md | 68 +++++++++++++++++++++++++---------------- lib/rac/example_test.go | 32 +++++++++++-------- lib/rac/rac.go | 4 +-- lib/rac/writer.go | 47 ++++++++++++++++++++-------- lib/rac/writer_test.go | 21 ++++++++----- 5 files changed, 110 insertions(+), 62 deletions(-) diff --git a/doc/spec/rac-spec.md b/doc/spec/rac-spec.md index 3dbee283..fbd45f9e 100644 --- a/doc/spec/rac-spec.md +++ b/doc/spec/rac-spec.md @@ -135,7 +135,7 @@ Within those unsigned integers, bit 0 is the least significant bit and e.g. bit 31 is the most significant bit of a `uint32_t`. The maximum supported `CFileSize` and the maximum supported `DFileSize` are the -same number: `0xFF_FFFF_FFFF_FFFF`, which is `((1 << 56) - 1)`. +same number: `0x0000_FFFF_FFFF_FFFF`, which is `((1 << 48) - 1)`. ## File Structure @@ -171,24 +171,25 @@ COffset`: +-+-+-+-+-+-+-+-+ |0|1|2|3|4|5|6|7| +-+-+-+-+-+-+-+-+ - |Magic|A|Che|V|T| Magic, Arity, Checksum, Version, TTag[0] - | DPtr[1] |T| DPtr[1], TTag[1] - | DPtr[2] |T| DPtr[2], TTag[2] - | ... |.| ..., ... - | DPtr[A-2] |T| DPtr[Arity-2], TTag[Arity-2] - | DPtr[A-1] |T| DPtr[Arity-1], TTag[Arity-1] - | DPtr[A] |C| DPtr[Arity] a.k.a. DPtrMax, Codec - | CPtr[0] |S| CPtr[0], STag[0] - | CPtr[1] |S| CPtr[1], STag[1] - | CPtr[2] |S| CPtr[2], STag[2] - | ... |.| ..., ... - | CPtr[A-2] |S| CPtr[Arity-2], STag[Arity-2] - | CPtr[A-1] |S| CPtr[Arity-1], STag[Arity-1] - | CPtr[A] |A| CPtr[Arity] a.k.a. CPtrMax, Arity + |Magic|A|Che|0|T| Magic, Arity, Checksum, Reserved (0), TTag[0] + | DPtr[1] |0|T| DPtr[1], Reserved (0), TTag[1] + | DPtr[2] |0|T| DPtr[2], Reserved (0), TTag[2] + | ... |0|.| ..., Reserved (0), ... + | DPtr[A-2] |0|T| DPtr[Arity-2], Reserved (0), TTag[Arity-2] + | DPtr[A-1] |0|T| DPtr[Arity-1], Reserved (0), TTag[Arity-1] + | DPtr[A] |0|C| DPtr[Arity] a.k.a. DPtrMax, Reserved (0), Codec + | CPtr[0] |L|S| CPtr[0], CLen[0], STag[0] + | CPtr[1] |L|S| CPtr[1], CLen[1], STag[1] + | CPtr[2] |L|S| CPtr[2], CLen[2], STag[2] + | ... |L|.| ..., ..., ... + | CPtr[A-2] |L|S| CPtr[Arity-2], CLen[Arity-2], STag[Arity-2] + | CPtr[A-1] |L|S| CPtr[Arity-1], CLen[Arity-2], STag[Arity-1] + | CPtr[A] |V|A| CPtr[Arity] a.k.a. CPtrMax, Version, Arity +-+-+-+-+-+-+-+-+ -For the `(XPtr | Other)` 8 byte fields, the `XPtr` occupies the low 56 bits (as -a little-endian `uint64_t`) and the `Other` occupies the high 8 bits. +For the `(XPtr | Other6 | Other7)` 8 byte fields, the `XPtr` occupies the low +48 bits (as a little-endian `uint64_t`) and the `Other` fields occupy the high +16 bits. The `CPtr` and `DPtr` values are what is explicitly written in the `CFile`'s bytes. These are added to a `Branch Node`'s implicit `Branch CBias` and `Branch @@ -248,9 +249,9 @@ the high 16 bits of the `uint32_t` CRC-32 IEEE checksum of the `((Arity * 16) + the `Arity` byte near the start is replicated by the `Arity` byte at the end. -### Version +### Reserved (0) -`Version` must have the value `0x01`, indicating version 1 of the RAC format. +The `Reserved (0)` bytes must have the value `0x00`. ### COffs and DOffs, STags and TTags @@ -277,12 +278,22 @@ A child `Branch Node`'s `SubBranch COffset` is defined to be `COff[a]`. Its A child `Leaf Node`'s `STag[a]` and `TTag[a]` values are also called its `Leaf STag` and `Leaf TTag`. It also has: - - A `Primary CRange`, equal to `[COff[a] .. COffMax)`. - - A `Secondary CRange`, equal to `[COff[STag[a]] .. COffMax)` when - `(STag[a] < Arity)`. If `(STag[a] >= Arity)` then the `Secondary CRange` is - the empty range `[COffMax .. COffMax)`. - - A `Tertiary CRange`, similar to the `Secondary CRange`, but using `TTag[a]` - instead of `STag[a]`. + - A `Primary CRange`, equal to `MakeCRange(a)`. + - A `Secondary CRange`, equal to `MakeCRange(STag[a])`. + - A `Tertiary CRange`, equal to `MakeCRange(TTag[a])`. + +The `MakeCRange(i)` function defines a `CRange`. If `(i >= Arity)` then that +`CRange` is the empty range `[COffMax .. COffMax)`. Otherwise, the lower bound +is `COff[i]` and the upper bound is: + + - `COffMax` when `CLen[i]` is zero. + - The minimum of `COffMax` and `(COff[i] + (CLen[i] * 1024))` when `CLen[i]` + is non-zero. + +In other words, the `COffMax` value clamps the `CRange` upper bound. The `CLen` +value, if non-zero, combines with the `COff` value to apply another clamp. The +`CLen` is given in units of 1024 bytes, but the `(COff[i] + (CLen[i] * 1024))` +value is not necessarily quantized to 1024 byte boundaries. Note that, since `Arity` is at most 255, an `STag[a]` of `0xFF` always results in a `CNeutral Branch Node` or an empty `Secondary CRange`. Likewise, a @@ -309,6 +320,11 @@ RAC files in `CSpace` and then appending a new `Root Node` with `CBiasing Branch Node`s pointing to each source RAC file's `Root Node`. +### Version + +`Version` must have the value `0x01`, indicating version 1 of the RAC format. + + ### Codec `Codec`s define specializations of RAC, such as "RAC + Zlib" or "RAC + Brotli". @@ -531,4 +547,4 @@ review. --- -Updated on March 2019. +Updated on April 2019. diff --git a/lib/rac/example_test.go b/lib/rac/example_test.go index 4e8f7c44..26156492 100644 --- a/lib/rac/example_test.go +++ b/lib/rac/example_test.go @@ -34,12 +34,6 @@ import ( // // See the RAC specification for an explanation of the file format. func ExampleILAEnd() { - buf := &bytes.Buffer{} - w := &rac.Writer{ - Writer: buf, - Codec: rac.CodecZlib, - } - // Manually construct a zlib encoding of "More!\n", one that uses a literal // block (that's easy to see in a hex dump) instead of a more compressible // Huffman block. @@ -72,6 +66,11 @@ func ExampleILAEnd() { } } + buf := &bytes.Buffer{} + w := &rac.Writer{ + Writer: buf, + Codec: rac.CodecZlib, + } if err := w.AddChunk(uint64(len(src)), enc, 0, 0); err != nil { log.Fatalf("AddChunk: %v", err) } @@ -84,9 +83,9 @@ func ExampleILAEnd() { // Output: // RAC file: // 00000000 72 c3 63 00 78 9c 01 06 00 f9 ff 4d 6f 72 65 21 |r.c.x......More!| - // 00000010 0a 07 42 01 bf 72 c3 63 01 49 44 01 ff 06 00 00 |..B..r.c.ID.....| - // 00000020 00 00 00 00 01 04 00 00 00 00 00 00 ff 35 00 00 |.............5..| - // 00000030 00 00 00 00 01 |.....| + // 00000010 0a 07 42 01 bf 72 c3 63 01 65 a9 00 ff 06 00 00 |..B..r.c.e......| + // 00000020 00 00 00 00 01 04 00 00 00 00 00 01 ff 35 00 00 |.............5..| + // 00000030 00 00 00 01 01 |.....| } // ExampleILAStart demonstrates using the low level "rac" package to encode and @@ -157,6 +156,12 @@ func ExampleILAStart() { // TODO: decode the encoded bytes (the RAC-formatted bytes) to recover the // original "One sheep.\nTwo sheep\.Three sheep.\n" source. + // Note that these exact bytes depends on the zlib encoder's algorithm, but + // there is more than one valid zlib encoding of any given input. This + // "compare to golden output" test is admittedly brittle, as the standard + // library's zlib package's output isn't necessarily stable across Go + // releases. + // Output: // Encoded dictionary resource: // 00000000 08 00 20 73 68 65 65 70 2e 0a 0b e0 02 6e |.. sheep.....n| @@ -174,14 +179,15 @@ func ExampleILAStart() { // 00000010 00 ff ff 21 6e 04 66 |...!n.f| // // RAC file: - // 00000000 72 c3 63 04 4e 72 01 ff 00 00 00 00 00 00 00 ff |r.c.Nr..........| + // 00000000 72 c3 63 04 71 b5 00 ff 00 00 00 00 00 00 00 ff |r.c.q...........| // 00000010 0b 00 00 00 00 00 00 ff 16 00 00 00 00 00 00 ff |................| - // 00000020 23 00 00 00 00 00 00 01 50 00 00 00 00 00 00 ff |#.......P.......| - // 00000030 5e 00 00 00 00 00 00 00 73 00 00 00 00 00 00 00 |^.......s.......| - // 00000040 88 00 00 00 00 00 00 00 9f 00 00 00 00 00 00 04 |................| + // 00000020 23 00 00 00 00 00 00 01 50 00 00 00 00 00 01 ff |#.......P.......| + // 00000030 5e 00 00 00 00 00 01 00 73 00 00 00 00 00 01 00 |^.......s.......| + // 00000040 88 00 00 00 00 00 01 00 9f 00 00 00 00 00 01 04 |................| // 00000050 08 00 20 73 68 65 65 70 2e 0a 0b e0 02 6e 78 f9 |.. sheep.....nx.| // 00000060 0b e0 02 6e f2 cf 4b 85 31 01 01 00 00 ff ff 17 |...n..K.1.......| // 00000070 21 03 90 78 f9 0b e0 02 6e 0a 29 cf 87 31 01 01 |!..x....n.)..1..| // 00000080 00 00 ff ff 18 0c 03 a8 78 f9 0b e0 02 6e 0a c9 |........x....n..| // 00000090 28 4a 4d 85 71 00 01 00 00 ff ff 21 6e 04 66 |(JM.q......!n.f| + } diff --git a/lib/rac/rac.go b/lib/rac/rac.go index 5e6ce67e..f6c8cbca 100644 --- a/lib/rac/rac.go +++ b/lib/rac/rac.go @@ -26,8 +26,8 @@ package rac const ( - // MaxSize is ((1 << 56) - 1). - MaxSize = 0x00FFFFFFFFFFFFFF + // MaxCSize is the maximum RAC file size (in both CSpace and DSpace). + MaxSize = (1 << 48) - 1 // invalidCOffset is ((1 << 64) - 1). invalidCOffset = 0xFFFFFFFFFFFFFFFF diff --git a/lib/rac/writer.go b/lib/rac/writer.go index 2063be48..810396fc 100644 --- a/lib/rac/writer.go +++ b/lib/rac/writer.go @@ -190,18 +190,21 @@ func (w *Writer) AddResource(resource []byte) (OptResource, error) { w.resourcesCOffsets = make([]uint64, 1, 8) } id := OptResource(len(w.resourcesCOffsets)) - w.resourcesCOffsets = append(w.resourcesCOffsets, w.dataSize) + cLength := calcCLength(len(resource)) + w.resourcesCOffsets = append(w.resourcesCOffsets, w.dataSize|(cLength<<48)) if err := w.write(resource); err != nil { return 0, err } return id, nil } -// AddChunk adds a chunk of compressed data (the [primary, secondary, tertiary] -// tuple) to the RAC file. Decompressing that chunk should produce dRangeSize -// bytes, although the rac.Writer does not attempt to verify that. +// AddChunk adds a chunk of compressed data - the (primary, secondary, +// tertiary) tuple - to the RAC file. Decompressing that chunk should produce +// dRangeSize bytes, although the rac.Writer does not attempt to verify that. // -// The OptResource arguments may be zero, meaning that no resource is used. +// The OptResource arguments may be zero, meaning that no resource is used. In +// that case, the corresponding STag or TTag (see the RAC specification for +// further discussion) will be 0xFF. // // The caller may modify primary's contents after this method returns. func (w *Writer) AddChunk(dRangeSize uint64, primary []byte, secondary OptResource, tertiary OptResource) error { @@ -217,10 +220,11 @@ func (w *Writer) AddChunk(dRangeSize uint64, primary []byte, secondary OptResour w.err = errors.New("rac: dRangeSize is too large") return w.err } + cLength := calcCLength(len(primary)) w.dFileSize += dRangeSize w.leafNodes = append(w.leafNodes, node{ dRangeSize: dRangeSize, - cOffset: w.dataSize, + cOffset: w.dataSize | (cLength << 48), secondary: secondary, tertiary: tertiary, }) @@ -313,11 +317,14 @@ type node struct { // // As a side effect, it also sets n.cOffset for branch nodes. func (n *node) calcEncodedSize(accumulator uint64) (newAccumulator uint64) { - if len(n.children)+len(n.resources) == 0 { + arity := len(n.children) + len(n.resources) + if arity == 0 { return accumulator } - n.cOffset = accumulator - accumulator += (uint64(len(n.children)+len(n.resources)) * 16) + 16 + size := (arity * 16) + 16 + cLength := calcCLength(size) + n.cOffset = accumulator | (cLength << 48) + accumulator += uint64(size) for i := range n.children { accumulator = n.children[i].calcEncodedSize(accumulator) } @@ -386,16 +393,16 @@ func (w *nodeWriter) writeIndex(n *node) error { buf = buf[8*len(n.children):] // CPtrMax. - putU64LE(buf, w.cFileSize|(arity<<56)) + const version = 0x01 + putU64LE(buf, w.cFileSize|(version<<48)|(arity<<56)) - // Magic, Arity, (Checksum is skipped), Version. + // Magic and Arity. w.buffer[0] = 0x72 w.buffer[1] = 0xC3 w.buffer[2] = 0x63 w.buffer[3] = uint8(arity) - w.buffer[6] = 0x01 - // Checksum (which depends on the Version). + // Checksum. size := (arity * 16) + 16 checksum := crc32.ChecksumIEEE(w.buffer[6:size]) checksum ^= checksum >> 16 @@ -406,6 +413,20 @@ func (w *nodeWriter) writeIndex(n *node) error { return err } +func calcCLength(primarySize int) uint64 { + if primarySize <= 0 { + return 1 + } + + // Divide by 1024, rounding up. + primarySize = ((primarySize - 1) >> 10) + 1 + + if primarySize > 255 { + return 0 + } + return uint64(primarySize) +} + func resourceToTag(resources []int, r OptResource) uint64 { if r != 0 { for i, res := range resources { diff --git a/lib/rac/writer_test.go b/lib/rac/writer_test.go index e992c008..bd9a0f66 100644 --- a/lib/rac/writer_test.go +++ b/lib/rac/writer_test.go @@ -25,23 +25,28 @@ import ( "testing" ) +// Note that these exact bytes depends on the zlib encoder's algorithm, but +// there is more than one valid zlib encoding of any given input. This "compare +// to golden output" test is admittedly brittle, as the standard library's zlib +// package's output isn't necessarily stable across Go releases. + const writerWantILAEnd = "" + "00000000 72 c3 63 00 52 72 72 53 73 41 61 61 42 62 62 62 |r.c.RrrSsAaaBbbb|\n" + "00000010 43 63 63 63 63 63 63 63 63 63 31 32 72 c3 63 05 |Cccccccccc12r.c.|\n" + - "00000020 99 75 01 ff 00 00 00 00 00 00 00 ff 00 00 00 00 |.u..............|\n" + + "00000020 1d 45 00 ff 00 00 00 00 00 00 00 ff 00 00 00 00 |.E..............|\n" + "00000030 00 00 00 ff 11 00 00 00 00 00 00 ff 33 00 00 00 |............3...|\n" + "00000040 00 00 00 01 77 00 00 00 00 00 00 ee 04 00 00 00 |....w...........|\n" + - "00000050 00 00 00 ff 07 00 00 00 00 00 00 ff 09 00 00 00 |................|\n" + - "00000060 00 00 00 ff 0c 00 00 00 00 00 00 00 10 00 00 00 |................|\n" + - "00000070 00 00 00 00 7c 00 00 00 00 00 00 05 |....|.......|\n" + "00000050 00 00 01 ff 07 00 00 00 00 00 01 ff 09 00 00 00 |................|\n" + + "00000060 00 00 01 ff 0c 00 00 00 00 00 01 00 10 00 00 00 |................|\n" + + "00000070 00 00 01 00 7c 00 00 00 00 00 01 05 |....|.......|\n" const writerWantILAStart = "" + - "00000000 72 c3 63 05 38 ec 01 ff 00 00 00 00 00 00 00 ff |r.c.8...........|\n" + + "00000000 72 c3 63 05 bc dc 00 ff 00 00 00 00 00 00 00 ff |r.c.............|\n" + "00000010 00 00 00 00 00 00 00 ff 11 00 00 00 00 00 00 ff |................|\n" + "00000020 33 00 00 00 00 00 00 01 77 00 00 00 00 00 00 ee |3.......w.......|\n" + - "00000030 60 00 00 00 00 00 00 ff 63 00 00 00 00 00 00 ff |`.......c.......|\n" + - "00000040 65 00 00 00 00 00 00 ff 68 00 00 00 00 00 00 00 |e.......h.......|\n" + - "00000050 6c 00 00 00 00 00 00 00 78 00 00 00 00 00 00 05 |l.......x.......|\n" + + "00000030 60 00 00 00 00 00 01 ff 63 00 00 00 00 00 01 ff |`.......c.......|\n" + + "00000040 65 00 00 00 00 00 01 ff 68 00 00 00 00 00 01 00 |e.......h.......|\n" + + "00000050 6c 00 00 00 00 00 01 00 78 00 00 00 00 00 01 05 |l.......x.......|\n" + "00000060 52 72 72 53 73 41 61 61 42 62 62 62 43 63 63 63 |RrrSsAaaBbbbCccc|\n" + "00000070 63 63 63 63 63 63 31 32 |cccccc12|\n"