From 8ac2af01224109d6552f08a3ba28063e4a6d555f Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Thu, 24 Oct 2019 10:47:48 +1100 Subject: [PATCH] Change raczlib's dictionary wire format This makes raczlib and raczstd use the same format: RAC's "common dictionary format". This is a breaking change to any existing RAC+Zlib files that use shared dictionaries. RAC+Zlib is still an experimental (unstable) file format. --- doc/spec/rac-spec.md | 111 ++++++++++++--------------- lib/rac/example_test.go | 46 ++++++----- lib/raczlib/raczlib.go | 147 ++++-------------------------------- lib/raczlib/raczlib_test.go | 17 +++-- test/data/sheep-more.rac | Bin 276 -> 278 bytes 5 files changed, 98 insertions(+), 223 deletions(-) diff --git a/doc/spec/rac-spec.md b/doc/spec/rac-spec.md index 3bb84871..0bc92f69 100644 --- a/doc/spec/rac-spec.md +++ b/doc/spec/rac-spec.md @@ -517,25 +517,8 @@ zero). The `CFile` data in the `Leaf Node`'s `Primary CRange` is decompressed as Zlib (RFC 1950), possibly referencing a LZ77 prefix dictionary (what the RFC calls a -"preset dictionary"). - -If a `Leaf Node`'s `Secondary CRange` is empty then there is no dictionary. -Otherwise, the `Secondary CRange` must be at least 6 bytes long: - - - 2 byte little-endian `uint16_t` `Dictionary Length`. - - `Dictionary Length` bytes `Dictionary`. - - 4 byte big-endian `uint32_t` `Dictionary Checksum`. - - Padding (ignored). - -The `Dictionary Checksum` is Zlib's Adler-32 checksum over the `Dictionary`'s -bytes (excluding the initial 2 bytes for the `Dictionary Length`). The checksum -is stored big-endian, like Zlib's other checksums, and its 4 byte value must -match the `DICTID` (in RFC 1950 terminology) given in the `Primary CRange`'s -Zlib-formatted data. - -The `Leaf TTag` must be `0xFF`. All other `Leaf TTag` values (below `0xC0`) are -reserved. The empty `Tertiary CRange` is ignored. The `Leaf STag` value is also -ignored, other than deriving the `Secondary CRange`. +"preset dictionary") wrapped in RAC's common dictionary format, described +above. ## RAC + LZ4 @@ -576,23 +559,24 @@ position `0x04`. Decompressing that chunk produces the 6 bytes The second example consists of a root node with four children: one metadata node (a shared dictionary) and three data nodes. The shared dictionary, -"\x20sheep.\n" is `0x0008` bytes long and its Adler-32 checksum is -`0x0BE0026E`. The third child's (the second data node)'s compressed contents -starts at position `0x73`. Decompressing that chunk, together with that shared +"\x20sheep.\n" is `0x00000008` bytes long and its CRC-32 IEEE checksum is +`0x477A8DD0`. The third child's (the second data node)'s compressed contents +starts at position `0x75`. Decompressing that chunk, together with that shared dictionary, produces the 11 bytes ["Two sheep.\n"](https://play.golang.org/p/Jh9Wyp6PLID). The complete decoding of all three data chunks is "One sheep.\nTwo sheep.\nThree sheep.\n". - 00000000 72 c3 63 04 71 b5 00 ff 00 00 00 00 00 00 00 ff |r.c.q...........| + 00000000 72 c3 63 04 37 39 00 ff 00 00 00 00 00 00 00 ff |r.c.79..........| 00000010 0b 00 00 00 00 00 00 ff 16 00 00 00 00 00 00 ff |................| 00000020 23 00 00 00 00 00 00 01 50 00 00 00 00 00 01 ff |#.......P.......| - 00000030 5e 00 00 00 00 00 01 00 73 00 00 00 00 00 01 00 |^.......s.......| - 00000040 88 00 00 00 00 00 01 00 9f 00 00 00 00 00 01 04 |................| - 00000050 08 00 20 73 68 65 65 70 2e 0a 0b e0 02 6e 78 f9 |.. sheep.....nx.| - 00000060 0b e0 02 6e f2 cf 4b 85 31 01 01 00 00 ff ff 17 |...n..K.1.......| - 00000070 21 03 90 78 f9 0b e0 02 6e 0a 29 cf 87 31 01 01 |!..x....n.)..1..| - 00000080 00 00 ff ff 18 0c 03 a8 78 f9 0b e0 02 6e 0a c9 |........x....n..| - 00000090 28 4a 4d 85 71 00 01 00 00 ff ff 21 6e 04 66 |(JM.q......!n.f| + 00000030 60 00 00 00 00 00 01 00 75 00 00 00 00 00 01 00 |`.......u.......| + 00000040 8a 00 00 00 00 00 01 00 a1 00 00 00 00 00 01 04 |................| + 00000050 08 00 00 00 20 73 68 65 65 70 2e 0a d0 8d 7a 47 |.... sheep....zG| + 00000060 78 f9 0b e0 02 6e f2 cf 4b 85 31 01 01 00 00 ff |x....n..K.1.....| + 00000070 ff 17 21 03 90 78 f9 0b e0 02 6e 0a 29 cf 87 31 |..!..x....n.)..1| + 00000080 01 01 00 00 ff ff 18 0c 03 a8 78 f9 0b e0 02 6e |..........x....n| + 00000090 0a c9 28 4a 4d 85 71 00 01 00 00 ff ff 21 6e 04 |..(JM.q......!n.| + 000000a0 66 |f| ## Zlib Example (Concatenation) @@ -606,7 +590,7 @@ the "more" RAC file's 6 bytes. In hexadecimal, `0x29 = 0x23 + 0x06`, and the The compressed form (what's shown in `hexdump -C` format below) is the concatenation of the two compressed forms, plus a new root node (64 bytes -starting at position `0xD4`). Even though the overall RAC file starts with the +starting at position `0xD6`). Even though the overall RAC file starts with the "sheep" RAC file, whose root node was at its start, those opening bytes are no longer a valid root node for the larger file. @@ -616,49 +600,50 @@ original RAC files' root node is not located at its start. Walking to that child branch node needs two `COffset` values: one for the embedded RAC file's start and one for the embedded RAC file's root node. - 00000000 72 c3 63 04 71 b5 00 ff 00 00 00 00 00 00 00 ff |r.c.q...........| + 00000000 72 c3 63 04 37 39 00 ff 00 00 00 00 00 00 00 ff |r.c.79..........| 00000010 0b 00 00 00 00 00 00 ff 16 00 00 00 00 00 00 ff |................| 00000020 23 00 00 00 00 00 00 01 50 00 00 00 00 00 01 ff |#.......P.......| - 00000030 5e 00 00 00 00 00 01 00 73 00 00 00 00 00 01 00 |^.......s.......| - 00000040 88 00 00 00 00 00 01 00 9f 00 00 00 00 00 01 04 |................| - 00000050 08 00 20 73 68 65 65 70 2e 0a 0b e0 02 6e 78 f9 |.. sheep.....nx.| - 00000060 0b e0 02 6e f2 cf 4b 85 31 01 01 00 00 ff ff 17 |...n..K.1.......| - 00000070 21 03 90 78 f9 0b e0 02 6e 0a 29 cf 87 31 01 01 |!..x....n.)..1..| - 00000080 00 00 ff ff 18 0c 03 a8 78 f9 0b e0 02 6e 0a c9 |........x....n..| - 00000090 28 4a 4d 85 71 00 01 00 00 ff ff 21 6e 04 66 72 |(JM.q......!n.fr| - 000000a0 c3 63 00 78 9c 01 06 00 f9 ff 4d 6f 72 65 21 0a |.c.x......More!.| - 000000b0 07 42 01 bf 72 c3 63 01 65 a9 00 ff 06 00 00 00 |.B..r.c.e.......| - 000000c0 00 00 00 01 04 00 00 00 00 00 01 ff 35 00 00 00 |............5...| - 000000d0 00 00 01 01 72 c3 63 03 f8 ec 00 ff 00 00 00 00 |....r.c.........| - 000000e0 00 00 00 fe 23 00 00 00 00 00 00 fe 29 00 00 00 |....#.......)...| - 000000f0 00 00 00 01 9f 00 00 00 00 00 00 ff 00 00 00 00 |................| - 00000100 00 00 04 01 b4 00 00 00 00 00 04 00 14 01 00 00 |................| - 00000110 00 00 01 03 |....| + 00000030 60 00 00 00 00 00 01 00 75 00 00 00 00 00 01 00 |`.......u.......| + 00000040 8a 00 00 00 00 00 01 00 a1 00 00 00 00 00 01 04 |................| + 00000050 08 00 00 00 20 73 68 65 65 70 2e 0a d0 8d 7a 47 |.... sheep....zG| + 00000060 78 f9 0b e0 02 6e f2 cf 4b 85 31 01 01 00 00 ff |x....n..K.1.....| + 00000070 ff 17 21 03 90 78 f9 0b e0 02 6e 0a 29 cf 87 31 |..!..x....n.)..1| + 00000080 01 01 00 00 ff ff 18 0c 03 a8 78 f9 0b e0 02 6e |..........x....n| + 00000090 0a c9 28 4a 4d 85 71 00 01 00 00 ff ff 21 6e 04 |..(JM.q......!n.| + 000000a0 66 72 c3 63 00 78 9c 01 06 00 f9 ff 4d 6f 72 65 |fr.c.x......More| + 000000b0 21 0a 07 42 01 bf 72 c3 63 01 65 a9 00 ff 06 00 |!..B..r.c.e.....| + 000000c0 00 00 00 00 00 01 04 00 00 00 00 00 01 ff 35 00 |..............5.| + 000000d0 00 00 00 00 01 01 72 c3 63 03 83 16 00 ff 00 00 |......r.c.......| + 000000e0 00 00 00 00 00 fe 23 00 00 00 00 00 00 fe 29 00 |......#.......).| + 000000f0 00 00 00 00 00 01 a1 00 00 00 00 00 00 ff 00 00 |................| + 00000100 00 00 00 00 04 01 b6 00 00 00 00 00 04 00 16 01 |................| + 00000110 00 00 00 00 01 03 |......| + Focusing on that new root node: - 000000d0 72 c3 63 03 f8 ec 00 ff 00 00 00 00 - 000000e0 00 00 00 fe 23 00 00 00 00 00 00 fe 29 00 00 00 - 000000f0 00 00 00 01 9f 00 00 00 00 00 00 ff 00 00 00 00 - 00000100 00 00 04 01 b4 00 00 00 00 00 04 00 14 01 00 00 - 00000110 00 00 01 03 + 000000d0 72 c3 63 03 83 16 00 ff 00 00 + 000000e0 00 00 00 00 00 fe 23 00 00 00 00 00 00 fe 29 00 + 000000f0 00 00 00 00 00 01 a1 00 00 00 00 00 00 ff 00 00 + 00000100 00 00 00 00 04 01 b6 00 00 00 00 00 04 00 16 01 + 00000110 00 00 00 00 01 03 Re-formatting to highlight the groups-of-8-bytes structure and reprise the "Branch Nodes" section's diagram: - 000000d4 72 c3 63 03 f8 ec 00 ff |Magic|A|Che|0|T| - 000000dc 00 00 00 00 00 00 00 fe | DPtr[1] |0|T| - 000000e4 23 00 00 00 00 00 00 fe | DPtr[2] |0|T| - 000000ec 29 00 00 00 00 00 00 01 | DPtr[A] |0|C| - 000000f4 9f 00 00 00 00 00 00 ff | CPtr[0] |L|S| - 000000fc 00 00 00 00 00 00 04 01 | CPtr[1] |L|S| - 00000104 b4 00 00 00 00 00 04 00 | CPtr[2] |L|S| - 0000010c 14 01 00 00 00 00 01 03 | CPtr[A] |V|A| + 000000d6 72 c3 63 03 83 16 00 ff |Magic|A|Che|0|T| + 000000de 00 00 00 00 00 00 00 fe | DPtr[1] |0|T| + 000000e6 23 00 00 00 00 00 00 fe | DPtr[2] |0|T| + 000000ee 29 00 00 00 00 00 00 01 | DPtr[A] |0|C| + 000000f6 a1 00 00 00 00 00 00 ff | CPtr[0] |L|S| + 000000fe 00 00 00 00 00 00 04 01 | CPtr[1] |L|S| + 00000106 b6 00 00 00 00 00 04 00 | CPtr[2] |L|S| + 0000010e 16 01 00 00 00 00 01 03 | CPtr[A] |V|A| -Its `CodecByte` (and therefore its `Codec`) is `0x01`, "RAC + Zlib", its +Its `CodecByte` (and therefore its `Short Codec`) is `0x01`, "RAC + Zlib", its Version is `0x01` and its `Arity` is `0x03`. The `DPtr` values are `0x0000` -(implicit), `0x0000`, `0x0023` and `0x0029`. The `CPtr` values are `0x009F`, -`0x0000`, `0x00B4` and `0x0114` (the size of the whole RAC file). Note that the +(implicit), `0x0000`, `0x0023` and `0x0029`. The `CPtr` values are `0x00A1`, +`0x0000`, `0x00B6` and `0x0116` (the size of the whole RAC file). Note that the `CPtr` values are not sorted. The last two children's `TTag`s are `0xFE` and `0xFE` and their `STag`s are `0x01` and `0x00`, which means that they are both `CBiasing Branch Node`s. diff --git a/lib/rac/example_test.go b/lib/rac/example_test.go index 5bc4bde6..3f588a11 100644 --- a/lib/rac/example_test.go +++ b/lib/rac/example_test.go @@ -21,6 +21,7 @@ import ( "encoding/hex" "fmt" "hash/adler32" + "hash/crc32" "io" "log" "os" @@ -106,17 +107,23 @@ func Example_indexLocationAtStart() { } dict := []byte(" sheep.\n") - hasher := adler32.New() - hasher.Write(dict) - if len(dict) > 0xFFFF { + if len(dict) >= (1 << 30) { log.Fatal("len(dict) is too large") } encodedDict := []byte{ uint8(len(dict) >> 0), uint8(len(dict) >> 8), + uint8(len(dict) >> 16), + uint8(len(dict) >> 24), } encodedDict = append(encodedDict, dict...) - encodedDict = hasher.Sum(encodedDict) + checksum := crc32.ChecksumIEEE(dict) + encodedDict = append(encodedDict, + uint8(checksum>>0), + uint8(checksum>>8), + uint8(checksum>>16), + uint8(checksum>>24), + ) fmt.Printf("Encoded dictionary resource:\n%s\n", hex.Dump(encodedDict)) dictResource, err := w.AddResource(encodedDict) @@ -180,17 +187,17 @@ func Example_indexLocationAtStart() { // https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib dict := []byte(nil) if secondary := encoded[chunk.CSecondary[0]:chunk.CSecondary[1]]; len(secondary) > 0 { - if len(secondary) < 6 { + if len(secondary) < 8 { log.Fatalf("invalid secondary data") } - dictLen := int(binary.LittleEndian.Uint16(secondary)) - secondary = secondary[2:] - if (dictLen + 4) > len(secondary) { + dictLen := int(binary.LittleEndian.Uint32(secondary)) + secondary = secondary[4:] + if (dictLen >= (1 << 30)) || ((dictLen + 4) > len(secondary)) { log.Fatalf("invalid secondary data") } - checksum := binary.BigEndian.Uint32(secondary[dictLen:]) + checksum := binary.LittleEndian.Uint32(secondary[dictLen:]) dict = secondary[:dictLen] - if checksum != adler32.Checksum(dict) { + if checksum != crc32.ChecksumIEEE(dict) { log.Fatalf("invalid checksum") } } @@ -222,7 +229,7 @@ func Example_indexLocationAtStart() { // Output: // Encoded dictionary resource: - // 00000000 08 00 20 73 68 65 65 70 2e 0a 0b e0 02 6e |.. sheep.....n| + // 00000000 08 00 00 00 20 73 68 65 65 70 2e 0a d0 8d 7a 47 |.... sheep....zG| // // Encoded chunk #0: // 00000000 78 f9 0b e0 02 6e f2 cf 4b 85 31 01 01 00 00 ff |x....n..K.1.....| @@ -237,16 +244,17 @@ func Example_indexLocationAtStart() { // 00000010 00 ff ff 21 6e 04 66 |...!n.f| // // RAC file: - // 00000000 72 c3 63 04 71 b5 00 ff 00 00 00 00 00 00 00 ff |r.c.q...........| + // 00000000 72 c3 63 04 37 39 00 ff 00 00 00 00 00 00 00 ff |r.c.79..........| // 00000010 0b 00 00 00 00 00 00 ff 16 00 00 00 00 00 00 ff |................| // 00000020 23 00 00 00 00 00 00 01 50 00 00 00 00 00 01 ff |#.......P.......| - // 00000030 5e 00 00 00 00 00 01 00 73 00 00 00 00 00 01 00 |^.......s.......| - // 00000040 88 00 00 00 00 00 01 00 9f 00 00 00 00 00 01 04 |................| - // 00000050 08 00 20 73 68 65 65 70 2e 0a 0b e0 02 6e 78 f9 |.. sheep.....nx.| - // 00000060 0b e0 02 6e f2 cf 4b 85 31 01 01 00 00 ff ff 17 |...n..K.1.......| - // 00000070 21 03 90 78 f9 0b e0 02 6e 0a 29 cf 87 31 01 01 |!..x....n.)..1..| - // 00000080 00 00 ff ff 18 0c 03 a8 78 f9 0b e0 02 6e 0a c9 |........x....n..| - // 00000090 28 4a 4d 85 71 00 01 00 00 ff ff 21 6e 04 66 |(JM.q......!n.f| + // 00000030 60 00 00 00 00 00 01 00 75 00 00 00 00 00 01 00 |`.......u.......| + // 00000040 8a 00 00 00 00 00 01 00 a1 00 00 00 00 00 01 04 |................| + // 00000050 08 00 00 00 20 73 68 65 65 70 2e 0a d0 8d 7a 47 |.... sheep....zG| + // 00000060 78 f9 0b e0 02 6e f2 cf 4b 85 31 01 01 00 00 ff |x....n..K.1.....| + // 00000070 ff 17 21 03 90 78 f9 0b e0 02 6e 0a 29 cf 87 31 |..!..x....n.)..1| + // 00000080 01 01 00 00 ff ff 18 0c 03 a8 78 f9 0b e0 02 6e |..........x....n| + // 00000090 0a c9 28 4a 4d 85 71 00 01 00 00 ff ff 21 6e 04 |..(JM.q......!n.| + // 000000a0 66 |f| // // Decoded: // [ 0, 11): One sheep. diff --git a/lib/raczlib/raczlib.go b/lib/raczlib/raczlib.go index df85c6b5..d07bb234 100644 --- a/lib/raczlib/raczlib.go +++ b/lib/raczlib/raczlib.go @@ -25,10 +25,10 @@ import ( "bytes" "compress/zlib" "errors" - "hash/adler32" "io" "github.com/google/wuffs/lib/compression" + "github.com/google/wuffs/lib/internal/racdict" "github.com/google/wuffs/lib/rac" "github.com/google/wuffs/lib/zlibcut" ) @@ -43,11 +43,11 @@ func u32BE(b []byte) uint32 { return (uint32(b[0]) << 24) | (uint32(b[1]) << 16) | (uint32(b[2]) << 8) | (uint32(b[3])) } -// suffix32K returns the last 32 KiB of b, or if b is shorter than that, it +// refine returns the last 32 KiB of b, or if b is shorter than that, it // returns just b. // // The Zlib format only supports up to 32 KiB of history or shared dictionary. -func suffix32K(b []byte) []byte { +func refine(b []byte) []byte { const n = 32 * 1024 if len(b) > n { return b[len(b)-n:] @@ -64,12 +64,8 @@ type CodecReader struct { // lim provides a limited view of a RAC file. lim io.LimitedReader - // These fields contain the most recently used shared dictionary. - cachedDictionary []byte - cachedDictionaryCRange rac.Range - - // buf is a scratch buffer. - buf [2]byte + // dictLoader loads shared dictionaries. + dictLoader racdict.Loader } // Close implements rac.CodecReader. @@ -89,7 +85,7 @@ func (r *CodecReader) Clone() rac.CodecReader { // MakeDecompressor implements rac.CodecReader. func (r *CodecReader) MakeDecompressor(racFile io.ReadSeeker, chunk rac.Chunk) (io.Reader, error) { - dict, err := r.loadDictionary(racFile, chunk) + dict, err := r.dictLoader.Load(racFile, chunk) if err != nil { return nil, err } @@ -101,77 +97,15 @@ func (r *CodecReader) MakeDecompressor(racFile io.ReadSeeker, chunk rac.Chunk) ( return r.makeDecompressor(&r.lim, dict) } -func (r *CodecReader) loadDictionary(rs io.ReadSeeker, chunk rac.Chunk) ([]byte, error) { - // For a description of the RAC+Zlib secondary-data format, see - // https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib - - if !chunk.CTertiary.Empty() { - return nil, errInvalidDictionary - } - if chunk.CSecondary.Empty() { - return nil, nil - } - cRange := chunk.CSecondary - - // Load from the MRU cache, if it was loaded from the same cRange. - if (cRange == r.cachedDictionaryCRange) && !cRange.Empty() { - return r.cachedDictionary, nil - } - - // Check the cRange size and the tTag. - if (cRange.Size() < 6) || (chunk.TTag != 0xFF) { - return nil, errInvalidDictionary - } - - // Read the dictionary size. - if _, err := rs.Seek(cRange[0], io.SeekStart); err != nil { - return nil, errInvalidDictionary - } - if _, err := io.ReadFull(rs, r.buf[:2]); err != nil { - return nil, errInvalidDictionary - } - dictSize := int64(r.buf[0]) | (int64(r.buf[1]) << 8) - - // Check the size. The +6 is for the 2 byte prefix (dictionary size) and - // the 4 byte suffix (checksum). - if (dictSize + 6) > cRange.Size() { - return nil, errInvalidDictionary - } - - // Allocate or re-use the cachedDictionary buffer. - buffer := []byte(nil) - if n := dictSize + 4; int64(cap(r.cachedDictionary)) >= n { - buffer = r.cachedDictionary[:n] - // Invalidate the cached dictionary, as we are re-using its memory. - r.cachedDictionaryCRange = rac.Range{} - } else { - buffer = make([]byte, n) - } - - // Read the dictionary and checksum. - if _, err := io.ReadFull(rs, buffer); err != nil { - return nil, errInvalidDictionary - } - - // Verify the checksum. - dict, checksum := buffer[:dictSize], buffer[dictSize:] - if u32BE(checksum) != adler32.Checksum(dict) { - return nil, errInvalidDictionary - } - - // Save to the MRU cache and return. - r.cachedDictionary = dict - r.cachedDictionaryCRange = cRange - return dict, nil -} - // CodecWriter specializes a rac.Writer to encode Zlib-compressed chunks. type CodecWriter struct { // These fields help reduce memory allocation by re-using previous byte // buffers or the zlib.Writer. compressed bytes.Buffer cachedWriter *zlib.Writer - stash []byte + + // dictSaver saves shared dictionaries. + dictSaver racdict.Saver } // Close implements rac.CodecWriter. @@ -187,51 +121,10 @@ func (w *CodecWriter) Clone() rac.CodecWriter { // Compress implements rac.CodecWriter. func (w *CodecWriter) Compress(p []byte, q []byte, resourcesData [][]byte) ( codec rac.Codec, compressed []byte, secondaryResource int, tertiaryResource int, retErr error) { - - secondaryResource = rac.NoResourceUsed - - // Compress p+q without any shared dictionary. - baseline, err := w.compress(p, q, nil) - if err != nil { - return 0, nil, 0, 0, err - } - // If there aren't any potential shared dictionaries, or if the baseline - // compressed form is small, just return the baseline. - const minBaselineSizeToConsiderDictionaries = 256 - if (len(resourcesData) == 0) || (len(baseline) < minBaselineSizeToConsiderDictionaries) { - return rac.CodecZlib, baseline, secondaryResource, rac.NoResourceUsed, nil - } - - // w.stash keeps a copy of the best compression so far. Every call to - // w.compress can clobber the bytes previously returned by w.compress. - w.stash = append(w.stash[:0], baseline...) - compressed = w.stash - - // Only use a shared dictionary if it results in something smaller than - // 98.4375% (an arbitrary heuristic threshold) of the baseline. Otherwise, - // it's arguably not worth the additional complexity. - threshold := (len(baseline) / 64) * 63 - - for i, data := range resourcesData { - candidate, err := w.compress(p, q, suffix32K(data)) - if err != nil { - return 0, nil, 0, 0, err - } - if n := len(candidate); (n >= threshold) || (n >= len(compressed)) { - continue - } - secondaryResource = i - - // As an optimization, if the final candidate is the winner, we don't - // have to copy its contents to w.stash. - if i == len(resourcesData)-1 { - compressed = candidate - } else { - w.stash = append(w.stash[:0], candidate...) - compressed = w.stash - } - } - return rac.CodecZlib, compressed, secondaryResource, rac.NoResourceUsed, nil + return w.dictSaver.Compress( + p, q, resourcesData, + rac.CodecZlib, w.compress, refine, + ) } func (w *CodecWriter) compress(p []byte, q []byte, dict []byte) ([]byte, error) { @@ -289,17 +182,5 @@ func (w *CodecWriter) Cut(codec rac.Codec, encoded []byte, maxEncodedLen int) (e // WrapResource implements rac.CodecWriter. func (w *CodecWriter) WrapResource(raw []byte) ([]byte, error) { - // For a description of the RAC+Zlib secondary-data format, see - // https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib - raw = suffix32K(raw) - wrapped := make([]byte, len(raw)+6) - wrapped[0] = uint8(len(raw) >> 0) - wrapped[1] = uint8(len(raw) >> 8) - copy(wrapped[2:], raw) - a := adler32.Checksum(raw) - wrapped[len(wrapped)-4] = uint8(a >> 24) - wrapped[len(wrapped)-3] = uint8(a >> 16) - wrapped[len(wrapped)-2] = uint8(a >> 8) - wrapped[len(wrapped)-1] = uint8(a >> 0) - return wrapped, nil + return w.dictSaver.WrapResource(raw, refine) } diff --git a/lib/raczlib/raczlib_test.go b/lib/raczlib/raczlib_test.go index 0bcee961..4816bc42 100644 --- a/lib/raczlib/raczlib_test.go +++ b/lib/raczlib/raczlib_test.go @@ -45,16 +45,17 @@ const ( "\x00\x00\x00\x01\x01" encodedSheep = "" + - "\x72\xC3\x63\x04\x71\xB5\x00\xFF\x00\x00\x00\x00\x00\x00\x00\xFF" + + "\x72\xC3\x63\x04\x37\x39\x00\xFF\x00\x00\x00\x00\x00\x00\x00\xFF" + "\x0B\x00\x00\x00\x00\x00\x00\xFF\x16\x00\x00\x00\x00\x00\x00\xFF" + "\x23\x00\x00\x00\x00\x00\x00\x01\x50\x00\x00\x00\x00\x00\x01\xFF" + - "\x5E\x00\x00\x00\x00\x00\x01\x00\x73\x00\x00\x00\x00\x00\x01\x00" + - "\x88\x00\x00\x00\x00\x00\x01\x00\x9F\x00\x00\x00\x00\x00\x01\x04" + - "\x08\x00\x20\x73\x68\x65\x65\x70\x2E\x0A\x0B\xE0\x02\x6E\x78\xF9" + - "\x0B\xE0\x02\x6E\xF2\xCF\x4B\x85\x31\x01\x01\x00\x00\xFF\xFF\x17" + - "\x21\x03\x90\x78\xF9\x0B\xE0\x02\x6E\x0A\x29\xCF\x87\x31\x01\x01" + - "\x00\x00\xFF\xFF\x18\x0C\x03\xA8\x78\xF9\x0B\xE0\x02\x6E\x0A\xC9" + - "\x28\x4A\x4D\x85\x71\x00\x01\x00\x00\xFF\xFF\x21\x6E\x04\x66" + "\x60\x00\x00\x00\x00\x00\x01\x00\x75\x00\x00\x00\x00\x00\x01\x00" + + "\x8A\x00\x00\x00\x00\x00\x01\x00\xA1\x00\x00\x00\x00\x00\x01\x04" + + "\x08\x00\x00\x00\x20\x73\x68\x65\x65\x70\x2E\x0A\xD0\x8D\x7A\x47" + + "\x78\xF9\x0B\xE0\x02\x6E\xF2\xCF\x4B\x85\x31\x01\x01\x00\x00\xFF" + + "\xFF\x17\x21\x03\x90\x78\xF9\x0B\xE0\x02\x6E\x0A\x29\xCF\x87\x31" + + "\x01\x01\x00\x00\xFF\xFF\x18\x0C\x03\xA8\x78\xF9\x0B\xE0\x02\x6E" + + "\x0A\xC9\x28\x4A\x4D\x85\x71\x00\x01\x00\x00\xFF\xFF\x21\x6E\x04" + + "\x66" ) func racCompress(original []byte, cChunkSize uint64, dChunkSize uint64, resourcesData [][]byte) ([]byte, error) { diff --git a/test/data/sheep-more.rac b/test/data/sheep-more.rac index 8d17f17a385fc91d5434b08fd4828a667f5a20a7..7856397bb2f480b1613f36c3bd99d80ddd69c8a5 100644 GIT binary patch delta 105 zcmbQjG>wU^=x{QNx#dJQErSFG5MX2|1yKx)3|&xqA%te(U|?WSD9%VtEzsk-&|Bp` iF|LZKS#07hDFuiMhW}uN3@nV>zzh}!F-8!Bkr@Elj1ceu delta 103 zcmbQnG=+(+=x{Pi;ns<4T6%E|Ai&5_45Anq89Jc!d