diff --git a/doc/spec/rac-spec.md b/doc/spec/rac-spec.md index 55ac25a7..3bb84871 100644 --- a/doc/spec/rac-spec.md +++ b/doc/spec/rac-spec.md @@ -486,6 +486,27 @@ if they are `Branch Node`s. # Specific Codecs +## Common Dictionary Format + +Unless otherwise noted, codecs use this common dictionary format. + +If a `Leaf Node`'s `Secondary CRange` is empty then there is no dictionary. +Otherwise, the `Secondary CRange` must be at least 8 bytes long: + + - 4 byte little-endian `uint32_t` `Dictionary Length`. The high 2 bits are + reserved and must be zero. The maximum (inclusive) `Dictionary Length` is + therefore `((1 << 30) - 1)`, or `1_073_741_823` bytes. + - `Dictionary Length` bytes `Dictionary`. + - 4 byte little-endian `uint32_t` `Dictionary Checksum`, the Checksum` is + CRC-32 IEEE checksum over the `Dictionary`'s bytes (excluding both the 4 + byte prefix and the 4 byte suffix). + - Padding (ignored). + +The `Leaf TTag` must be `0xFF`. All other `Leaf TTag` values (below `0xC0`) are +reserved. The empty `Tertiary CRange` is ignored. The `Leaf STag` value is also +ignored, other than deriving the `Secondary CRange`. + + ## RAC + Zeroes The `CRange`s are ignored. The `DRange` is filled with NUL bytes (`memset` to @@ -525,35 +546,9 @@ TODO. ## RAC + Zstandard The `CFile` data in the `Leaf Node`'s `Primary CRange` is decompressed as -Zstandard (RFC 8478), possibly referencing a LZ77 prefix dictionary. - -If a `Leaf Node`'s `Secondary CRange` is empty then there is no dictionary. -Otherwise, the `Secondary CRange` must be at least 12 bytes long: - - - 4 byte little-endian `uint32_t` `Dictionary ID`. - - 8 byte little-endian `uint64_t` `Dictionary Length`. - - `Dictionary Length` bytes `Dictionary`. - - Padding (ignored). - -The `Dictionary ID` is per section 3.1.1.1.3 in the RFC, although in RAC it -always occupies 4 bytes, even if Zstandard itself could represent it in fewer -bytes. In the terminology of the RFC section 5, the `Dictionary` bytes are "raw -content" dictionaries. - -As per the RFC section 3.1.1.1.2, the Zstandard format allows for a window size -up to 3.75 TiB, which is more than the maximum `uint32_t` value. The -`Dictionary Length` field here is therefore 8 bytes, not 4. However, the RFC -acknowledges that "a decoder is allowed to reject a compressed frame that -requests a memory size beyond decoder's authorized range", and likewise, a -RAC + Zstandard decoder is allowed to reject a `Dictionary Length` that it -considers too large. The RFC goes on to say "it's recommended for decoders to -support values of Window Size up to 8 MiB and for encoders not to generate -frames requiring a Window Size larger than 8 MiB", and that recommendation -similarly applies to RAC + Zstandard. - -The `Leaf TTag` must be `0xFF`. All other `Leaf TTag` values (below `0xC0`) are -reserved. The empty `Tertiary CRange` is ignored. The `Leaf STag` value is also -ignored, other than deriving the `Secondary CRange`. +Zstandard (RFC 8478), possibly referencing a dictionary wrapped in RAC's common +dictionary format, described above. After unwrapping, the dictionary's bytes +can be either a "raw" or "trained" dictionary, as per RFC 8478 section 5. # Examples diff --git a/lib/internal/racdict/racdict.go b/lib/internal/racdict/racdict.go new file mode 100644 index 00000000..65a99b74 --- /dev/null +++ b/lib/internal/racdict/racdict.go @@ -0,0 +1,222 @@ +// Copyright 2019 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ---------------- + +// Package racdict provides support for the RAC (Random Access Compression) +// common dictionary format. +// +// See +// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#common-dictionary-format +package racdict + +import ( + "errors" + "hash/crc32" + "io" + + "github.com/google/wuffs/lib/rac" +) + +const ( + // MaxInclLength is the maximum (inclusive) length of a dictionary, not + // including the length prefix and the checksum suffix. + // + // Its value is (1 GiB - 1), or 1_073_741_823 bytes. + MaxInclLength = (1 << 30) - 1 +) + +var ( + errDictionaryIsTooLong = errors.New("racdict: dictionary is too long") + errInvalidDictionary = errors.New("racdict: invalid dictionary") +) + +func u32LE(b []byte) uint32 { + _ = b[3] // Early bounds check to guarantee safety of reads below. + return (uint32(b[0]) << 0) | + (uint32(b[1]) << 8) | + (uint32(b[2]) << 16) | + (uint32(b[3]) << 24) +} + +// Loader loads a dictionary wrapped in the RAC common dictionary format. +// +// It can cache previously loaded values, keyed by a rac.Chunk. +type Loader struct { + cachedBytes []byte + cachedRange rac.Range + buf [4]byte +} + +// Load reads and unwraps a wrapped dictionary. +func (r *Loader) Load(rs io.ReadSeeker, chunk rac.Chunk) (dictionary []byte, retErr error) { + if !chunk.CTertiary.Empty() { + return nil, errInvalidDictionary + } + if chunk.CSecondary.Empty() { + return nil, nil + } + cRange := chunk.CSecondary + + // Load from the MRU cache, if it was loaded from the same cRange. + if (cRange == r.cachedRange) && !cRange.Empty() { + return r.cachedBytes, nil + } + + // Check the cRange size and the tTag. + if (cRange.Size() < 8) || (chunk.TTag != 0xFF) { + return nil, errInvalidDictionary + } + + // Read the dictionary size. + if _, err := rs.Seek(cRange[0], io.SeekStart); err != nil { + return nil, err + } + if _, err := io.ReadFull(rs, r.buf[:4]); err != nil { + return nil, err + } + + // Check that the high 2 bits are zero (reserved). In other words, we + // enforce that the maximum (inclusive) length is MaxInclLength. + dictSize := int64(u32LE(r.buf[:4])) + if (dictSize >> 30) != 0 { + return nil, errInvalidDictionary + } + + // Check the overall size. The +8 is for the 4 byte prefix (dictionary + // size) and the 4 byte suffix (checksum). + if (dictSize + 8) > cRange.Size() { + return nil, errInvalidDictionary + } + + // Allocate or re-use the cachedBytes buffer. + buffer := []byte(nil) + if n := dictSize + 4; int64(cap(r.cachedBytes)) >= n { + buffer = r.cachedBytes[:n] + // Invalidate the cached dictionary, as we are re-using its memory. + r.cachedRange = rac.Range{} + } else { + buffer = make([]byte, n) + } + + // Read the dictionary and checksum. + if _, err := io.ReadFull(rs, buffer); err != nil { + return nil, err + } + + // Verify the checksum. + dict, checksum := buffer[:dictSize], buffer[dictSize:] + if u32LE(checksum) != crc32.ChecksumIEEE(dict) { + return nil, errInvalidDictionary + } + + // Save to the MRU cache and return. + r.cachedBytes = dict + r.cachedRange = cRange + return dict, nil +} + +// Saver saves a dictionary wrapped in the RAC common dictionary format. +type Saver struct { + stash []byte +} + +// WrapResource wraps a dictionary. +func (w *Saver) WrapResource( + raw []byte, + refineResourceData func([]byte) []byte, +) ([]byte, error) { + refined := refineResourceData(raw) + if len(refined) > MaxInclLength { + return nil, errDictionaryIsTooLong + } + wrapped := make([]byte, len(refined)+8) + wrapped[0] = uint8(len(refined) >> 0) + wrapped[1] = uint8(len(refined) >> 8) + wrapped[2] = uint8(len(refined) >> 16) + wrapped[3] = uint8(len(refined) >> 24) + copy(wrapped[4:], refined) + checksum := crc32.ChecksumIEEE(refined) + wrapped[len(wrapped)-4] = uint8(checksum >> 0) + wrapped[len(wrapped)-3] = uint8(checksum >> 8) + wrapped[len(wrapped)-2] = uint8(checksum >> 16) + wrapped[len(wrapped)-1] = uint8(checksum >> 24) + return wrapped, nil +} + +// Compress helps implement rac.CodecWriter. +func (w *Saver) Compress( + p []byte, + q []byte, + resourcesData [][]byte, + codec rac.Codec, + compress func(p []byte, q []byte, dict []byte) ([]byte, error), + refineResourceData func([]byte) []byte, +) ( + retCodec rac.Codec, + compressed []byte, + secondaryResource int, + tertiaryResource int, + retErr error, +) { + secondaryResource = rac.NoResourceUsed + + // Compress p+q without any shared dictionary. + baseline, err := compress(p, q, nil) + if err != nil { + return 0, nil, 0, 0, err + } + // If there aren't any potential shared dictionaries, or if the baseline + // compressed form is small, just return the baseline. + const minBaselineSizeToConsiderDictionaries = 256 + if (len(resourcesData) == 0) || (len(baseline) < minBaselineSizeToConsiderDictionaries) { + return codec, baseline, secondaryResource, rac.NoResourceUsed, nil + } + + // w.stash keeps a copy of the best compression so far. Every call to + // compress can clobber the bytes previously returned by compress. + w.stash = append(w.stash[:0], baseline...) + compressed = w.stash + + // Only use a shared dictionary if it results in something smaller than + // 98.4375% (an arbitrary heuristic threshold) of the baseline. Otherwise, + // it's arguably not worth the additional complexity. + threshold := (len(baseline) / 64) * 63 + + for i, resourceData := range resourcesData { + refined := refineResourceData(resourceData) + if len(refined) > MaxInclLength { + return 0, nil, 0, 0, errDictionaryIsTooLong + } + + candidate, err := compress(p, q, refined) + if err != nil { + return 0, nil, 0, 0, err + } + if n := len(candidate); (n >= threshold) || (n >= len(compressed)) { + continue + } + secondaryResource = i + + // As an optimization, if the final candidate is the winner, we don't + // have to copy its contents to w.stash. + if i == len(resourcesData)-1 { + compressed = candidate + } else { + w.stash = append(w.stash[:0], candidate...) + compressed = w.stash + } + } + return codec, compressed, secondaryResource, rac.NoResourceUsed, nil +} diff --git a/lib/raczstd/raczstd.go b/lib/raczstd/raczstd.go index e9a2e90e..8e6bdd6b 100644 --- a/lib/raczstd/raczstd.go +++ b/lib/raczstd/raczstd.go @@ -28,19 +28,33 @@ import ( "github.com/google/wuffs/lib/cgozstd" "github.com/google/wuffs/lib/compression" + "github.com/google/wuffs/lib/internal/racdict" "github.com/google/wuffs/lib/rac" ) var ( - errCannotCut = errors.New("raczstd: cannot cut") - errInvalidCodec = errors.New("raczstd: invalid codec") + errCannotCut = errors.New("raczstd: cannot cut") ) +func refine(b []byte) []byte { + if len(b) > racdict.MaxInclLength { + return b[len(b)-racdict.MaxInclLength:] + } + return b +} + // CodecReader specializes a rac.Reader to decode Zstd-compressed chunks. type CodecReader struct { + // cachedReader lets us re-use the memory allocated for a zstd reader, when + // decompressing multiple chunks. cachedReader compression.Reader recycler cgozstd.ReaderRecycler - lim io.LimitedReader + + // lim provides a limited view of a RAC file. + lim io.LimitedReader + + // dictLoader loads shared dictionaries. + dictLoader racdict.Loader } // Close implements rac.CodecReader. @@ -60,6 +74,10 @@ func (r *CodecReader) Clone() rac.CodecReader { // MakeDecompressor implements rac.CodecReader. func (r *CodecReader) MakeDecompressor(racFile io.ReadSeeker, chunk rac.Chunk) (io.Reader, error) { + dict, err := r.dictLoader.Load(racFile, chunk) + if err != nil { + return nil, err + } if _, err := racFile.Seek(chunk.CPrimary[0], io.SeekStart); err != nil { return nil, err } @@ -71,7 +89,7 @@ func (r *CodecReader) MakeDecompressor(racFile io.ReadSeeker, chunk rac.Chunk) ( r.cachedReader = zr r.recycler.Bind(zr) } - if err := r.cachedReader.Reset(&r.lim, nil); err != nil { + if err := r.cachedReader.Reset(&r.lim, dict); err != nil { return nil, err } return r.cachedReader, nil @@ -82,6 +100,9 @@ type CodecWriter struct { compressed bytes.Buffer cachedWriter compression.Writer recycler cgozstd.WriterRecycler + + // dictSaver saves shared dictionaries. + dictSaver racdict.Saver } // Close implements rac.CodecWriter. @@ -97,13 +118,10 @@ func (w *CodecWriter) Clone() rac.CodecWriter { // Compress implements rac.CodecWriter. func (w *CodecWriter) Compress(p []byte, q []byte, resourcesData [][]byte) ( codec rac.Codec, compressed []byte, secondaryResource int, tertiaryResource int, retErr error) { - - // Compress p+q without any shared dictionary. - baseline, err := w.compress(p, q, nil) - if err != nil { - return 0, nil, 0, 0, err - } - return rac.CodecZstandard, baseline, rac.NoResourceUsed, rac.NoResourceUsed, nil + return w.dictSaver.Compress( + p, q, resourcesData, + rac.CodecZstandard, w.compress, refine, + ) } func (w *CodecWriter) compress(p []byte, q []byte, dict []byte) ([]byte, error) { @@ -148,5 +166,5 @@ func (w *CodecWriter) Cut(codec rac.Codec, encoded []byte, maxEncodedLen int) (e // WrapResource implements rac.CodecWriter. func (w *CodecWriter) WrapResource(raw []byte) ([]byte, error) { - return nil, nil + return w.dictSaver.WrapResource(raw, refine) }