mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 08:32:11 +01:00
Implement raczstd dictionaries
This commit is contained in:
1 parent
3b466277dc
commit
2d8648ae35
3 files changed
+276
-41
No files matched your search
+24
-29
@@ -486,6 +486,27 @@ if they are `Branch Node`s.
|
||||
# Specific Codecs
|
||||
|
||||
|
||||
## Common Dictionary Format
|
||||
|
||||
Unless otherwise noted, codecs use this common dictionary format.
|
||||
|
||||
If a `Leaf Node`'s `Secondary CRange` is empty then there is no dictionary.
|
||||
Otherwise, the `Secondary CRange` must be at least 8 bytes long:
|
||||
|
||||
- 4 byte little-endian `uint32_t` `Dictionary Length`. The high 2 bits are
|
||||
reserved and must be zero. The maximum (inclusive) `Dictionary Length` is
|
||||
therefore `((1 << 30) - 1)`, or `1_073_741_823` bytes.
|
||||
- `Dictionary Length` bytes `Dictionary`.
|
||||
- 4 byte little-endian `uint32_t` `Dictionary Checksum`, the Checksum` is
|
||||
CRC-32 IEEE checksum over the `Dictionary`'s bytes (excluding both the 4
|
||||
byte prefix and the 4 byte suffix).
|
||||
- Padding (ignored).
|
||||
|
||||
The `Leaf TTag` must be `0xFF`. All other `Leaf TTag` values (below `0xC0`) are
|
||||
reserved. The empty `Tertiary CRange` is ignored. The `Leaf STag` value is also
|
||||
ignored, other than deriving the `Secondary CRange`.
|
||||
|
||||
|
||||
## RAC + Zeroes
|
||||
|
||||
The `CRange`s are ignored. The `DRange` is filled with NUL bytes (`memset` to
|
||||
@@ -525,35 +546,9 @@ TODO.
|
||||
## RAC + Zstandard
|
||||
|
||||
The `CFile` data in the `Leaf Node`'s `Primary CRange` is decompressed as
|
||||
Zstandard (RFC 8478), possibly referencing a LZ77 prefix dictionary.
|
||||
|
||||
If a `Leaf Node`'s `Secondary CRange` is empty then there is no dictionary.
|
||||
Otherwise, the `Secondary CRange` must be at least 12 bytes long:
|
||||
|
||||
- 4 byte little-endian `uint32_t` `Dictionary ID`.
|
||||
- 8 byte little-endian `uint64_t` `Dictionary Length`.
|
||||
- `Dictionary Length` bytes `Dictionary`.
|
||||
- Padding (ignored).
|
||||
|
||||
The `Dictionary ID` is per section 3.1.1.1.3 in the RFC, although in RAC it
|
||||
always occupies 4 bytes, even if Zstandard itself could represent it in fewer
|
||||
bytes. In the terminology of the RFC section 5, the `Dictionary` bytes are "raw
|
||||
content" dictionaries.
|
||||
|
||||
As per the RFC section 3.1.1.1.2, the Zstandard format allows for a window size
|
||||
up to 3.75 TiB, which is more than the maximum `uint32_t` value. The
|
||||
`Dictionary Length` field here is therefore 8 bytes, not 4. However, the RFC
|
||||
acknowledges that "a decoder is allowed to reject a compressed frame that
|
||||
requests a memory size beyond decoder's authorized range", and likewise, a
|
||||
RAC + Zstandard decoder is allowed to reject a `Dictionary Length` that it
|
||||
considers too large. The RFC goes on to say "it's recommended for decoders to
|
||||
support values of Window Size up to 8 MiB and for encoders not to generate
|
||||
frames requiring a Window Size larger than 8 MiB", and that recommendation
|
||||
similarly applies to RAC + Zstandard.
|
||||
|
||||
The `Leaf TTag` must be `0xFF`. All other `Leaf TTag` values (below `0xC0`) are
|
||||
reserved. The empty `Tertiary CRange` is ignored. The `Leaf STag` value is also
|
||||
ignored, other than deriving the `Secondary CRange`.
|
||||
Zstandard (RFC 8478), possibly referencing a dictionary wrapped in RAC's common
|
||||
dictionary format, described above. After unwrapping, the dictionary's bytes
|
||||
can be either a "raw" or "trained" dictionary, as per RFC 8478 section 5.
|
||||
|
||||
|
||||
# Examples
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
// Copyright 2019 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// ----------------
|
||||
|
||||
// Package racdict provides support for the RAC (Random Access Compression)
|
||||
// common dictionary format.
|
||||
//
|
||||
// See
|
||||
// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#common-dictionary-format
|
||||
package racdict
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
|
||||
"github.com/google/wuffs/lib/rac"
|
||||
)
|
||||
|
||||
const (
|
||||
// MaxInclLength is the maximum (inclusive) length of a dictionary, not
|
||||
// including the length prefix and the checksum suffix.
|
||||
//
|
||||
// Its value is (1 GiB - 1), or 1_073_741_823 bytes.
|
||||
MaxInclLength = (1 << 30) - 1
|
||||
)
|
||||
|
||||
var (
|
||||
errDictionaryIsTooLong = errors.New("racdict: dictionary is too long")
|
||||
errInvalidDictionary = errors.New("racdict: invalid dictionary")
|
||||
)
|
||||
|
||||
func u32LE(b []byte) uint32 {
|
||||
_ = b[3] // Early bounds check to guarantee safety of reads below.
|
||||
return (uint32(b[0]) << 0) |
|
||||
(uint32(b[1]) << 8) |
|
||||
(uint32(b[2]) << 16) |
|
||||
(uint32(b[3]) << 24)
|
||||
}
|
||||
|
||||
// Loader loads a dictionary wrapped in the RAC common dictionary format.
|
||||
//
|
||||
// It can cache previously loaded values, keyed by a rac.Chunk.
|
||||
type Loader struct {
|
||||
cachedBytes []byte
|
||||
cachedRange rac.Range
|
||||
buf [4]byte
|
||||
}
|
||||
|
||||
// Load reads and unwraps a wrapped dictionary.
|
||||
func (r *Loader) Load(rs io.ReadSeeker, chunk rac.Chunk) (dictionary []byte, retErr error) {
|
||||
if !chunk.CTertiary.Empty() {
|
||||
return nil, errInvalidDictionary
|
||||
}
|
||||
if chunk.CSecondary.Empty() {
|
||||
return nil, nil
|
||||
}
|
||||
cRange := chunk.CSecondary
|
||||
|
||||
// Load from the MRU cache, if it was loaded from the same cRange.
|
||||
if (cRange == r.cachedRange) && !cRange.Empty() {
|
||||
return r.cachedBytes, nil
|
||||
}
|
||||
|
||||
// Check the cRange size and the tTag.
|
||||
if (cRange.Size() < 8) || (chunk.TTag != 0xFF) {
|
||||
return nil, errInvalidDictionary
|
||||
}
|
||||
|
||||
// Read the dictionary size.
|
||||
if _, err := rs.Seek(cRange[0], io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := io.ReadFull(rs, r.buf[:4]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check that the high 2 bits are zero (reserved). In other words, we
|
||||
// enforce that the maximum (inclusive) length is MaxInclLength.
|
||||
dictSize := int64(u32LE(r.buf[:4]))
|
||||
if (dictSize >> 30) != 0 {
|
||||
return nil, errInvalidDictionary
|
||||
}
|
||||
|
||||
// Check the overall size. The +8 is for the 4 byte prefix (dictionary
|
||||
// size) and the 4 byte suffix (checksum).
|
||||
if (dictSize + 8) > cRange.Size() {
|
||||
return nil, errInvalidDictionary
|
||||
}
|
||||
|
||||
// Allocate or re-use the cachedBytes buffer.
|
||||
buffer := []byte(nil)
|
||||
if n := dictSize + 4; int64(cap(r.cachedBytes)) >= n {
|
||||
buffer = r.cachedBytes[:n]
|
||||
// Invalidate the cached dictionary, as we are re-using its memory.
|
||||
r.cachedRange = rac.Range{}
|
||||
} else {
|
||||
buffer = make([]byte, n)
|
||||
}
|
||||
|
||||
// Read the dictionary and checksum.
|
||||
if _, err := io.ReadFull(rs, buffer); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Verify the checksum.
|
||||
dict, checksum := buffer[:dictSize], buffer[dictSize:]
|
||||
if u32LE(checksum) != crc32.ChecksumIEEE(dict) {
|
||||
return nil, errInvalidDictionary
|
||||
}
|
||||
|
||||
// Save to the MRU cache and return.
|
||||
r.cachedBytes = dict
|
||||
r.cachedRange = cRange
|
||||
return dict, nil
|
||||
}
|
||||
|
||||
// Saver saves a dictionary wrapped in the RAC common dictionary format.
|
||||
type Saver struct {
|
||||
stash []byte
|
||||
}
|
||||
|
||||
// WrapResource wraps a dictionary.
|
||||
func (w *Saver) WrapResource(
|
||||
raw []byte,
|
||||
refineResourceData func([]byte) []byte,
|
||||
) ([]byte, error) {
|
||||
refined := refineResourceData(raw)
|
||||
if len(refined) > MaxInclLength {
|
||||
return nil, errDictionaryIsTooLong
|
||||
}
|
||||
wrapped := make([]byte, len(refined)+8)
|
||||
wrapped[0] = uint8(len(refined) >> 0)
|
||||
wrapped[1] = uint8(len(refined) >> 8)
|
||||
wrapped[2] = uint8(len(refined) >> 16)
|
||||
wrapped[3] = uint8(len(refined) >> 24)
|
||||
copy(wrapped[4:], refined)
|
||||
checksum := crc32.ChecksumIEEE(refined)
|
||||
wrapped[len(wrapped)-4] = uint8(checksum >> 0)
|
||||
wrapped[len(wrapped)-3] = uint8(checksum >> 8)
|
||||
wrapped[len(wrapped)-2] = uint8(checksum >> 16)
|
||||
wrapped[len(wrapped)-1] = uint8(checksum >> 24)
|
||||
return wrapped, nil
|
||||
}
|
||||
|
||||
// Compress helps implement rac.CodecWriter.
|
||||
func (w *Saver) Compress(
|
||||
p []byte,
|
||||
q []byte,
|
||||
resourcesData [][]byte,
|
||||
codec rac.Codec,
|
||||
compress func(p []byte, q []byte, dict []byte) ([]byte, error),
|
||||
refineResourceData func([]byte) []byte,
|
||||
) (
|
||||
retCodec rac.Codec,
|
||||
compressed []byte,
|
||||
secondaryResource int,
|
||||
tertiaryResource int,
|
||||
retErr error,
|
||||
) {
|
||||
secondaryResource = rac.NoResourceUsed
|
||||
|
||||
// Compress p+q without any shared dictionary.
|
||||
baseline, err := compress(p, q, nil)
|
||||
if err != nil {
|
||||
return 0, nil, 0, 0, err
|
||||
}
|
||||
// If there aren't any potential shared dictionaries, or if the baseline
|
||||
// compressed form is small, just return the baseline.
|
||||
const minBaselineSizeToConsiderDictionaries = 256
|
||||
if (len(resourcesData) == 0) || (len(baseline) < minBaselineSizeToConsiderDictionaries) {
|
||||
return codec, baseline, secondaryResource, rac.NoResourceUsed, nil
|
||||
}
|
||||
|
||||
// w.stash keeps a copy of the best compression so far. Every call to
|
||||
// compress can clobber the bytes previously returned by compress.
|
||||
w.stash = append(w.stash[:0], baseline...)
|
||||
compressed = w.stash
|
||||
|
||||
// Only use a shared dictionary if it results in something smaller than
|
||||
// 98.4375% (an arbitrary heuristic threshold) of the baseline. Otherwise,
|
||||
// it's arguably not worth the additional complexity.
|
||||
threshold := (len(baseline) / 64) * 63
|
||||
|
||||
for i, resourceData := range resourcesData {
|
||||
refined := refineResourceData(resourceData)
|
||||
if len(refined) > MaxInclLength {
|
||||
return 0, nil, 0, 0, errDictionaryIsTooLong
|
||||
}
|
||||
|
||||
candidate, err := compress(p, q, refined)
|
||||
if err != nil {
|
||||
return 0, nil, 0, 0, err
|
||||
}
|
||||
if n := len(candidate); (n >= threshold) || (n >= len(compressed)) {
|
||||
continue
|
||||
}
|
||||
secondaryResource = i
|
||||
|
||||
// As an optimization, if the final candidate is the winner, we don't
|
||||
// have to copy its contents to w.stash.
|
||||
if i == len(resourcesData)-1 {
|
||||
compressed = candidate
|
||||
} else {
|
||||
w.stash = append(w.stash[:0], candidate...)
|
||||
compressed = w.stash
|
||||
}
|
||||
}
|
||||
return codec, compressed, secondaryResource, rac.NoResourceUsed, nil
|
||||
}
|
||||
+30
-12
@@ -28,19 +28,33 @@ import (
|
||||
|
||||
"github.com/google/wuffs/lib/cgozstd"
|
||||
"github.com/google/wuffs/lib/compression"
|
||||
"github.com/google/wuffs/lib/internal/racdict"
|
||||
"github.com/google/wuffs/lib/rac"
|
||||
)
|
||||
|
||||
var (
|
||||
errCannotCut = errors.New("raczstd: cannot cut")
|
||||
errInvalidCodec = errors.New("raczstd: invalid codec")
|
||||
errCannotCut = errors.New("raczstd: cannot cut")
|
||||
)
|
||||
|
||||
func refine(b []byte) []byte {
|
||||
if len(b) > racdict.MaxInclLength {
|
||||
return b[len(b)-racdict.MaxInclLength:]
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// CodecReader specializes a rac.Reader to decode Zstd-compressed chunks.
|
||||
type CodecReader struct {
|
||||
// cachedReader lets us re-use the memory allocated for a zstd reader, when
|
||||
// decompressing multiple chunks.
|
||||
cachedReader compression.Reader
|
||||
recycler cgozstd.ReaderRecycler
|
||||
lim io.LimitedReader
|
||||
|
||||
// lim provides a limited view of a RAC file.
|
||||
lim io.LimitedReader
|
||||
|
||||
// dictLoader loads shared dictionaries.
|
||||
dictLoader racdict.Loader
|
||||
}
|
||||
|
||||
// Close implements rac.CodecReader.
|
||||
@@ -60,6 +74,10 @@ func (r *CodecReader) Clone() rac.CodecReader {
|
||||
|
||||
// MakeDecompressor implements rac.CodecReader.
|
||||
func (r *CodecReader) MakeDecompressor(racFile io.ReadSeeker, chunk rac.Chunk) (io.Reader, error) {
|
||||
dict, err := r.dictLoader.Load(racFile, chunk)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := racFile.Seek(chunk.CPrimary[0], io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -71,7 +89,7 @@ func (r *CodecReader) MakeDecompressor(racFile io.ReadSeeker, chunk rac.Chunk) (
|
||||
r.cachedReader = zr
|
||||
r.recycler.Bind(zr)
|
||||
}
|
||||
if err := r.cachedReader.Reset(&r.lim, nil); err != nil {
|
||||
if err := r.cachedReader.Reset(&r.lim, dict); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.cachedReader, nil
|
||||
@@ -82,6 +100,9 @@ type CodecWriter struct {
|
||||
compressed bytes.Buffer
|
||||
cachedWriter compression.Writer
|
||||
recycler cgozstd.WriterRecycler
|
||||
|
||||
// dictSaver saves shared dictionaries.
|
||||
dictSaver racdict.Saver
|
||||
}
|
||||
|
||||
// Close implements rac.CodecWriter.
|
||||
@@ -97,13 +118,10 @@ func (w *CodecWriter) Clone() rac.CodecWriter {
|
||||
// Compress implements rac.CodecWriter.
|
||||
func (w *CodecWriter) Compress(p []byte, q []byte, resourcesData [][]byte) (
|
||||
codec rac.Codec, compressed []byte, secondaryResource int, tertiaryResource int, retErr error) {
|
||||
|
||||
// Compress p+q without any shared dictionary.
|
||||
baseline, err := w.compress(p, q, nil)
|
||||
if err != nil {
|
||||
return 0, nil, 0, 0, err
|
||||
}
|
||||
return rac.CodecZstandard, baseline, rac.NoResourceUsed, rac.NoResourceUsed, nil
|
||||
return w.dictSaver.Compress(
|
||||
p, q, resourcesData,
|
||||
rac.CodecZstandard, w.compress, refine,
|
||||
)
|
||||
}
|
||||
|
||||
func (w *CodecWriter) compress(p []byte, q []byte, dict []byte) ([]byte, error) {
|
||||
@@ -148,5 +166,5 @@ func (w *CodecWriter) Cut(codec rac.Codec, encoded []byte, maxEncodedLen int) (e
|
||||
|
||||
// WrapResource implements rac.CodecWriter.
|
||||
func (w *CodecWriter) WrapResource(raw []byte) ([]byte, error) {
|
||||
return nil, nil
|
||||
return w.dictSaver.WrapResource(raw, refine)
|
||||
}
|
||||
Reference in new issue
Block a user