Change raczlib's dictionary wire format

This makes raczlib and raczstd use the same format: RAC's "common
dictionary format".

This is a breaking change to any existing RAC+Zlib files that use shared
dictionaries. RAC+Zlib is still an experimental (unstable) file format.
This commit is contained in:
Nigel Tao committed 2019-10-24 10:47:48 +11:00
1 parent cdb60a950d
commit 8ac2af0122
5 files changed
+98 -223

No files matched your search

+14 -133
View File
@@ -25,10 +25,10 @@ import (
"bytes"
"compress/zlib"
"errors"
"hash/adler32"
"io"
"github.com/google/wuffs/lib/compression"
"github.com/google/wuffs/lib/internal/racdict"
"github.com/google/wuffs/lib/rac"
"github.com/google/wuffs/lib/zlibcut"
)
@@ -43,11 +43,11 @@ func u32BE(b []byte) uint32 {
return (uint32(b[0]) << 24) | (uint32(b[1]) << 16) | (uint32(b[2]) << 8) | (uint32(b[3]))
}
// suffix32K returns the last 32 KiB of b, or if b is shorter than that, it
// refine returns the last 32 KiB of b, or if b is shorter than that, it
// returns just b.
//
// The Zlib format only supports up to 32 KiB of history or shared dictionary.
func suffix32K(b []byte) []byte {
func refine(b []byte) []byte {
const n = 32 * 1024
if len(b) > n {
return b[len(b)-n:]
@@ -64,12 +64,8 @@ type CodecReader struct {
// lim provides a limited view of a RAC file.
lim io.LimitedReader
// These fields contain the most recently used shared dictionary.
cachedDictionary []byte
cachedDictionaryCRange rac.Range
// buf is a scratch buffer.
buf [2]byte
// dictLoader loads shared dictionaries.
dictLoader racdict.Loader
}
// Close implements rac.CodecReader.
@@ -89,7 +85,7 @@ func (r *CodecReader) Clone() rac.CodecReader {
// MakeDecompressor implements rac.CodecReader.
func (r *CodecReader) MakeDecompressor(racFile io.ReadSeeker, chunk rac.Chunk) (io.Reader, error) {
dict, err := r.loadDictionary(racFile, chunk)
dict, err := r.dictLoader.Load(racFile, chunk)
if err != nil {
return nil, err
}
@@ -101,77 +97,15 @@ func (r *CodecReader) MakeDecompressor(racFile io.ReadSeeker, chunk rac.Chunk) (
return r.makeDecompressor(&r.lim, dict)
}
func (r *CodecReader) loadDictionary(rs io.ReadSeeker, chunk rac.Chunk) ([]byte, error) {
// For a description of the RAC+Zlib secondary-data format, see
// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib
if !chunk.CTertiary.Empty() {
return nil, errInvalidDictionary
}
if chunk.CSecondary.Empty() {
return nil, nil
}
cRange := chunk.CSecondary
// Load from the MRU cache, if it was loaded from the same cRange.
if (cRange == r.cachedDictionaryCRange) && !cRange.Empty() {
return r.cachedDictionary, nil
}
// Check the cRange size and the tTag.
if (cRange.Size() < 6) || (chunk.TTag != 0xFF) {
return nil, errInvalidDictionary
}
// Read the dictionary size.
if _, err := rs.Seek(cRange[0], io.SeekStart); err != nil {
return nil, errInvalidDictionary
}
if _, err := io.ReadFull(rs, r.buf[:2]); err != nil {
return nil, errInvalidDictionary
}
dictSize := int64(r.buf[0]) | (int64(r.buf[1]) << 8)
// Check the size. The +6 is for the 2 byte prefix (dictionary size) and
// the 4 byte suffix (checksum).
if (dictSize + 6) > cRange.Size() {
return nil, errInvalidDictionary
}
// Allocate or re-use the cachedDictionary buffer.
buffer := []byte(nil)
if n := dictSize + 4; int64(cap(r.cachedDictionary)) >= n {
buffer = r.cachedDictionary[:n]
// Invalidate the cached dictionary, as we are re-using its memory.
r.cachedDictionaryCRange = rac.Range{}
} else {
buffer = make([]byte, n)
}
// Read the dictionary and checksum.
if _, err := io.ReadFull(rs, buffer); err != nil {
return nil, errInvalidDictionary
}
// Verify the checksum.
dict, checksum := buffer[:dictSize], buffer[dictSize:]
if u32BE(checksum) != adler32.Checksum(dict) {
return nil, errInvalidDictionary
}
// Save to the MRU cache and return.
r.cachedDictionary = dict
r.cachedDictionaryCRange = cRange
return dict, nil
}
// CodecWriter specializes a rac.Writer to encode Zlib-compressed chunks.
type CodecWriter struct {
// These fields help reduce memory allocation by re-using previous byte
// buffers or the zlib.Writer.
compressed bytes.Buffer
cachedWriter *zlib.Writer
stash []byte
// dictSaver saves shared dictionaries.
dictSaver racdict.Saver
}
// Close implements rac.CodecWriter.
@@ -187,51 +121,10 @@ func (w *CodecWriter) Clone() rac.CodecWriter {
// Compress implements rac.CodecWriter.
func (w *CodecWriter) Compress(p []byte, q []byte, resourcesData [][]byte) (
codec rac.Codec, compressed []byte, secondaryResource int, tertiaryResource int, retErr error) {
secondaryResource = rac.NoResourceUsed
// Compress p+q without any shared dictionary.
baseline, err := w.compress(p, q, nil)
if err != nil {
return 0, nil, 0, 0, err
}
// If there aren't any potential shared dictionaries, or if the baseline
// compressed form is small, just return the baseline.
const minBaselineSizeToConsiderDictionaries = 256
if (len(resourcesData) == 0) || (len(baseline) < minBaselineSizeToConsiderDictionaries) {
return rac.CodecZlib, baseline, secondaryResource, rac.NoResourceUsed, nil
}
// w.stash keeps a copy of the best compression so far. Every call to
// w.compress can clobber the bytes previously returned by w.compress.
w.stash = append(w.stash[:0], baseline...)
compressed = w.stash
// Only use a shared dictionary if it results in something smaller than
// 98.4375% (an arbitrary heuristic threshold) of the baseline. Otherwise,
// it's arguably not worth the additional complexity.
threshold := (len(baseline) / 64) * 63
for i, data := range resourcesData {
candidate, err := w.compress(p, q, suffix32K(data))
if err != nil {
return 0, nil, 0, 0, err
}
if n := len(candidate); (n >= threshold) || (n >= len(compressed)) {
continue
}
secondaryResource = i
// As an optimization, if the final candidate is the winner, we don't
// have to copy its contents to w.stash.
if i == len(resourcesData)-1 {
compressed = candidate
} else {
w.stash = append(w.stash[:0], candidate...)
compressed = w.stash
}
}
return rac.CodecZlib, compressed, secondaryResource, rac.NoResourceUsed, nil
return w.dictSaver.Compress(
p, q, resourcesData,
rac.CodecZlib, w.compress, refine,
)
}
func (w *CodecWriter) compress(p []byte, q []byte, dict []byte) ([]byte, error) {
@@ -289,17 +182,5 @@ func (w *CodecWriter) Cut(codec rac.Codec, encoded []byte, maxEncodedLen int) (e
// WrapResource implements rac.CodecWriter.
func (w *CodecWriter) WrapResource(raw []byte) ([]byte, error) {
// For a description of the RAC+Zlib secondary-data format, see
// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib
raw = suffix32K(raw)
wrapped := make([]byte, len(raw)+6)
wrapped[0] = uint8(len(raw) >> 0)
wrapped[1] = uint8(len(raw) >> 8)
copy(wrapped[2:], raw)
a := adler32.Checksum(raw)
wrapped[len(wrapped)-4] = uint8(a >> 24)
wrapped[len(wrapped)-3] = uint8(a >> 16)
wrapped[len(wrapped)-2] = uint8(a >> 8)
wrapped[len(wrapped)-1] = uint8(a >> 0)
return wrapped, nil
return w.dictSaver.WrapResource(raw, refine)
}
+9 -8
View File
@@ -45,16 +45,17 @@ const (
"\x00\x00\x00\x01\x01"
encodedSheep = "" +
"\x72\xC3\x63\x04\x71\xB5\x00\xFF\x00\x00\x00\x00\x00\x00\x00\xFF" +
"\x72\xC3\x63\x04\x37\x39\x00\xFF\x00\x00\x00\x00\x00\x00\x00\xFF" +
"\x0B\x00\x00\x00\x00\x00\x00\xFF\x16\x00\x00\x00\x00\x00\x00\xFF" +
"\x23\x00\x00\x00\x00\x00\x00\x01\x50\x00\x00\x00\x00\x00\x01\xFF" +
"\x5E\x00\x00\x00\x00\x00\x01\x00\x73\x00\x00\x00\x00\x00\x01\x00" +
"\x88\x00\x00\x00\x00\x00\x01\x00\x9F\x00\x00\x00\x00\x00\x01\x04" +
"\x08\x00\x20\x73\x68\x65\x65\x70\x2E\x0A\x0B\xE0\x02\x6E\x78\xF9" +
"\x0B\xE0\x02\x6E\xF2\xCF\x4B\x85\x31\x01\x01\x00\x00\xFF\xFF\x17" +
"\x21\x03\x90\x78\xF9\x0B\xE0\x02\x6E\x0A\x29\xCF\x87\x31\x01\x01" +
"\x00\x00\xFF\xFF\x18\x0C\x03\xA8\x78\xF9\x0B\xE0\x02\x6E\x0A\xC9" +
"\x28\x4A\x4D\x85\x71\x00\x01\x00\x00\xFF\xFF\x21\x6E\x04\x66"
"\x60\x00\x00\x00\x00\x00\x01\x00\x75\x00\x00\x00\x00\x00\x01\x00" +
"\x8A\x00\x00\x00\x00\x00\x01\x00\xA1\x00\x00\x00\x00\x00\x01\x04" +
"\x08\x00\x00\x00\x20\x73\x68\x65\x65\x70\x2E\x0A\xD0\x8D\x7A\x47" +
"\x78\xF9\x0B\xE0\x02\x6E\xF2\xCF\x4B\x85\x31\x01\x01\x00\x00\xFF" +
"\xFF\x17\x21\x03\x90\x78\xF9\x0B\xE0\x02\x6E\x0A\x29\xCF\x87\x31" +
"\x01\x01\x00\x00\xFF\xFF\x18\x0C\x03\xA8\x78\xF9\x0B\xE0\x02\x6E" +
"\x0A\xC9\x28\x4A\x4D\x85\x71\x00\x01\x00\x00\xFF\xFF\x21\x6E\x04" +
"\x66"
)
func racCompress(original []byte, cChunkSize uint64, dChunkSize uint64, resourcesData [][]byte) ([]byte, error) {