mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-21 00:47:28 +01:00
Let rac.Writer support shared resources
This commit is contained in:
1 parent
1cb1f1d527
commit
ac4fed2275
5 files changed
+284
-46
No files matched your search
+103
-13
@@ -60,6 +60,18 @@ func readAt(r io.ReadSeeker, p []byte, offset int64) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// suffix32K returns the last 32 KiB of b, or if b is shorter than that, it
|
||||
// returns just b.
|
||||
//
|
||||
// The Zlib format only supports up to 32 KiB of history or shared dictionary.
|
||||
func suffix32K(b []byte) []byte {
|
||||
const n = 32 * 1024
|
||||
if len(b) > n {
|
||||
return b[len(b)-n:]
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// CodecReader specializes a rac.Reader to decode Zlib-compressed chunks.
|
||||
type CodecReader struct {
|
||||
// cachedZlibReader lets us re-use the memory allocated for a zlib reader,
|
||||
@@ -164,38 +176,99 @@ func (r *CodecReader) MakeReaderContext(
|
||||
|
||||
// CodecWriter specializes a rac.Writer to encode Zlib-compressed chunks.
|
||||
type CodecWriter struct {
|
||||
// These fields help reduce memory allocation by re-using previous byte
|
||||
// buffers or the zlib.Writer.
|
||||
compressed bytes.Buffer
|
||||
zlibWriter *zlib.Writer
|
||||
stash []byte
|
||||
}
|
||||
|
||||
// Compress implements rac.CodecWriter.
|
||||
func (w *CodecWriter) Compress(p []byte, q []byte) (rac.Codec, []byte, error) {
|
||||
w.compressed.Reset()
|
||||
func (w *CodecWriter) Compress(p []byte, q []byte, resourcesData [][]byte) (
|
||||
codec rac.Codec, compressed []byte, secondaryResource int, tertiaryResource int, retErr error) {
|
||||
|
||||
if w.zlibWriter == nil {
|
||||
zw, err := zlib.NewWriterLevel(&w.compressed, zlib.BestCompression)
|
||||
secondaryResource = rac.NoResourceUsed
|
||||
|
||||
// Compress p+q without any shared dictionary.
|
||||
baseline, err := w.compress(p, q, nil)
|
||||
if err != nil {
|
||||
return 0, nil, 0, 0, err
|
||||
}
|
||||
// If there aren't any potential shared dictionaries, or if the baseline
|
||||
// compressed form is small, just return the baseline.
|
||||
const minBaselineSizeToConsiderDictionaries = 256
|
||||
if (len(resourcesData) == 0) || (len(baseline) < minBaselineSizeToConsiderDictionaries) {
|
||||
return rac.CodecZlib, baseline, secondaryResource, rac.NoResourceUsed, nil
|
||||
}
|
||||
|
||||
// w.stash keeps a copy of the best compression so far. Every call to
|
||||
// w.compress can clobber the bytes previously returned by w.compress.
|
||||
w.stash = append(w.stash[:0], baseline...)
|
||||
compressed = w.stash
|
||||
|
||||
// Only use a shared dictionary if it results in something smaller than
|
||||
// 98.4375% (an arbitrary heuristic threshold) of the baseline. Otherwise,
|
||||
// it's arguably not worth the additional complexity.
|
||||
threshold := (len(baseline) / 64) * 63
|
||||
|
||||
for i, data := range resourcesData {
|
||||
candidate, err := w.compress(p, q, suffix32K(data))
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
return 0, nil, 0, 0, err
|
||||
}
|
||||
if n := len(candidate); (n >= threshold) || (n >= len(compressed)) {
|
||||
continue
|
||||
}
|
||||
secondaryResource = i
|
||||
|
||||
// As an optimization, if the final candidate is the winner, we don't
|
||||
// have to copy its contents to w.stash.
|
||||
if i == len(resourcesData)-1 {
|
||||
compressed = candidate
|
||||
} else {
|
||||
w.stash = append(w.stash[:0], candidate...)
|
||||
compressed = w.stash
|
||||
}
|
||||
}
|
||||
return rac.CodecZlib, compressed, secondaryResource, rac.NoResourceUsed, nil
|
||||
}
|
||||
|
||||
func (w *CodecWriter) compress(p []byte, q []byte, dict []byte) ([]byte, error) {
|
||||
w.compressed.Reset()
|
||||
zw, err := (*zlib.Writer)(nil), error(nil)
|
||||
if len(dict) != 0 {
|
||||
// A zlib.Writer doesn't have a Reset(etc, dict) method, so we have to
|
||||
// allocate a new zlib.Writer.
|
||||
zw, err = zlib.NewWriterLevelDict(&w.compressed, zlib.BestCompression, dict)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if w.zlibWriter == nil {
|
||||
zw, err = zlib.NewWriterLevelDict(&w.compressed, zlib.BestCompression, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w.zlibWriter = zw
|
||||
} else {
|
||||
w.zlibWriter.Reset(&w.compressed)
|
||||
zw = w.zlibWriter
|
||||
zw.Reset(&w.compressed)
|
||||
}
|
||||
|
||||
if len(p) > 0 {
|
||||
if _, err := w.zlibWriter.Write(p); err != nil {
|
||||
return 0, nil, err
|
||||
if _, err := zw.Write(p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if len(q) > 0 {
|
||||
if _, err := w.zlibWriter.Write(q); err != nil {
|
||||
return 0, nil, err
|
||||
if _, err := zw.Write(q); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := w.zlibWriter.Close(); err != nil {
|
||||
return 0, nil, err
|
||||
|
||||
if err := zw.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rac.CodecZlib, w.compressed.Bytes(), nil
|
||||
return w.compressed.Bytes(), nil
|
||||
}
|
||||
|
||||
// Cut implements rac.CodecWriter.
|
||||
@@ -205,3 +278,20 @@ func (w *CodecWriter) Cut(codec rac.Codec, encoded []byte, maxEncodedLen int) (e
|
||||
}
|
||||
return zlibcut.Cut(nil, encoded, maxEncodedLen)
|
||||
}
|
||||
|
||||
// WrapResource implements rac.CodecWriter.
|
||||
func (w *CodecWriter) WrapResource(raw []byte) ([]byte, error) {
|
||||
// For a description of the RAC+Zlib secondary-data format, see
|
||||
// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib
|
||||
raw = suffix32K(raw)
|
||||
wrapped := make([]byte, len(raw)+6)
|
||||
wrapped[0] = uint8(len(raw) >> 0)
|
||||
wrapped[1] = uint8(len(raw) >> 8)
|
||||
copy(wrapped[2:], raw)
|
||||
a := adler32.Checksum(raw)
|
||||
wrapped[len(wrapped)-4] = uint8(a >> 24)
|
||||
wrapped[len(wrapped)-3] = uint8(a >> 16)
|
||||
wrapped[len(wrapped)-2] = uint8(a >> 8)
|
||||
wrapped[len(wrapped)-1] = uint8(a >> 0)
|
||||
return wrapped, nil
|
||||
}
|
||||
+86
-18
@@ -17,9 +17,9 @@ package raczlib
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/wuffs/lib/rac"
|
||||
@@ -56,17 +56,42 @@ const (
|
||||
"\x28\x4A\x4D\x85\x71\x00\x01\x00\x00\xFF\xFF\x21\x6E\x04\x66"
|
||||
)
|
||||
|
||||
func testReader(t *testing.T, decoded string, encoded string) {
|
||||
func racCompress(original []byte, dChunkSize uint64, resourcesData [][]byte) ([]byte, error) {
|
||||
buf := &bytes.Buffer{}
|
||||
w := &rac.Writer{
|
||||
Writer: buf,
|
||||
CodecWriter: &CodecWriter{},
|
||||
DChunkSize: dChunkSize,
|
||||
ResourcesData: resourcesData,
|
||||
}
|
||||
if _, err := w.Write(original); err != nil {
|
||||
return nil, fmt.Errorf("Write: %v", err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, fmt.Errorf("Close: %v", err)
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func racDecompress(compressed []byte) ([]byte, error) {
|
||||
buf := &bytes.Buffer{}
|
||||
r := &rac.Reader{
|
||||
ReadSeeker: strings.NewReader(encoded),
|
||||
CompressedSize: int64(len(encoded)),
|
||||
ReadSeeker: bytes.NewReader(compressed),
|
||||
CompressedSize: int64(len(compressed)),
|
||||
CodecReaders: []rac.CodecReader{&CodecReader{}},
|
||||
}
|
||||
if _, err := io.Copy(buf, r); err != nil {
|
||||
t.Fatalf("Copy: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
if got, want := buf.String(), decoded; got != want {
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func testReader(t *testing.T, decoded string, encoded string) {
|
||||
g, err := racDecompress([]byte(encoded))
|
||||
if err != nil {
|
||||
t.Fatalf("racDecompress: %v", err)
|
||||
}
|
||||
if got, want := string(g), decoded; got != want {
|
||||
t.Fatalf("got:\n%s\nwant:\n%s", got, want)
|
||||
}
|
||||
}
|
||||
@@ -151,19 +176,10 @@ func TestReaderConcatenation(t *testing.T) {
|
||||
|
||||
func TestZeroedBytes(t *testing.T) {
|
||||
original := []byte("abcde\x00\x00\x00\x00j")
|
||||
cBuf := &bytes.Buffer{}
|
||||
w := &rac.Writer{
|
||||
Writer: cBuf,
|
||||
CodecWriter: &CodecWriter{},
|
||||
DChunkSize: 8,
|
||||
compressed, err := racCompress(original, 8, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write(original); err != nil {
|
||||
t.Fatalf("Write: %v", err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
compressed := cBuf.Bytes()
|
||||
|
||||
r := &rac.Reader{
|
||||
ReadSeeker: bytes.NewReader(compressed),
|
||||
@@ -191,3 +207,55 @@ func TestZeroedBytes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedDictionary(t *testing.T) {
|
||||
// Make some "dictionary" data that, as an independent chunk, does not
|
||||
// compress very well.
|
||||
const n = 256
|
||||
dictionary := make([]byte, n)
|
||||
for i := range dictionary {
|
||||
dictionary[i] = uint8(i)
|
||||
}
|
||||
|
||||
// Replicate it 32 times.
|
||||
original := make([]byte, 0, 32*n)
|
||||
for len(original) < 32*n {
|
||||
original = append(original, dictionary...)
|
||||
}
|
||||
|
||||
// Measure the RAC-compressed form of that replicated data, without and
|
||||
// with a shared dictionary.
|
||||
compressedLengths := [2]int{}
|
||||
for i := range compressedLengths {
|
||||
resourcesData := [][]byte{}
|
||||
if i > 0 {
|
||||
resourcesData = [][]byte{dictionary}
|
||||
}
|
||||
|
||||
// Compress.
|
||||
compressed, err := racCompress(original, n, resourcesData)
|
||||
if err != nil {
|
||||
t.Fatalf("i=%d: racCompress: %v", i, err)
|
||||
}
|
||||
if len(compressed) == 0 {
|
||||
t.Fatalf("i=%d: compressed form is empty", i)
|
||||
}
|
||||
compressedLengths[i] = len(compressed)
|
||||
|
||||
// Decompress.
|
||||
decompressed, err := racDecompress(compressed)
|
||||
if err != nil {
|
||||
t.Fatalf("i=%d: racDecompress: %v", i, err)
|
||||
}
|
||||
if !bytes.Equal(decompressed, original) {
|
||||
t.Fatalf("i=%d: racDecompress: round trip did not match original", i)
|
||||
}
|
||||
}
|
||||
|
||||
// Using a shared dictionary should improve the compression ratio. The
|
||||
// exact value depends on the Zlib compression algorithm, but we should
|
||||
// expect at least a 4x improvement.
|
||||
if ratio := compressedLengths[0] / compressedLengths[1]; ratio < 4 {
|
||||
t.Fatalf("ratio: got %dx, want at least 4x", ratio)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user