mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 00:32:11 +01:00
Calculate deflate Huffman table worst case size
This commit is contained in:
1 parent
358ad7f918
commit
ff7bffcffc
9 files changed
+1008
-63
No files matched your search
+160
-57
@@ -244,47 +244,83 @@ var deflateGlobals struct {
|
||||
stream deflateBitStream
|
||||
|
||||
// Dynamic Huffman state.
|
||||
numLCodes uint32
|
||||
numDCodes uint32
|
||||
numCLCodeLengths uint32
|
||||
whichHuffman uint32
|
||||
whichHuffman uint32
|
||||
// 0=Unused, 1=CodeLength, 2=Length/Literal, 3=Distance.
|
||||
huffmans [4]deflateHuffmanTable
|
||||
|
||||
// DHH (Dynamic Huffman, inside a Huffman table) state.
|
||||
prevLine string
|
||||
etcetera bool
|
||||
}
|
||||
|
||||
func deflateGlobalsClearDynamicHuffmanState() {
|
||||
deflateGlobals.numLCodes = 0
|
||||
deflateGlobals.numDCodes = 0
|
||||
deflateGlobals.numCLCodeLengths = 0
|
||||
deflateGlobals.whichHuffman = 0
|
||||
deflateGlobals.huffmans = [4]deflateHuffmanTable{}
|
||||
deflateGlobals.prevLine = ""
|
||||
deflateGlobals.etcetera = false
|
||||
}
|
||||
|
||||
func deflateGlobalsCountCodes() (numLCodes uint32, numDCodes uint32, numCLCodeLengths uint32, retErr error) {
|
||||
for k := range deflateGlobals.huffmans[2] {
|
||||
if numLCodes < (k + 1) {
|
||||
numLCodes = (k + 1)
|
||||
}
|
||||
}
|
||||
|
||||
for k := range deflateGlobals.huffmans[3] {
|
||||
if numDCodes < (k + 1) {
|
||||
numDCodes = (k + 1)
|
||||
}
|
||||
}
|
||||
|
||||
for k := range deflateGlobals.huffmans[1] {
|
||||
if (k < 0) || (18 < k) {
|
||||
return 0, 0, 0, fmt.Errorf("bad CodeLength: %d", k)
|
||||
}
|
||||
}
|
||||
for i := len(deflateCodeOrder) - 1; i >= 0; i-- {
|
||||
cl := deflateCodeOrder[i]
|
||||
if _, ok := deflateGlobals.huffmans[1][cl]; ok {
|
||||
numCLCodeLengths = uint32(i + 1)
|
||||
break
|
||||
}
|
||||
}
|
||||
if numCLCodeLengths < 4 {
|
||||
numCLCodeLengths = 4
|
||||
}
|
||||
|
||||
return numLCodes, numDCodes, numCLCodeLengths, nil
|
||||
}
|
||||
|
||||
func deflateGlobalsWriteDynamicHuffmanTables() error {
|
||||
g := &deflateGlobals
|
||||
if (g.numLCodes < 257) || (257+31 < g.numLCodes) {
|
||||
return fmt.Errorf("bad numLCodes: %d", g.numLCodes)
|
||||
numLCodes, numDCodes, numCLCodeLengths, err := deflateGlobalsCountCodes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g.stream.writeBits(g.numLCodes-257, 5)
|
||||
if (g.numDCodes < 1) || (1+31 < g.numDCodes) {
|
||||
return fmt.Errorf("bad numDCodes: %d", g.numDCodes)
|
||||
if (numLCodes < 257) || (257+31 < numLCodes) {
|
||||
return fmt.Errorf("bad numLCodes: %d", numLCodes)
|
||||
}
|
||||
g.stream.writeBits(g.numDCodes-1, 5)
|
||||
if (g.numCLCodeLengths < 4) || (4+15 < g.numCLCodeLengths) {
|
||||
return fmt.Errorf("bad numCLCodeLengths: %d", g.numCLCodeLengths)
|
||||
g.stream.writeBits(numLCodes-257, 5)
|
||||
if (numDCodes < 1) || (1+31 < numDCodes) {
|
||||
return fmt.Errorf("bad numDCodes: %d", numDCodes)
|
||||
}
|
||||
g.stream.writeBits(g.numCLCodeLengths-4, 4)
|
||||
g.stream.writeBits(numDCodes-1, 5)
|
||||
if (numCLCodeLengths < 4) || (4+15 < numCLCodeLengths) {
|
||||
return fmt.Errorf("bad numCLCodeLengths: %d", numCLCodeLengths)
|
||||
}
|
||||
g.stream.writeBits(numCLCodeLengths-4, 4)
|
||||
|
||||
// Write the Huffman table for CodeLength.
|
||||
{
|
||||
for i := uint32(0); i < g.numCLCodeLengths; i++ {
|
||||
for i := uint32(0); i < numCLCodeLengths; i++ {
|
||||
n := len(g.huffmans[1][deflateCodeOrder[i]])
|
||||
g.stream.writeBits(uint32(n), 3)
|
||||
}
|
||||
for i := g.numCLCodeLengths; i < uint32(len(deflateCodeOrder)); i++ {
|
||||
for i := numCLCodeLengths; i < uint32(len(deflateCodeOrder)); i++ {
|
||||
n := len(g.huffmans[1][deflateCodeOrder[i]])
|
||||
if n > 0 {
|
||||
return fmt.Errorf("short numCLCodeLengths: %d", g.numCLCodeLengths)
|
||||
return fmt.Errorf("short numCLCodeLengths: %d", numCLCodeLengths)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -292,14 +328,14 @@ func deflateGlobalsWriteDynamicHuffmanTables() error {
|
||||
// Write the Huffman tables for Length/Literal and Distance.
|
||||
{
|
||||
numZeroes := uint32(0)
|
||||
for i := uint32(0); i < g.numLCodes+g.numDCodes; i++ {
|
||||
s := ""
|
||||
if i < g.numLCodes {
|
||||
s = g.huffmans[2][i]
|
||||
for i := uint32(0); i < numLCodes+numDCodes; i++ {
|
||||
codeLen := uint32(0)
|
||||
if i < numLCodes {
|
||||
codeLen = uint32(len(g.huffmans[2][i]))
|
||||
} else {
|
||||
s = g.huffmans[3][i-g.numLCodes]
|
||||
codeLen = uint32(len(g.huffmans[3][i-numLCodes]))
|
||||
}
|
||||
if s == "" {
|
||||
if codeLen == 0 {
|
||||
numZeroes++
|
||||
continue
|
||||
}
|
||||
@@ -309,7 +345,11 @@ func deflateGlobalsWriteDynamicHuffmanTables() error {
|
||||
}
|
||||
numZeroes = 0
|
||||
|
||||
deflateGlobalsWriteDynamicHuffmanBits(g.huffmans[1][uint32(len(s))])
|
||||
codeLenCode := g.huffmans[1][codeLen]
|
||||
if codeLenCode == "" {
|
||||
return fmt.Errorf("no code for code-length %d", codeLen)
|
||||
}
|
||||
deflateGlobalsWriteDynamicHuffmanBits(g.huffmans[1][codeLen])
|
||||
}
|
||||
if err := deflateGlobalsWriteDynamicHuffmanZeroes(numZeroes); err != nil {
|
||||
return err
|
||||
@@ -512,10 +552,7 @@ func stateDeflateFixedHuffman(line string) (stateFunc, error) {
|
||||
func stateDeflateDynamicHuffman(line string) (stateFunc, error) {
|
||||
g := &deflateGlobals
|
||||
const (
|
||||
cmdH = "huffman "
|
||||
cmdNCLCL = "numCLCodeLengths "
|
||||
cmdNDC = "numDCodes "
|
||||
cmdNLC = "numLCodes "
|
||||
cmdH = "huffman "
|
||||
)
|
||||
switch {
|
||||
case line == "}":
|
||||
@@ -530,33 +567,6 @@ func stateDeflateDynamicHuffman(line string) (stateFunc, error) {
|
||||
}
|
||||
g.whichHuffman = n
|
||||
return stateDeflateDynamicHuffmanHuffman, nil
|
||||
|
||||
case strings.HasPrefix(line, cmdNLC):
|
||||
s := line[len(cmdNLC):]
|
||||
n, s, ok := parseNum(s)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
g.numLCodes = n
|
||||
return stateDeflateDynamicHuffman, nil
|
||||
|
||||
case strings.HasPrefix(line, cmdNDC):
|
||||
s := line[len(cmdNDC):]
|
||||
n, s, ok := parseNum(s)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
g.numDCodes = n
|
||||
return stateDeflateDynamicHuffman, nil
|
||||
|
||||
case strings.HasPrefix(line, cmdNCLCL):
|
||||
s := line[len(cmdNCLCL):]
|
||||
n, s, ok := parseNum(s)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
g.numCLCodeLengths = n
|
||||
return stateDeflateDynamicHuffman, nil
|
||||
}
|
||||
|
||||
if lit, ok := deflateParseLiteral(line); ok {
|
||||
@@ -568,6 +578,25 @@ func stateDeflateDynamicHuffman(line string) (stateFunc, error) {
|
||||
deflateGlobalsWriteDynamicHuffmanBits(s)
|
||||
}
|
||||
return stateDeflateDynamicHuffman, nil
|
||||
|
||||
} else if l, d, ok := deflateParseLenDist(line); ok {
|
||||
if (l != 3) || (d != 2) {
|
||||
return nil, fmt.Errorf("TODO: support len/dist pairs for dynamic Huffman blocks")
|
||||
}
|
||||
// len 3 is code 257, with no extra bits.
|
||||
if s := g.huffmans[2][257]; s != "" {
|
||||
deflateGlobalsWriteDynamicHuffmanBits(s)
|
||||
} else {
|
||||
return nil, fmt.Errorf("no code for literal/length symbol 257")
|
||||
}
|
||||
// dist 2 is code 1, with no extra bits.
|
||||
if s := g.huffmans[3][1]; s != "" {
|
||||
deflateGlobalsWriteDynamicHuffmanBits(s)
|
||||
} else {
|
||||
return nil, fmt.Errorf("no code for distance symbol 2")
|
||||
}
|
||||
return stateDeflateDynamicHuffman, nil
|
||||
|
||||
} else if line == "endOfBlock" {
|
||||
s := g.huffmans[2][256]
|
||||
if s == "" {
|
||||
@@ -586,6 +615,8 @@ outer:
|
||||
switch {
|
||||
case line == "}":
|
||||
g.whichHuffman = 0
|
||||
g.prevLine = ""
|
||||
g.etcetera = false
|
||||
|
||||
// If we have all three Huffman tables, write them.
|
||||
for i := 1; ; i++ {
|
||||
@@ -602,6 +633,10 @@ outer:
|
||||
|
||||
return stateDeflateDynamicHuffman, nil
|
||||
|
||||
case line == "etcetera":
|
||||
g.etcetera = true
|
||||
return stateDeflateDynamicHuffmanHuffman, nil
|
||||
|
||||
default:
|
||||
s := line
|
||||
n, s, ok := parseNum(s)
|
||||
@@ -613,16 +648,84 @@ outer:
|
||||
break outer
|
||||
}
|
||||
}
|
||||
if (len(s) < 1) || (15 < len(s)) {
|
||||
return nil, fmt.Errorf("%q code length, %d, is out of range", s, len(s))
|
||||
}
|
||||
|
||||
if g.etcetera {
|
||||
g.etcetera = false
|
||||
n0, s0, ok := parseNum(g.prevLine)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("bad etcetera command")
|
||||
}
|
||||
if err := stateDeflateDHHEtcetera(n0, s0, n, s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if g.huffmans[g.whichHuffman] == nil {
|
||||
g.huffmans[g.whichHuffman] = deflateHuffmanTable{}
|
||||
}
|
||||
g.huffmans[g.whichHuffman][n] = s
|
||||
g.prevLine = line
|
||||
return stateDeflateDynamicHuffmanHuffman, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("bad stateDeflateDynamicHuffmanHuffman command: %q", line)
|
||||
}
|
||||
|
||||
// stateDeflateDHHEtcetera expands the "etcetera" line in:
|
||||
//
|
||||
// 0 0000
|
||||
// 1 0001
|
||||
// etcetera
|
||||
// 7 0111
|
||||
//
|
||||
// to produce the implicit lines:
|
||||
//
|
||||
// 0 0000
|
||||
// 1 0001
|
||||
// 2 0010
|
||||
// 3 0011
|
||||
// 4 0100
|
||||
// 5 0101
|
||||
// 6 0110
|
||||
// 7 0111
|
||||
func stateDeflateDHHEtcetera(n0 uint32, s0 string, n1 uint32, s1 string) error {
|
||||
b := []byte(s0)
|
||||
if !incrementBitstring(b) {
|
||||
return fmt.Errorf("etcetera: could not increment bitstring")
|
||||
}
|
||||
for n := n0 + 1; n < n1; n++ {
|
||||
line := fmt.Sprintf("%d %s", n, b)
|
||||
if _, err := stateDeflateDynamicHuffmanHuffman(line); err != nil {
|
||||
return err
|
||||
}
|
||||
if !incrementBitstring(b) {
|
||||
return fmt.Errorf("etcetera: could not increment bitstring")
|
||||
}
|
||||
}
|
||||
if string(b) != s1 {
|
||||
return fmt.Errorf("etcetera: final bitstring: got %q, want %q", b, s1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func incrementBitstring(b []byte) (ok bool) {
|
||||
for i := len(b) - 1; i >= 0; i-- {
|
||||
switch b[i] {
|
||||
case '0':
|
||||
b[i] = '1'
|
||||
return true
|
||||
case '1':
|
||||
b[i] = '0'
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type deflateBitStream struct {
|
||||
bits uint32
|
||||
nBits uint32 // Always within [0, 7].
|
||||
|
||||
@@ -0,0 +1,561 @@
|
||||
// Copyright 2019 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build ignore
|
||||
|
||||
package main
|
||||
|
||||
// print-deflate-huff-table-size.go prints the worst-case number of elements in
|
||||
// the std/deflate decoder's huffs tables.
|
||||
//
|
||||
// Usage: go run print-deflate-huff-table-size.go
|
||||
//
|
||||
// This program might take tens of seconds to run.
|
||||
//
|
||||
// It is algorithmically similar to examples/enough.c in the zlib C library.
|
||||
// Much of the initial commentary in that file is also relevant for this
|
||||
// program. Not all of it applies, though. For example, the Wuffs decoder
|
||||
// implementation's primary (root) table length is fixed at compile time. It
|
||||
// does not grow that at run-time, even if the shortest Huffman code is longer
|
||||
// than that primary length.
|
||||
//
|
||||
// This program should print:
|
||||
//
|
||||
// ----------------
|
||||
// -------- Lit/Len (up to 286 symbols, 149142 combos) @1 @2 @3 @4 @5 @6 @7 @8 @9 @10 @11 @12 @13 @14 @15
|
||||
// primLen 3: 4376 entries = 8 prim + 4368 seco 1 0 0; 1 1 1 17 1 1 257 1 1 1 1 2
|
||||
// primLen 4: 2338 entries = 16 prim + 2322 seco 0 0 0 0; 3 1 1 177 97 1 1 1 1 1 2
|
||||
// primLen 5: 1330 entries = 32 prim + 1298 seco 1 0 0 0 0; 3 1 1 177 97 1 1 1 1 2
|
||||
// primLen 6: 852 entries = 64 prim + 788 seco 0 0 0 0 0 0; 1 229 49 1 1 1 1 1 2
|
||||
// primLen 7: 660 entries = 128 prim + 532 seco 1 0 0 0 0 0 0; 1 229 49 1 1 1 1 2
|
||||
// primLen 8: 660 entries = 256 prim + 404 seco 1 1 0 0 0 0 0 0; 1 229 49 1 1 1 2
|
||||
// primLen 9: 852 entries = 512 prim + 340 seco 1 1 1 0 0 0 0 0 0; 1 229 49 1 1 2
|
||||
// primLen 10: 1332 entries = 1024 prim + 308 seco 1 1 1 1 0 0 0 0 0 0; 1 229 49 1 2
|
||||
// primLen 11: 2340 entries = 2048 prim + 292 seco 1 1 1 1 1 0 0 0 0 0 0; 1 229 49 2
|
||||
// primLen 12: 4380 entries = 4096 prim + 284 seco 1 1 1 1 1 1 0 0 0 0 0 0; 1 229 50
|
||||
// primLen 13: 8472 entries = 8192 prim + 280 seco 1 1 1 1 1 1 0 1 0 0 0 0 0;105 174
|
||||
// primLen 14: 16658 entries = 16384 prim + 274 seco 1 1 1 1 1 1 0 1 1 1 0 1 1 1;274
|
||||
// primLen 15: 32768 entries = 32768 prim + 0 seco 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
//
|
||||
// -------- Distance (up to 30 symbols, 1666 combos) @1 @2 @3 @4 @5 @6 @7 @8 @9 @10 @11 @12 @13 @14 @15
|
||||
// primLen 3: 4120 entries = 8 prim + 4112 seco 1 0 0; 1 9 9 1 1 1 1 1 1 1 1 2
|
||||
// primLen 4: 2080 entries = 16 prim + 2064 seco 1 1 0 0; 1 9 9 1 1 1 1 1 1 1 2
|
||||
// primLen 5: 1072 entries = 32 prim + 1040 seco 1 1 1 0 0; 1 9 9 1 1 1 1 1 1 2
|
||||
// primLen 6: 592 entries = 64 prim + 528 seco 1 1 1 1 0 0; 1 9 9 1 1 1 1 1 2
|
||||
// primLen 7: 400 entries = 128 prim + 272 seco 1 1 1 1 1 0 0; 1 9 9 1 1 1 1 2
|
||||
// primLen 8: 400 entries = 256 prim + 144 seco 1 1 1 1 1 1 0 0; 1 9 9 1 1 1 2
|
||||
// primLen 9: 592 entries = 512 prim + 80 seco 1 1 1 1 1 1 1 0 0; 1 9 9 1 1 2
|
||||
// primLen 10: 1072 entries = 1024 prim + 48 seco 1 1 1 1 1 1 1 1 0 0; 1 9 9 1 2
|
||||
// primLen 11: 2080 entries = 2048 prim + 32 seco 1 1 1 1 1 1 1 1 1 0 0; 1 9 9 2
|
||||
// primLen 12: 4120 entries = 4096 prim + 24 seco 1 1 1 1 1 1 1 1 1 1 0 0; 1 9 10
|
||||
// primLen 13: 8212 entries = 8192 prim + 20 seco 1 1 1 1 1 1 1 1 1 1 0 1 0; 5 14
|
||||
// primLen 14: 16400 entries = 16384 prim + 16 seco 1 1 1 1 1 1 1 1 1 1 1 0 0 0; 16
|
||||
// primLen 15: 32768 entries = 32768 prim + 0 seco 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
// ----------------
|
||||
//
|
||||
// Deflate's canonical Huffman codes are up to 15 bits long, but the decoder
|
||||
// can use fewer than (1 << 15) entries in its look-up table, by splitting it
|
||||
// into one primary table (of an at-compile-time fixed length <= 15) and zero
|
||||
// or more secondary tables (of variable lengths, determined at-run-time).
|
||||
//
|
||||
// Symbols whose Huffman codes' length are less than or equal to the primary
|
||||
// table length do not require any secondary tables. Longer codes are grouped
|
||||
// by their primLen length prefix. Each group occupies one entry in the primary
|
||||
// table, which redirects to a secondary table of size (1 << (N - primLen)),
|
||||
// where N is the length of the longest code in that group.
|
||||
//
|
||||
// This program calculates that, for a primary table length of 9 bits, the
|
||||
// worst case Huffman table size is 852 for the Literal/Length table (852 being
|
||||
// a 512 entry primary table and the secondary tables totalling 340 further
|
||||
// entries) and 592 (512 + 80) for the Distance table.
|
||||
//
|
||||
// Copy/pasting one row of that output (and the column headers):
|
||||
//
|
||||
// -------- Lit/Len (up to 286 symbols, 149142 combos) @1 @2 @3 @4 @5 @6 @7 @8 @9 @10 @11 @12 @13 @14 @15
|
||||
// primLen 9: 852 entries = 512 prim + 340 seco 1 1 1 0 0 0 0 0 0; 1 229 49 1 1 2
|
||||
//
|
||||
// The "@1, @2, @3, ..." columns mean that one combination that hits that 852
|
||||
// worst case is having 1 1-length code, 1 2-length code, 1 3-length code, 0
|
||||
// 4-length codes, ..., 1 10-length code, 229 11-length codes, etc. There are a
|
||||
// total of 286 (1 + 1 + 1 + 0 + ... + 49 + 1 + 1 + 2) codes, for 286 symbols.
|
||||
//
|
||||
// See test/data/artificial/deflate-huffman-primlen-9.deflate* for an actual
|
||||
// example of valid Deflate-formatted data that exercises these worst-case code
|
||||
// lengths (for a primary table length of 9 bits).
|
||||
//
|
||||
// Hypothetically, suppose that we used a primLen of 13 for the Distance table.
|
||||
// Here is a code length assignment that produces 20 secondary table entries:
|
||||
//
|
||||
// -------- Distance (up to 30 symbols, 1666 combos) @1 @2 @3 @4 @5 @6 @7 @8 @9 @10 @11 @12 @13 @14 @15
|
||||
// primLen 13: 8212 entries = 8192 prim + 20 seco 1 1 1 1 1 1 1 1 1 1 0 1 0; 5 14
|
||||
//
|
||||
// In detail: call the 14 length codes "e0", "e1", ..., "e4" and the 15 length
|
||||
// codes "f00", "f01", "f02", ..., "f13".
|
||||
//
|
||||
// Recall that secondary tables' lengths are (1 << (N - primLen)), where N is
|
||||
// the longest code for a given primLen length prefix. In this case, (N -
|
||||
// primLen) is (15 - 13) if the secondary table contains a "f??" code,
|
||||
// otherwise it N is (14 - 13). When the secondary table contains a mixture of
|
||||
// lengths, some of the shorter codes' entries are duplicated.
|
||||
//
|
||||
// Also, for canonical Huffman codes, the codes (bitstrings) for 14-length
|
||||
// codes are all assigned (sequentially) before any 15-length codes. There are
|
||||
// therefore 6 secondary tables:
|
||||
//
|
||||
// - length 2: e0, e1.
|
||||
// - length 2: e2, e3.
|
||||
// - length 4: e4, e4, f00, f01.
|
||||
// - length 4: f02, f03, f04, f05.
|
||||
// - length 4: f06, f07, f08, f09.
|
||||
// - length 4: f10, f11, f12, f13.
|
||||
//
|
||||
// The total size of the secondary tables is (2 + 2 + 4 + 4 + 4 + 4) = 20.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
)
|
||||
|
||||
const (
|
||||
maxMaxNSyms = 286 // RFC 1951 lists 286 literal/length codes and 30 distance codes.
|
||||
maxCodeLen = 15 // Each code's encoding is at most 15 bits.
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := main1(); err != nil {
|
||||
os.Stderr.WriteString(err.Error() + "\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func main1() error {
|
||||
do("Lit/Len", 286)
|
||||
do("Distance", 30)
|
||||
return nil
|
||||
}
|
||||
|
||||
func do(name string, maxNSyms uint32) {
|
||||
ftab := &feasibleTable{}
|
||||
ftab.build(maxNSyms)
|
||||
|
||||
// We can do the per-primLen computations concurrently, speeding up the
|
||||
// overall wall time taken.
|
||||
ch := make(chan string)
|
||||
numPending := 0
|
||||
for primLen := uint32(3); primLen <= maxCodeLen; primLen++ {
|
||||
go doPrimLen(ch, maxNSyms, ftab, primLen)
|
||||
numPending++
|
||||
}
|
||||
|
||||
// Gather, sort and print the results for each primLen.
|
||||
results := make([]string, 0, numPending)
|
||||
for ; numPending > 0; numPending-- {
|
||||
results = append(results, <-ch)
|
||||
}
|
||||
sort.Strings(results)
|
||||
fmt.Printf("\n-------- %8s (up to %3d symbols, %6d combos)"+
|
||||
" @1 @2 @3 @4 @5 @6 @7 @8 @9 @10 @11 @12 @13 @14 @15\n",
|
||||
name, maxNSyms, ftab.count())
|
||||
for _, s := range results {
|
||||
fmt.Println(s)
|
||||
}
|
||||
}
|
||||
|
||||
type state struct {
|
||||
ftab *feasibleTable
|
||||
primLen uint32
|
||||
|
||||
currCase struct {
|
||||
nAssignedCodes nAssignedCodes
|
||||
}
|
||||
|
||||
worstCase struct {
|
||||
nAssignedCodes nAssignedCodes
|
||||
nSecondary uint32
|
||||
}
|
||||
|
||||
visited [maxCodeLen + 1][maxMaxNSyms + 1][maxMaxNSyms + 1]bitVector
|
||||
}
|
||||
|
||||
func doPrimLen(ch chan<- string, maxNSyms uint32, ftab *feasibleTable, primLen uint32) {
|
||||
z := &state{
|
||||
ftab: ftab,
|
||||
primLen: primLen,
|
||||
}
|
||||
|
||||
if z.primLen > maxCodeLen {
|
||||
panic("unreachable")
|
||||
|
||||
} else if z.primLen == maxCodeLen {
|
||||
// There's only the primary table and no secondary tables. For an
|
||||
// example that fills that primary table, create a trivial encoding of
|
||||
// two symbols: one symbol's code is "0", the other's is "1".
|
||||
z.worstCase.nAssignedCodes[1] = 2
|
||||
for n := uint32(2); n <= maxCodeLen; n++ {
|
||||
z.worstCase.nAssignedCodes[n] = 0
|
||||
}
|
||||
|
||||
} else {
|
||||
// Brute force search the tree of possible code length assignments.
|
||||
//
|
||||
// nCodes is always even: the number of unassigned Huffman codes at
|
||||
// length L is twice the number at length (L-1). At the start, there
|
||||
// are two unassigned codes of length 1: "0" and "1".
|
||||
//
|
||||
// nSyms starts at nCodes, since (nCodes > nSyms) is infeasible: there
|
||||
// will be unassigned codes.
|
||||
for nCodes := uint32(2); nCodes <= maxNSyms; nCodes += 2 {
|
||||
for nSyms := nCodes; nSyms <= maxNSyms; nSyms++ {
|
||||
if z.ftab[z.primLen+1][nCodes][nSyms] {
|
||||
z.visit(z.primLen+1, nCodes, nSyms, 0, 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
z.worstCase.nAssignedCodes.fillPrimaryValues(z.primLen)
|
||||
}
|
||||
|
||||
// Collect the details: the number of codes assigned to each length in the
|
||||
// worst case.
|
||||
details := ""
|
||||
for n := uint32(1); n <= maxCodeLen; n++ {
|
||||
if n == z.primLen+1 {
|
||||
details += ";"
|
||||
} else {
|
||||
details += " "
|
||||
}
|
||||
details += fmt.Sprintf("%3d", z.worstCase.nAssignedCodes[n])
|
||||
}
|
||||
|
||||
nPrimary := uint32(1) << z.primLen
|
||||
nSecondary := z.worstCase.nSecondary
|
||||
nEntries := nPrimary + nSecondary
|
||||
|
||||
ch <- fmt.Sprintf("primLen %2d: %5d entries = %5d prim + %4d seco%s",
|
||||
z.primLen, nEntries, nPrimary, nSecondary, details)
|
||||
}
|
||||
|
||||
// visit updates z.worstCase, starting with nCodes unassigned Huffman codes of
|
||||
// length codeLen for nSyms symbols. nSecondary is the total size (number of
|
||||
// entries) so far of all of the secondary tables.
|
||||
//
|
||||
// nRemain > 0 iff we have an incomplete secondary table, whose size was
|
||||
// partially accounted for in nSecondary, by (1 << (codeLen - z.primLen)). If
|
||||
// positive, nRemain is the number of unassigned entries in that incomplete
|
||||
// secondary table.
|
||||
//
|
||||
// visit can recursively call itself with longer codeLen values. As it does so,
|
||||
// it temporarily sets z.currCase.nAssignedCodes[codeLen], restoring that to
|
||||
// zero before it returns.
|
||||
func (z *state) visit(codeLen uint32, nCodes uint32, nSyms uint32, nSecondary uint32, nRemain uint32) {
|
||||
if z.alreadyVisited(codeLen, nCodes, nSyms, nSecondary, nRemain) {
|
||||
// No-op.
|
||||
|
||||
} else if nCodes > nSyms {
|
||||
// Infeasible: there will be unassigned codes.
|
||||
|
||||
} else if nCodes == nSyms {
|
||||
// Fill in the incomplete secondary table (if present) and any new
|
||||
// secondary tables (if necessary) to assign the nCodes codes to the
|
||||
// nSyms symbols. At the end, we should have no remaining entries.
|
||||
n := nCodes
|
||||
for n > nRemain {
|
||||
n -= nRemain
|
||||
nRemain = 1 << (codeLen - z.primLen)
|
||||
nSecondary += nRemain
|
||||
}
|
||||
if n != nRemain {
|
||||
panic("inconsistent secondary table size")
|
||||
}
|
||||
|
||||
// Update z.worstCase if we have a new record for nSecondary.
|
||||
if z.worstCase.nSecondary < nSecondary {
|
||||
z.worstCase.nSecondary = nSecondary
|
||||
z.worstCase.nAssignedCodes = z.currCase.nAssignedCodes
|
||||
z.worstCase.nAssignedCodes[codeLen] = nCodes
|
||||
}
|
||||
|
||||
} else { // nCodes < nSyms.
|
||||
codeLen1 := codeLen + 1
|
||||
// Brute force assigning n codes of this codeLen. The other (nCodes -
|
||||
// n) codes will have longer codes.
|
||||
for n := uint32(0); n < nCodes; n++ {
|
||||
nCodes1 := (nCodes - n) * 2
|
||||
nSyms1 := nSyms - n
|
||||
|
||||
if nCodes1 > nSyms1 {
|
||||
// Infeasible: there will be unassigned codes.
|
||||
|
||||
} else if z.ftab[codeLen1][nCodes1][nSyms1] {
|
||||
nSecondary1 := nSecondary
|
||||
nRemain1 := nRemain * 2
|
||||
|
||||
if nRemain1 > 0 {
|
||||
// We have an incomplete secondary table, which has been
|
||||
// partially accounted for: nSecondary has previously been
|
||||
// incremented by X, where X equals (1 << (codeLen -
|
||||
// z.primLen)).
|
||||
//
|
||||
// As we recursively call visit, with codeLen increased by
|
||||
// 1, then we need to double the accounted size of that
|
||||
// secondary table to 2*X. Since X out of that 2*X has
|
||||
// already been added, we need to increment nSecondary1 by
|
||||
// just the difference, which is also X.
|
||||
nSecondary1 += 1 << (codeLen - z.primLen)
|
||||
}
|
||||
|
||||
z.currCase.nAssignedCodes[codeLen] = n
|
||||
z.visit(codeLen1, nCodes1, nSyms1, nSecondary1, nRemain1)
|
||||
z.currCase.nAssignedCodes[codeLen] = 0
|
||||
}
|
||||
|
||||
// Open a new secondary table if necessary: one whose size is (1 <<
|
||||
// (codeLen - z.primLen)).
|
||||
if nRemain == 0 {
|
||||
nRemain = 1 << (codeLen - z.primLen)
|
||||
nSecondary += nRemain
|
||||
}
|
||||
|
||||
// Assign one of the incomplete secondary table's entries.
|
||||
nRemain--
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// alreadyVisited returns whether z has already visited the (nSecondary,
|
||||
// nRemain) pair. As a side effect, it also marks that pair as visited.
|
||||
func (z *state) alreadyVisited(codeLen uint32, nCodes uint32, nSyms uint32, nSecondary uint32, nRemain uint32) bool {
|
||||
// Use the zig re-numbering to minimize the memory requirements for the
|
||||
// visited slice. There's an additional 8-fold memory reduction by using 1
|
||||
// bit per element, not 1 bool (1 byte) per element.
|
||||
i := zig(nSecondary/8, nRemain)
|
||||
mask := uint8(1) << (nSecondary & 7)
|
||||
|
||||
visited := z.visited[codeLen][nCodes][nSyms]
|
||||
if uint64(i) < uint64(len(visited)) {
|
||||
x := visited[i]
|
||||
visited[i] = x | mask
|
||||
return x&mask != 0
|
||||
}
|
||||
|
||||
oldDone := visited
|
||||
visited = make(bitVector, i+1)
|
||||
copy(visited, oldDone)
|
||||
visited[i] = mask
|
||||
|
||||
z.visited[codeLen][nCodes][nSyms] = visited
|
||||
return false
|
||||
}
|
||||
|
||||
type bitVector []uint8
|
||||
|
||||
// zig packs a pair of non-negative integers (i, j) into a single unique
|
||||
// non-negative integer, in a zig-zagging pattern:
|
||||
//
|
||||
// i=0 i=1 i=2 i=3 i=4 etc
|
||||
// j=0: 0 1 3 6 10 ...
|
||||
// j=1: 2 4 7 11 16 ...
|
||||
// j=2: 5 8 12 17 23 ...
|
||||
// j=3: 9 13 18 24 31 ...
|
||||
// j=4: 14 19 25 32 40 ...
|
||||
// etc: ... ... ... ... ... ...
|
||||
//
|
||||
// The closed from expression relies on the fact that the sum (0 + 1 + ... + n)
|
||||
// equals ((n * (n + 1)) / 2).
|
||||
//
|
||||
// This lets us minimize the memory requirements of a triangular array: one for
|
||||
// which a[i][j] is only accessed when ((i+j) < someBound).
|
||||
func zig(i uint32, j uint32) uint32 {
|
||||
if (i > 0x4000) || (j > 0x4000) {
|
||||
panic("overflow")
|
||||
}
|
||||
n := i + j
|
||||
return ((n * (n + 1)) / 2) + j
|
||||
}
|
||||
|
||||
// nAssignedCodes[i] holds the number of codes of length i.
|
||||
type nAssignedCodes [maxCodeLen + 1]uint32
|
||||
|
||||
// fillPrimaryValues fills in feasible a[n] values for n <= primLen. It assumes
|
||||
// that the caller has already filled in a[n] for n > primLen.
|
||||
func (a *nAssignedCodes) fillPrimaryValues(primLen uint32) {
|
||||
// Looping backwards from the maxCodeLen, figure out how many unassigned
|
||||
// primLen length codes there were.
|
||||
//
|
||||
// For example, if there were 10 assigned 15 length codes, then there must
|
||||
// have been (10 / 2 = 5) unassigned 14 length codes. If there were also 7
|
||||
// 14 length codes, then there must have been ((5 + 7) / 2 = 6) unassigned
|
||||
// 13 length codes. Etcetera.
|
||||
nUnassignedPrimLen := uint32(0)
|
||||
for n := uint32(maxCodeLen); n > primLen; n-- {
|
||||
nUnassignedPrimLen += a[n]
|
||||
if nUnassignedPrimLen%2 != 0 {
|
||||
panic("cannot undo Huffman code split")
|
||||
}
|
||||
nUnassignedPrimLen /= 2
|
||||
}
|
||||
if nUnassignedPrimLen < 1 {
|
||||
panic("not enough unassigned primLen length codes")
|
||||
}
|
||||
|
||||
// Pick a combination of assigning shorter length codes (i.e. setting a[n]
|
||||
// values) to reach that nUnassignedPrimLen target. There are many possible
|
||||
// ways to do so, but only one unique way with the additional constraint
|
||||
// that each value is either 0 or 1: unique because every positive number
|
||||
// has a unique binary representation.
|
||||
//
|
||||
// For example, suppose we need to have 12 unassigned primLen length codes.
|
||||
// With that additional constraint, the way to do this is:
|
||||
//
|
||||
// - 12 unassigned code of length primLen-0.
|
||||
// - 6 unassigned code of length primLen-1.
|
||||
// - 3 unassigned code of length primLen-2.
|
||||
// - 2 unassigned code of length primLen-3.
|
||||
// - 1 unassigned code of length primLen-4.
|
||||
// - 1 unassigned code of length primLen-5.
|
||||
// - 1 unassigned code of length primLen-6.
|
||||
// - etc
|
||||
//
|
||||
// Each left hand column value is either ((2*b)-0) or ((2*b)-1), where b is
|
||||
// the the number below. Subtracting 0 or 1 means that we assign 0 or 1
|
||||
// codes of that row's length:
|
||||
//
|
||||
// - 0 assigned code of length primLen-0.
|
||||
// - 0 assigned code of length primLen-1.
|
||||
// - 1 assigned code of length primLen-2.
|
||||
// - 0 assigned code of length primLen-3.
|
||||
// - 1 assigned code of length primLen-4.
|
||||
// - 1 assigned code of length primLen-5.
|
||||
// - 1 assigned code of length primLen-6.
|
||||
// - etc
|
||||
//
|
||||
// Reading upwards, this is the binary string "...1110100", which is the
|
||||
// two's complement representation of the decimal number -12: the negative
|
||||
// of nUnassignedPrimLen.
|
||||
bits := -int32(nUnassignedPrimLen)
|
||||
for n := primLen; n >= 1; n-- {
|
||||
a[n] = uint32(bits & 1)
|
||||
bits >>= 1
|
||||
}
|
||||
if bits != -1 {
|
||||
panic("did not finish with one unassigned zero-length code")
|
||||
}
|
||||
|
||||
// As a sanity check of the "negative two's complement" theory, loop
|
||||
// forwards to calculate the number of unassigned primLen length codes,
|
||||
// which should match nUnassignedPrimLen.
|
||||
nUnassigned := uint32(1)
|
||||
for n := uint32(1); n <= primLen; n++ {
|
||||
nUnassigned *= 2
|
||||
if nUnassigned <= a[n] {
|
||||
panic("not enough unassigned codes")
|
||||
}
|
||||
nUnassigned -= a[n]
|
||||
}
|
||||
if nUnassigned != nUnassignedPrimLen {
|
||||
panic("inconsistent unassigned codes")
|
||||
}
|
||||
}
|
||||
|
||||
// feasibleTable[codeLen][nCodes][nSyms] is whether it is feasible to assign
|
||||
// Huffman codes (of length at least codeLen) to nSyms symbols, given nCodes
|
||||
// unassigned codes of length codeLen. Each unassigned code of length L can be
|
||||
// split into 2 codes of length L+1, 4 codes of length L+2, etc, up to a
|
||||
// maximum code length of maxCodeLen. A feasible assignment ends with zero
|
||||
// unassigned codes, no more and no less.
|
||||
type feasibleTable [maxCodeLen + 1][maxMaxNSyms + 1][maxMaxNSyms + 1]bool
|
||||
|
||||
func (f *feasibleTable) count() (ret uint64) {
|
||||
for i := range *f {
|
||||
for j := range f[i] {
|
||||
for k := range f[i][j] {
|
||||
if f[i][j][k] {
|
||||
ret++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func (f *feasibleTable) build(maxNSyms uint32) {
|
||||
for nSyms := uint32(2); nSyms <= maxNSyms; nSyms++ {
|
||||
if f.buildMore(1, 2, nSyms) != bmResultOK {
|
||||
panic("infeasible [codeLen][nCodes][nSyms] combination")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
bmResultOK = 0
|
||||
bmResultUseFewerCodes = 1
|
||||
bmResultUseMoreCodes = 2
|
||||
bmResultUnsatisfiable = 3
|
||||
)
|
||||
|
||||
func (f *feasibleTable) buildMore(codeLen uint32, nCodes uint32, nSyms uint32) uint32 {
|
||||
if nCodes == nSyms {
|
||||
if nCodes%2 == 1 {
|
||||
panic("odd nCodes declared feasible")
|
||||
}
|
||||
// This is trivial: assign every symbol a code of length codeLen.
|
||||
f[codeLen][nCodes][nSyms] = true
|
||||
return bmResultOK
|
||||
} else if nCodes > nSyms {
|
||||
// Infeasible: there will be unassigned codes.
|
||||
return bmResultUseMoreCodes
|
||||
}
|
||||
// From here onwards, we have (nCodes < nSyms).
|
||||
|
||||
if (nCodes << (maxCodeLen - codeLen)) < nSyms {
|
||||
// Infeasible: there will be unassigned symbols, even if we extend the
|
||||
// codes out to maxCodeLen.
|
||||
return bmResultUseFewerCodes
|
||||
}
|
||||
|
||||
// If we've already visited this [codeLen][nCodes][nSyms] combination,
|
||||
// there's no need to re-do the computation.
|
||||
if f[codeLen][nCodes][nSyms] {
|
||||
return bmResultOK
|
||||
}
|
||||
|
||||
// Try assigning n out of the nCodes codes 1-to-1 to a symbol, remembering
|
||||
// that we have checked above that (nCodes < nSyms). The remaining (nCodes
|
||||
// - n) codes are lengthened by 1, doubling the number of them, and try to
|
||||
// assign those longer codes to the remaining (nSyms - n) symbols.
|
||||
ok := false
|
||||
codeLen1 := codeLen + 1
|
||||
for n := uint32(0); n < nCodes; n++ {
|
||||
nCodes1 := (nCodes - n) * 2
|
||||
nSyms1 := nSyms - n
|
||||
|
||||
if x := f.buildMore(codeLen1, nCodes1, nSyms1); x == bmResultOK {
|
||||
ok = true
|
||||
} else if x == bmResultUseFewerCodes {
|
||||
// This value of n didn't succeed, but also any larger n also won't
|
||||
// succeed, so we can break the loop early.
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !ok {
|
||||
return bmResultUnsatisfiable
|
||||
}
|
||||
if nCodes%2 == 1 {
|
||||
panic("odd nCodes declared feasible")
|
||||
}
|
||||
f[codeLen][nCodes][nSyms] = true
|
||||
return bmResultOK
|
||||
}
|
||||
@@ -57,8 +57,11 @@ pri const dcode_magic_numbers array[32] base.u32 = [
|
||||
0x401000B0, 0x401800B0, 0x402000C0, 0x403000C0, 0x404000D0, 0x406000D0, 0x08000000, 0x08000000,
|
||||
]
|
||||
|
||||
// TODO: replace the magic "big enough" 1024 with something more principled,
|
||||
// perhaps discovered via an exhaustive search.
|
||||
// huffs_table_size is the smallest power of 2 that is greater than or equal to
|
||||
// the worst-case size of the Huffman tables. See
|
||||
// script/print-deflate-huff-table-size.go which calculates that, for a 9-bit
|
||||
// primary table, the worst-case size is 852 for the Lit/Len table and 592 for
|
||||
// the Distance table.
|
||||
pri const huffs_table_size base.u32 = 1024
|
||||
pri const huffs_table_mask base.u32 = 1023
|
||||
|
||||
|
||||
@@ -118,6 +118,15 @@ golden_test deflate_deflate_distance_code_31_gt = {
|
||||
"deflate-distance-code-31.deflate",
|
||||
};
|
||||
|
||||
golden_test deflate_deflate_huffman_primlen_9_gt = {
|
||||
.want_filename =
|
||||
"test/data/artificial/"
|
||||
"deflate-huffman-primlen-9.deflate.decompressed",
|
||||
.src_filename =
|
||||
"test/data/artificial/"
|
||||
"deflate-huffman-primlen-9.deflate",
|
||||
};
|
||||
|
||||
golden_test deflate_midsummer_gt = {
|
||||
.want_filename = "test/data/midsummer.txt", //
|
||||
.src_filename = "test/data/midsummer.txt.gz", //
|
||||
@@ -217,6 +226,63 @@ const char* test_wuffs_deflate_decode_deflate_distance_code_31() {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* test_wuffs_deflate_decode_deflate_huffman_primlen_9() {
|
||||
CHECK_FOCUS(__func__);
|
||||
|
||||
// First, treat this like any other compare-to-golden test.
|
||||
const char* status = do_test_io_buffers(wuffs_deflate_decode,
|
||||
&deflate_deflate_huffman_primlen_9_gt,
|
||||
UINT64_MAX, UINT64_MAX);
|
||||
if (status) {
|
||||
return status;
|
||||
}
|
||||
|
||||
// Second, check that the decoder's huffman table sizes match those predicted
|
||||
// by the script/print-deflate-huff-table-size.go program.
|
||||
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
|
||||
.data = global_src_slice,
|
||||
});
|
||||
wuffs_base__io_buffer got = ((wuffs_base__io_buffer){
|
||||
.data = global_got_slice,
|
||||
});
|
||||
|
||||
golden_test* gt = &deflate_deflate_huffman_primlen_9_gt;
|
||||
status = read_file(&src, gt->src_filename);
|
||||
if (status) {
|
||||
return status;
|
||||
}
|
||||
|
||||
wuffs_deflate__decoder dec;
|
||||
status = wuffs_deflate__decoder__initialize(
|
||||
&dec, sizeof dec, WUFFS_VERSION, WUFFS_INITIALIZE__DEFAULT_OPTIONS);
|
||||
if (status) {
|
||||
RETURN_FAIL("initialize: \"%s\"", status);
|
||||
}
|
||||
status = wuffs_deflate__decoder__decode_io_writer(&dec, &got, &src,
|
||||
global_work_slice);
|
||||
if (status) {
|
||||
RETURN_FAIL("decode_io_writer: \"%s\"", status);
|
||||
}
|
||||
|
||||
int i;
|
||||
for (i = 0; i < 2; i++) {
|
||||
// Find the first unused (i.e. zero) entry in the i'th huffs table.
|
||||
int got = wuffs_deflate__huffs_table_size;
|
||||
while ((got > 0) && (dec.private_data.f_huffs[i][got - 1] == 0)) {
|
||||
got--;
|
||||
}
|
||||
|
||||
// See script/print-deflate-huff-table-size.go with primLen = 9 for how
|
||||
// these expected values are derived.
|
||||
int want = (i == 0) ? 852 : 592;
|
||||
if (got != want) {
|
||||
RETURN_FAIL("i=%d: got %d, want %d", i, got, want);
|
||||
}
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* test_wuffs_deflate_decode_midsummer() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_test_io_buffers(wuffs_deflate_decode, &deflate_midsummer_gt,
|
||||
@@ -682,6 +748,13 @@ const char* test_mimic_deflate_decode_deflate_distance_code_31() {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* test_mimic_deflate_decode_deflate_huffman_primlen_9() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_test_io_buffers(mimic_deflate_decode,
|
||||
&deflate_deflate_huffman_primlen_9_gt, UINT64_MAX,
|
||||
UINT64_MAX);
|
||||
}
|
||||
|
||||
const char* test_mimic_deflate_decode_midsummer() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_test_io_buffers(mimic_deflate_decode, &deflate_midsummer_gt,
|
||||
@@ -803,6 +876,7 @@ proc tests[] = {
|
||||
test_wuffs_deflate_decode_deflate_degenerate_huffman_unused, //
|
||||
test_wuffs_deflate_decode_deflate_distance_32768, //
|
||||
test_wuffs_deflate_decode_deflate_distance_code_31, //
|
||||
test_wuffs_deflate_decode_deflate_huffman_primlen_9, //
|
||||
test_wuffs_deflate_decode_midsummer, //
|
||||
test_wuffs_deflate_decode_pi_just_one_read, //
|
||||
test_wuffs_deflate_decode_pi_many_big_reads, //
|
||||
@@ -822,6 +896,7 @@ proc tests[] = {
|
||||
test_mimic_deflate_decode_deflate_degenerate_huffman_unused, //
|
||||
test_mimic_deflate_decode_deflate_distance_32768, //
|
||||
test_mimic_deflate_decode_deflate_distance_code_31, //
|
||||
test_mimic_deflate_decode_deflate_huffman_primlen_9, //
|
||||
test_mimic_deflate_decode_midsummer, //
|
||||
test_mimic_deflate_decode_pi_just_one_read, //
|
||||
test_mimic_deflate_decode_pi_many_big_reads, //
|
||||
|
||||
@@ -3,10 +3,6 @@
|
||||
make deflate
|
||||
|
||||
blockDynamicHuffman (final) {
|
||||
numLCodes 257
|
||||
numDCodes 1
|
||||
numCLCodeLengths 18
|
||||
|
||||
huffman CodeLength {
|
||||
1 00
|
||||
2 01
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
逹m�$I�$I#quwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww�面面面面面面面面面面面爺=��Wwwww鰐面勿黠璧嫌萱
|
||||
@@ -0,0 +1,112 @@
|
||||
Running deflate-huffman-primlen-9.deflate through script/print-bits.go and
|
||||
adding commentary:
|
||||
|
||||
offset xoffset ASCII hex binary
|
||||
000000 0x0000 . 0xED 0b_...._.101 Dynamic Huffman block, final
|
||||
000000 0x0000 . 0xED 0b_1110_1... NumLCodes: 286
|
||||
000001 0x0001 . 0xFD 0b_...1_1101 NumDCodes: 30
|
||||
000001 0x0001 . 0xFD 0b_111._.... NumCLCodeLengths: 19
|
||||
000002 0x0002 m 0x6D 0b_...._...1
|
||||
|
||||
Decode the H-CL Huffman table (NumCLCodeLengths = 18). Recall the peculiar
|
||||
code_order: 16, 17, 18, 0, 8, ..., 2, 14, 1, 15:
|
||||
|
||||
000002 0x0002 m 0x6D 0b_0110_110. CLCodeLengths: 19 x 3 bits
|
||||
000003 0x0003 . 0x93 0b_1001_0011 CLCLs[ 0] is 4
|
||||
000004 0x0004 $ 0x24 0b_0010_0100 CLCLs[...] is 4
|
||||
000005 0x0005 I 0x49 0b_0100_1001 CLCLs[ 14] is 4
|
||||
000006 0x0006 . 0x92 0b_1001_0010 CLCLs[ 15] is 6
|
||||
000007 0x0007 $ 0x24 0b_0010_0100 CLCLs[ 16] is 6
|
||||
000008 0x0008 I 0x49 0b_0100_1001 CLCLs[ 17] is 6
|
||||
000009 0x0009 # 0x23 0b_...._..11 CLCLs[ 18] is 6
|
||||
|
||||
The H-CL Huffman table is:
|
||||
"0000" -> CodeLength=0
|
||||
"0001" -> CodeLength=1
|
||||
"0010" -> CodeLength=2
|
||||
etc
|
||||
"1110" -> CodeLength=14
|
||||
"111100" -> CodeLength=15
|
||||
"111101" -> CodeLength=16 which means run-length encoding
|
||||
"111110" -> CodeLength=17 which means a block of ( 3 + 3_extra_bits) zeroes
|
||||
"111111" -> CodeLength=18 which means a block of (11 + 7_extra_bits) zeroes
|
||||
|
||||
Decode the H-L Huffman table (NumLCodes = 286):
|
||||
|
||||
000009 0x0009 # 0x23 0b_..10_00.. "0001" is CL=1
|
||||
000009 0x0009 # 0x23 0b_00.._.... "0010" is CL=2
|
||||
000010 0x000A q 0x71 0b_...._..01
|
||||
000010 0x000A q 0x71 0b_..11_00.. "0011" is CL=3
|
||||
000010 0x000A q 0x71 0b_01.._....
|
||||
000011 0x000B u 0x75 0b_...._..01 "1010" is CL=10
|
||||
000011 0x000B u 0x75 0b_..11_01.. "1011" is CL=11
|
||||
000011 0x000B u 0x75 0b_01.._.... "1011" is CL=11
|
||||
000012 0x000C w 0x77 0b_...._..11
|
||||
000012 0x000C w 0x77 0b_..11_01.. "1011" is CL=11
|
||||
etc
|
||||
000149 0x0095 . 0xCC 0b_..00_11.. "1100" is CL=12
|
||||
000149 0x0095 . 0xCC 0b_11.._.... "1100" is CL=12
|
||||
000150 0x0096 . 0xEC 0b_...._..00
|
||||
000150 0x0096 . 0xEC 0b_..10_11.. "1101" is CL=13
|
||||
000150 0x0096 . 0xEC 0b_11.._.... "1110" is CL=14
|
||||
000151 0x0097 = 0x3D 0b_...._..01
|
||||
000151 0x0097 = 0x3D 0b_0011_11.. "111100" is CL=15
|
||||
000152 0x0098 . 0x0F 0b_..00_1111 "111100" is CL=15
|
||||
|
||||
The H-L Huffman table is:
|
||||
"0" -> '\x00'
|
||||
"10" -> '\x01'
|
||||
"110" -> '\x02'
|
||||
"1110000000" -> '\x03'
|
||||
"11100000010" -> '\x04'
|
||||
etc
|
||||
"11101011111" -> '\x61' aka 'a' aka 97
|
||||
"11101100000" -> '\x62' aka 'b' aka 98
|
||||
etc
|
||||
"111111100101" -> Lit/Len code 256 (EOB)
|
||||
"111111100110" -> Lit/Len code 257 (Length = 3)
|
||||
etc
|
||||
"111111111111111" -> Lit/Len code 285 (Length = 258)
|
||||
|
||||
Decode the H-D Huffman table (NumDCodes = 30):
|
||||
|
||||
000152 0x0098 . 0x0F 0b_00.._.... "0001" is CL=1
|
||||
000153 0x0099 . 0x12 0b_...._..10
|
||||
000153 0x0099 . 0x12 0b_..01_00.. "0010" is CL=2
|
||||
000153 0x0099 . 0x12 0b_00.._....
|
||||
000154 0x009A . 0x8B 0b_...._..11 "0011" is CL=3
|
||||
000154 0x009A . 0x8B 0b_..00_10.. "0100" is CL=4
|
||||
etc
|
||||
000165 0x00A5 . 0xCC 0b_..00_11.. "1100" is CL=12
|
||||
000165 0x00A5 . 0xCC 0b_11.._.... "1101" is CL=13
|
||||
000166 0x00A6 . 0xDE 0b_...._..10
|
||||
000166 0x00A6 . 0xDE 0b_..01_11.. "1110" is CL=14
|
||||
000166 0x00A6 . 0xDE 0b_11.._.... "111100" is CL=15
|
||||
000167 0x00A7 . 0xF3 0b_...._0011
|
||||
000167 0x00A7 . 0xF3 0b_1111_.... "111100" is CL=15
|
||||
000168 0x00A8 . 0xDC 0b_...._..00
|
||||
|
||||
The H-D Huffman table is:
|
||||
"0" -> Distance code 0 (Distance = 1)
|
||||
"10" -> Distance code 1 (Distance = 2)
|
||||
"110" -> Distance code 2 (Distance = 3)
|
||||
etc
|
||||
"11111110010" -> Distance code 8 (Distance = 17 + 3_extra_bits)
|
||||
etc
|
||||
"111111111111111" -> Distance code 29 (Distance = 24577 + 13_extra_bits)
|
||||
|
||||
Apply H-L and H-D. The length=3, distance=2 back-reference decodes to "ana", so
|
||||
the overall decoding is "banana".
|
||||
|
||||
000168 0x00A8 . 0xDC 0b_1101_11.. lcode: 98 literal 'b'
|
||||
000169 0x00A9 . 0xE0 0b_...0_0000
|
||||
000169 0x00A9 . 0xE0 0b_111._.... lcode: 97 literal 'a'
|
||||
000170 0x00AA . 0xFA 0b_1111_1010
|
||||
000171 0x00AB . 0xB7 0b_1011_0111 lcode: 110 literal 'n'
|
||||
000172 0x00AC . 0xF9 0b_...._.001
|
||||
000172 0x00AC . 0xF9 0b_1111_1... lcode: 256 length=3
|
||||
000173 0x00AD . 0xB3 0b_.011_0011
|
||||
000173 0x00AD . 0xB3 0b_1..._.... dcode: 1 distance=2
|
||||
000174 0x00AE . 0xFE 0b_...._...0
|
||||
000174 0x00AE . 0xFE 0b_1111_111. lcode: 256 end of block
|
||||
000175 0x00AF . 0x14 0b_...1_0100
|
||||
@@ -0,0 +1 @@
|
||||
banana
|
||||
@@ -0,0 +1,93 @@
|
||||
# Feed this file to script/make-artificial.go
|
||||
|
||||
make deflate
|
||||
|
||||
blockDynamicHuffman (final) {
|
||||
huffman CodeLength {
|
||||
# 15 codes of length 4.
|
||||
0 0000
|
||||
1 0001
|
||||
etcetera
|
||||
13 1101
|
||||
14 1110
|
||||
|
||||
# 4 codes of length 6.
|
||||
15 111100
|
||||
16 111101
|
||||
17 111110
|
||||
18 111111
|
||||
}
|
||||
|
||||
# script/print-deflate-huff-table-size.go prints:
|
||||
# primLen 9: 852 entries = 512 prim + 340 seco
|
||||
# @1 @2 @3 @4 @5 @6 @7 @8 @9 @10 @11 @12 @13 @14 @15
|
||||
# 1 1 1 0 0 0 0 0 0; 1 229 49 1 1 2
|
||||
huffman Length/Literal {
|
||||
0 0
|
||||
1 10
|
||||
2 110
|
||||
3 1110000000
|
||||
|
||||
# 229 codes of length 11.
|
||||
4 11100000010
|
||||
5 11100000011
|
||||
etcetera
|
||||
231 11111100101
|
||||
232 11111100110
|
||||
|
||||
# 49 codes of length 12.
|
||||
233 111111001110
|
||||
234 111111001111
|
||||
etcetera
|
||||
256 111111100101
|
||||
257 111111100110
|
||||
etcetera
|
||||
280 111111111101
|
||||
281 111111111110
|
||||
|
||||
# 1+1+2 = 4 codes of length 13+.
|
||||
282 1111111111110
|
||||
283 11111111111110
|
||||
284 111111111111110
|
||||
285 111111111111111
|
||||
}
|
||||
|
||||
# script/print-deflate-huff-table-size.go prints:
|
||||
# primLen 9: 592 entries = 512 prim + 80 seco
|
||||
# @1 @2 @3 @4 @5 @6 @7 @8 @9 @10 @11 @12 @13 @14 @15
|
||||
# 1 1 1 1 1 1 1 0 0; 1 9 9 1 1 2
|
||||
huffman Distance {
|
||||
0 0
|
||||
1 10
|
||||
2 110
|
||||
3 1110
|
||||
4 11110
|
||||
5 111110
|
||||
6 1111110
|
||||
7 1111111000
|
||||
|
||||
# 9 codes of length 11.
|
||||
8 11111110010
|
||||
9 11111110011
|
||||
etcetera
|
||||
15 11111111001
|
||||
16 11111111010
|
||||
|
||||
# 9 codes of length 12.
|
||||
17 111111110110
|
||||
18 111111110111
|
||||
etcetera
|
||||
24 111111111101
|
||||
25 111111111110
|
||||
|
||||
# 1+1+2 = 4 codes of length 13+.
|
||||
26 1111111111110
|
||||
27 11111111111110
|
||||
28 111111111111110
|
||||
29 111111111111111
|
||||
}
|
||||
|
||||
literal "ban"
|
||||
len 3 dist 2
|
||||
endOfBlock
|
||||
}
|
||||
Reference in new issue
Block a user