Tighten std/bzip2 huffman_trees array size

This commit is contained in:
Nigel Tao committed 2022-06-10 14:02:17 +10:00
1 parent 5d7b113172
commit 8c33dd503b
9 files changed
+137 -30

No files matched your search

+6 -8
View File
@@ -6828,7 +6828,6 @@ extern const char wuffs_bzip2__error__bad_block_length[];
extern const char wuffs_bzip2__error__bad_checksum[];
extern const char wuffs_bzip2__error__bad_header[];
extern const char wuffs_bzip2__error__bad_number_of_sections[];
extern const char wuffs_bzip2__error__unsupported_huffman_code[];
extern const char wuffs_bzip2__error__unsupported_block_randomization[];
// ---------------- Public Consts
@@ -6966,7 +6965,7 @@ struct wuffs_bzip2__decoder__struct {
uint8_t f_presence[256];
uint8_t f_mtft[256];
uint8_t f_huffman_selectors[32768];
uint16_t f_huffman_trees[6][1024][2];
uint16_t f_huffman_trees[6][257][2];
uint16_t f_huffman_tables[6][256];
uint32_t f_bwt[1048576];
@@ -24866,7 +24865,6 @@ const char wuffs_bzip2__error__bad_block_length[] = "#bzip2: bad block length";
const char wuffs_bzip2__error__bad_checksum[] = "#bzip2: bad checksum";
const char wuffs_bzip2__error__bad_header[] = "#bzip2: bad header";
const char wuffs_bzip2__error__bad_number_of_sections[] = "#bzip2: bad number of sections";
const char wuffs_bzip2__error__unsupported_huffman_code[] = "#bzip2: unsupported Huffman code";
const char wuffs_bzip2__error__unsupported_block_randomization[] = "#bzip2: unsupported block randomization";
const char wuffs_bzip2__error__internal_error_inconsistent_huffman_decoder_state[] = "#bzip2: internal error: inconsistent Huffman decoder state";
@@ -25839,8 +25837,8 @@ wuffs_bzip2__decoder__build_huffman_tree(
} else {
self->private_data.f_huffman_trees[a_which][v_node_index][1] = ((uint16_t)(v_num_nodes));
}
if (v_num_nodes >= 1023) {
return wuffs_base__make_status(wuffs_bzip2__error__unsupported_huffman_code);
if (v_num_nodes >= 257) {
return wuffs_base__make_status(wuffs_bzip2__error__bad_huffman_code_under_subscribed);
}
v_stack_values[v_stack_height] = v_num_nodes;
self->private_data.f_huffman_trees[a_which][v_num_nodes][0] = 0;
@@ -25896,7 +25894,7 @@ wuffs_bzip2__decoder__build_huffman_table(
v_bits = (v_i << 24);
v_n_bits = 0;
v_child = 0;
while ((v_child < 1024) && (v_n_bits < 8)) {
while ((v_child < 257) && (v_n_bits < 8)) {
v_child = self->private_data.f_huffman_trees[a_which][v_child][(v_bits >> 31)];
v_bits <<= 1;
#if defined(__GNUC__)
@@ -26203,7 +26201,7 @@ wuffs_bzip2__decoder__decode_huffman_fast(
v_bits <<= (v_table_entry >> 12);
v_n_bits -= ((uint32_t)((v_table_entry >> 12)));
v_child = (v_table_entry & 2047);
while (v_child < 1024) {
while (v_child < 257) {
v_child = self->private_data.f_huffman_trees[v_which][v_child][(v_bits >> 31)];
v_bits <<= 1;
if (v_n_bits <= 0) {
@@ -26338,7 +26336,7 @@ wuffs_bzip2__decoder__decode_huffman_slow(
v_child = self->private_data.f_huffman_trees[self->private_impl.f_decode_huffman_which][v_node_index][(self->private_impl.f_bits >> 31)];
self->private_impl.f_bits <<= 1;
self->private_impl.f_n_bits -= 1;
if (v_child < 1024) {
if (v_child < 257) {
v_node_index = ((uint32_t)(v_child));
goto label__0__continue;
} else if (v_child < 1280) {
+26 -15
View File
@@ -19,7 +19,6 @@ pub status "#bad block length"
pub status "#bad checksum"
pub status "#bad header"
pub status "#bad number of sections"
pub status "#unsupported Huffman code"
pub status "#unsupported block randomization"
pri status "#internal error: inconsistent Huffman decoder state"
@@ -73,12 +72,22 @@ pub struct decoder? implements base.io_transformer(
// 50-symbol section.
huffman_selectors : array[32768] base.u8,
// Each array[2] base.u16 is a Huffman tree node. The build_huffman_tree
// method comment details what the base.u16 values mean.
// Each array[2] base.u16 is a branch node in the Huffman tree. The
// build_huffman_tree method comment details what the base.u16 values mean.
//
// 1024 is just a guess for the worst case number of nodes needed. TODO: be
// more rigorous about deriving this upper bound.
huffman_trees : array[6] array[1024] array[2] base.u16,
// Each Huffman tree has num_symbols leaf nodes and hence (num_symbols - 1)
// branch nodes, also called internal nodes, as per
// https://stackoverflow.com/questions/69399994/maximum-number-of-nodes-in-a-huffman-tree/69401083#69401083
//
// This can be proven inductively. The base case Huffman code (two symbols)
// has one branch node (the root node). The inductive case, adding one
// symbol to a (complete but possibly non-canonical) Huffman code replaces
// one leaf node with one branch node (and two leaf nodes). The net gain in
// branch nodes is +1.
//
// Since num_symbols' inclusive maximum is 258 (for RUNA, RUNB, 255 × mXYZ,
// EOB), there are at most 257 branch nodes.
huffman_trees : array[6] array[257] array[2] base.u16,
// huffman_tables[which] caches the result of walking huffman_trees[which]
// for each possible 8-bit input. The base.u16 value decomposes as:
@@ -550,9 +559,11 @@ pri func decoder.read_code_lengths?(src: base.io_reader) {
// - "110" 0x402 (m02)
// - "111" 0x7FF (EOB)
//
// The Huffman tree has 5 nodes. Each node has two children, represented by a
// base.u16 value. 1023 or less means that the child is a branch node. 1024 =
// 0x400 or above means a leaf node (with the symbol values as above).
// The Huffman tree has 5 branch nodes (where 5+1 is both the number of leaf
// nodes and the number of symbols). Each branch node has two children,
// represented by a base.u16 value. 256 = 0x100 or less means that the child is
// a branch node. 1024 = 0x400 or above means a leaf node (with the symbol
// values as above).
// - node0 = [0x001 (node1), 0x002 (node2)]
// - node1 = [0x401 (m01) , 0x403 (m03) ]
// - node2 = [0x003 (node3), 0x004 (node4)]
@@ -580,10 +591,10 @@ pri func decoder.read_code_lengths?(src: base.io_reader) {
pri func decoder.build_huffman_tree!(which: base.u32[..= 5]) base.status {
var code_length : base.u32
var symbol_index : base.u32
var num_nodes : base.u32[..= 1023]
var num_nodes : base.u32[..= 257]
var stack_height : base.u32[..= 21]
var stack_values : array[21] base.u32[..= 1023]
var node_index : base.u32[..= 1023]
var stack_values : array[21] base.u32[..= 256]
var node_index : base.u32[..= 256]
var leaf_value : base.u16
// Push a root node.
@@ -627,8 +638,8 @@ pri func decoder.build_huffman_tree!(which: base.u32[..= 5]) base.status {
} else {
this.huffman_trees[args.which][node_index][1] = num_nodes as base.u16
}
if num_nodes >= 1023 {
return "#unsupported Huffman code"
if num_nodes >= 257 {
return "#bad Huffman code (under-subscribed)"
}
stack_values[stack_height] = num_nodes
this.huffman_trees[args.which][num_nodes][0] = 0
@@ -684,7 +695,7 @@ pri func decoder.build_huffman_table!(which: base.u32[..= 5]) {
bits = i << 24
n_bits = 0
child = 0
while (child < 1024) and (n_bits < 8),
while (child < 257) and (n_bits < 8),
inv i < 256,
{
child = this.huffman_trees[args.which][child][bits >> 31]
+1 -1
View File
@@ -59,7 +59,7 @@ pri func decoder.decode_huffman_fast!(src: base.io_reader) base.status {
bits ~mod<<= table_entry >> 12
n_bits -= (table_entry >> 12) as base.u32
child = table_entry & 0x7FF
while child < 1024 {
while child < 257 {
child = this.huffman_trees[which][child][bits >> 31]
bits ~mod<<= 1
if n_bits <= 0 {
+2 -2
View File
@@ -14,7 +14,7 @@
pri func decoder.decode_huffman_slow?(src: base.io_reader) {
var c : base.u8
var node_index : base.u32[..= 1023]
var node_index : base.u32[..= 256]
var child : base.u16
var child_ff : base.u32[..= 255]
@@ -49,7 +49,7 @@ pri func decoder.decode_huffman_slow?(src: base.io_reader) {
this.bits ~mod<<= 1
this.n_bits -= 1
if child < 1024 { // Branch node.
if child < 257 { // Branch node.
node_index = child as base.u32
continue
+13
View File
@@ -79,6 +79,11 @@ golden_test g_bzip2_bad_number_of_sections_gt = {
.src_filename = "test/data/artificial-bzip2/bad-number-of-sections.bz2",
};
golden_test g_bzip2_huffman_258_gt = {
.want_filename = "test/data/abraca.txt",
.src_filename = "test/data/artificial-bzip2/huffman-258.bz2",
};
golden_test g_bzip2_midsummer_gt = {
.want_filename = "test/data/midsummer.txt",
.src_filename = "test/data/midsummer.txt.bz2",
@@ -155,6 +160,13 @@ test_wuffs_bzip2_decode_bad_number_of_sections() {
return NULL;
}
const char* //
test_wuffs_bzip2_decode_huffman_258() {
CHECK_FOCUS(__func__);
return do_test_io_buffers(wuffs_bzip2_decode, &g_bzip2_huffman_258_gt,
UINT64_MAX, UINT64_MAX);
}
const char* //
test_wuffs_bzip2_decode_midsummer() {
CHECK_FOCUS(__func__);
@@ -246,6 +258,7 @@ proc g_tests[] = {
test_wuffs_bzip2_decode_256_bytes,
test_wuffs_bzip2_decode_bad_number_of_sections,
test_wuffs_bzip2_decode_huffman_258,
test_wuffs_bzip2_decode_interface,
test_wuffs_bzip2_decode_midsummer,
test_wuffs_bzip2_decode_pi,
@@ -30,8 +30,9 @@ bits 15 0x0001
bits 1 0x0
# Bitslices (n) ..= (o)
bits 19 0x0CD34
bits 19 0x0CD34
repeat 2 [
bits 19 0x0CD34
]
# Bitslice (p)
bits 3 0x6
@@ -26,8 +26,9 @@ bits 15 0x0001
bits 1 0x0
# Bitslices (n) ..= (o)
bits 19 0x0CD34
bits 19 0x0CD34
repeat 2 [
bits 19 0x0CD34
]
# Bitslice (p)
repeat 50 [
Binary file not shown.
@@ -0,0 +1,83 @@
# Feed this file to script/make-artificial.go
# This modifies abraca.txt.bz2.make-artificial.txt to produce the same
# decompressed output, "abraca", but in a more elaborate way. The Huffman codes
# here have the full 258 symbols instead of just 6.
make bzip2
# Bitslices (a) ..= (g)
bits 16 0x425A
bits 8 0x68
bits 8 0x39
bits 48 0x314159265359
bits 32 0x76A70995
bits 1 0x0
bits 24 0x000001
# Bitslices (h) ..= (j)
bits 16 0xFFFF
repeat 16 [
bits 16 0xFFFF
]
# Bitslices (k) ..= (m)
bits 3 0x2
bits 15 0x0001
bits 1 0x0
# Bitslices (n) ..= (o)
#
# Both Huffman codes have 258 symbols: 254 × 8-bit symbols and then 4 × 9-bit
# symbols.
#
# For small XYZ (where XYZ < 253), the mXYZ symbol's bitstring is (XYZ+1) in
# binary. For example, the m099 symbol's bitstring is "01100100", because the
# binary representation of 0d099+1 = 0d100 = 0x64 = 0b01100100.
#
# RUNA is the 8-bit bitstring "00000000".
# RUNB is the 8-bit bitstring "00000001".
# m001 is the 8-bit bitstring "00000010".
# m002 is the 8-bit bitstring "00000011".
# m003 is the 8-bit bitstring "00000100".
# etc
# m250 is the 8-bit bitstring "11111011".
# m251 is the 8-bit bitstring "11111100".
# m252 is the 8-bit bitstring "11111101".
# m253 is the 9-bit bitstring "111111100".
# m254 is the 9-bit bitstring "111111101".
# m255 is the 9-bit bitstring "111111110".
# EOB is the 9-bit bitstring "111111111".
repeat 2 [
bits 5 0x08
repeat 254 [
bits 1 0x0
]
bits 3 0x4
bits 1 0x0
bits 1 0x0
bits 1 0x0
]
# Bitslice (p)
#
# There are 7 symbols.
# Order = [0x00, 0x01, 0x02, 0x03, ... 0xFF] m099 picks 'c'. 0d099+1 = 0x64.
# Order = ['c', 0x00, 0x01, 0x02, ... 0xFF] m098 picks 'a'. 0d098+1 = 0x63.
# Order = ['a', 'c', 0x00, 0x01, ... 0xFF] m114 picks 'r'. 0d114+1 = 0x73.
# Order = ['r', 'a', 'c', 0x00, ... 0xFF] m001 picks 'a'. 0d001+1 = 0x02.
# Order = ['a', 'r', 'c', 0x00, ... 0xFF] RUNA picks 'a'.
# Order = ['a', 'r', 'c', 0x00, ... 0xFF] m100 picks 'b'. 0d100+1 = 0x65.
# Order = ['b', 'a', 'r', 'c', ... 0xFF] EOB.
bits 8 0x64
bits 8 0x63
bits 8 0x73
bits 8 0x02
bits 8 0x00
bits 8 0x65
bits 9 0x1FF
# Bitslices (q) ..= (s)
bits 48 0x177245385090
bits 32 0x76A70995
bits 1 0x0