Fix deflate decode when the distance is 32768

This commit is contained in:
Nigel Tao committed 2018-03-23 12:49:42 +11:00
1 parent 7de631fb96
commit 4675efdcc5
9 files changed
+115 -76

No files matched your search

+18 -28
View File
@@ -803,11 +803,11 @@ static const uint32_t wuffs_deflate__lcode_magic_numbers[32] = {
};
static const uint32_t wuffs_deflate__dcode_magic_numbers[32] = {
1073742080, 1073742336, 1073742592, 1073742848, 1073743120, 1073743632,
1073744160, 1073745184, 1073746224, 1073748272, 1073750336, 1073754432,
1073758544, 1073766736, 1073774944, 1073791328, 1073807728, 1073840496,
1073873280, 1073938816, 1074004368, 1074135440, 1074266528, 1074528672,
1074790832, 1075315120, 1075839424, 1076888000, 1077936592, 1080033744,
1073741824, 1073742080, 1073742336, 1073742592, 1073742864, 1073743376,
1073743904, 1073744928, 1073745968, 1073748016, 1073750080, 1073754176,
1073758288, 1073766480, 1073774688, 1073791072, 1073807472, 1073840240,
1073873024, 1073938560, 1074004112, 1074135184, 1074266272, 1074528416,
1074790576, 1075314864, 1075839168, 1076887744, 1077936336, 1080033488,
134217728, 134217728,
};
@@ -2234,14 +2234,9 @@ label_0_continue:;
v_bits >>= v_table_entry_n_bits;
v_n_bits -= v_table_entry_n_bits;
}
if (v_distance <= 0) {
status =
WUFFS_DEFLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
v_n_copied = 0;
while (true) {
if (((uint64_t)(v_distance)) >
if (((uint64_t)((v_distance + 1))) >
((uint64_t)(
((wuffs_base__slice_u8){
.ptr = a_dst.private_impl.mark,
@@ -2252,7 +2247,7 @@ label_0_continue:;
.len))) {
v_hlen = 0;
v_hdist = ((uint32_t)((
((uint64_t)(v_distance)) -
((uint64_t)((v_distance + 1))) -
((uint64_t)(
((wuffs_base__slice_u8){
.ptr = a_dst.private_impl.mark,
@@ -2277,14 +2272,14 @@ label_0_continue:;
status = WUFFS_DEFLATE__ERROR_BAD_DISTANCE;
goto exit;
}
v_hdist = ((self->private_impl.f_history_index - v_hdist) & 32767);
v_hdist = (self->private_impl.f_history_index - v_hdist);
while (true) {
v_n_copied = wuffs_base__writer1__copy_from_slice32(
&b_wptr_dst, b_wend_dst,
wuffs_base__slice_u8__subslice_i(
((wuffs_base__slice_u8){.ptr = self->private_impl.f_history,
.len = 32768}),
v_hdist),
v_hdist & 32767),
v_hlen);
if (v_hlen <= v_n_copied) {
goto label_1_break;
@@ -2301,7 +2296,7 @@ label_0_continue:;
if (v_length == 0) {
goto label_0_continue;
}
if (((uint64_t)(v_distance)) >
if (((uint64_t)((v_distance + 1))) >
((uint64_t)(
((wuffs_base__slice_u8){
.ptr = a_dst.private_impl.mark,
@@ -2315,7 +2310,7 @@ label_0_continue:;
}
}
wuffs_base__writer1__copy_from_history32__bco(
&b_wptr_dst, a_dst.private_impl.mark, b_wend_dst, v_distance,
&b_wptr_dst, a_dst.private_impl.mark, b_wend_dst, v_distance + 1,
v_length);
goto label_2_break;
}
@@ -2645,14 +2640,9 @@ static wuffs_deflate__status wuffs_deflate__decoder__decode_huffman_slow(
v_bits >>= v_table_entry_n_bits;
v_n_bits -= v_table_entry_n_bits;
}
if (v_distance <= 0) {
status =
WUFFS_DEFLATE__ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
v_n_copied = 0;
while (true) {
if (((uint64_t)(v_distance)) >
if (((uint64_t)((v_distance + 1))) >
((uint64_t)(
((wuffs_base__slice_u8){
.ptr = a_dst.private_impl.mark,
@@ -2663,7 +2653,7 @@ static wuffs_deflate__status wuffs_deflate__decoder__decode_huffman_slow(
.len))) {
v_hlen = 0;
v_hdist = ((uint32_t)(
(((uint64_t)(v_distance)) -
(((uint64_t)((v_distance + 1))) -
((uint64_t)(((wuffs_base__slice_u8){
.ptr = a_dst.private_impl.mark,
.len = a_dst.private_impl.mark
@@ -2683,14 +2673,14 @@ static wuffs_deflate__status wuffs_deflate__decoder__decode_huffman_slow(
status = WUFFS_DEFLATE__ERROR_BAD_DISTANCE;
goto exit;
}
v_hdist = ((self->private_impl.f_history_index - v_hdist) & 32767);
v_hdist = (self->private_impl.f_history_index - v_hdist);
while (true) {
v_n_copied = wuffs_base__writer1__copy_from_slice32(
&b_wptr_dst, b_wend_dst,
wuffs_base__slice_u8__subslice_i(
((wuffs_base__slice_u8){.ptr = self->private_impl.f_history,
.len = 32768}),
v_hdist),
v_hdist & 32767),
v_hlen);
if (v_hlen <= v_n_copied) {
v_hlen = 0;
@@ -2714,14 +2704,14 @@ static wuffs_deflate__status wuffs_deflate__decoder__decode_huffman_slow(
wuffs_base__slice_u8__subslice_i(
((wuffs_base__slice_u8){
.ptr = self->private_impl.f_history, .len = 32768}),
v_hdist),
v_hdist & 32767),
v_hlen);
if (v_hlen <= v_n_copied) {
v_hlen = 0;
goto label_6_break;
}
v_hlen -= v_n_copied;
v_hdist = ((v_hdist + (v_n_copied & 32767)) & 32767);
v_hdist = (v_hdist + v_n_copied);
status = WUFFS_DEFLATE__SUSPENSION_SHORT_WRITE;
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10);
}
@@ -2732,7 +2722,7 @@ static wuffs_deflate__status wuffs_deflate__decoder__decode_huffman_slow(
}
}
v_n_copied = wuffs_base__writer1__copy_from_history32(
&b_wptr_dst, a_dst.private_impl.mark, b_wend_dst, v_distance,
&b_wptr_dst, a_dst.private_impl.mark, b_wend_dst, v_distance + 1,
v_length);
if (v_length <= v_n_copied) {
v_length = 0;
+24
View File
@@ -26,6 +26,18 @@ var reasons = [...]struct {
r reason
}{
{`"a < b: b > a"`, func(q *checker, n *a.Assert) error {
op, xa, xb := parseBinaryOp(n.Condition())
if op.Key() != t.KeyXBinaryLessThan {
return errFailed
}
// b > a
if err := proveReasonRequirement(q, t.IDXBinaryGreaterThan, xb, xa); err != nil {
return err
}
return nil
}},
{`"a < b: a < c; c < b"`, func(q *checker, n *a.Assert) error {
op, xa, xb := parseBinaryOp(n.Condition())
if op.Key() != t.KeyXBinaryLessThan {
@@ -126,6 +138,18 @@ var reasons = [...]struct {
return nil
}},
{`"a <= b: b >= a"`, func(q *checker, n *a.Assert) error {
op, xa, xb := parseBinaryOp(n.Condition())
if op.Key() != t.KeyXBinaryLessEq {
return errFailed
}
// b >= a
if err := proveReasonRequirement(q, t.IDXBinaryGreaterEq, xb, xa); err != nil {
return err
}
return nil
}},
{`"a <= b: a <= c; c <= b"`, func(q *checker, n *a.Assert) error {
op, xa, xb := parseBinaryOp(n.Condition())
if op.Key() != t.KeyXBinaryLessEq {
+2
View File
@@ -30,12 +30,14 @@ import (
)
var reasons = [...]string{
"a < b: b > a",
"a < b: a < c; c < b",
"a < b: a < c; c == b",
"a < b: a == c; c < b",
"a < b: a < c; c <= b",
"a < b: a <= c; c < b",
"a <= b: b >= a",
"a <= b: a <= c; c <= b",
"a <= b: a <= c; c == b",
"a <= b: a == c; c <= b",
+12 -3
View File
@@ -17,7 +17,14 @@
package main
// print-deflate-magic-numbers.go prints the std/deflate lcode_magic_numbers
// values based on the tables in RFC 1951 secion 3.2.5.
// and dcode_magic_numbers values based on the tables in RFC 1951 secion 3.2.5.
//
// The dcode base numbers are biased by -1 so that (base_number_minus_1 +
// extra_bits) fits in the range [0, 32767]. This makes a bitwise and with
// 0x7FFF a no-op, in terms of computed value, but proves to the compiler that
// the result is within a certain range. Furthermore, proving that (d + 1) > 0
// is trivial, for d of type u32[..something], compared to proving that d > 0,
// which usually requires a runtime check (an if branch).
//
// Usage: go run print-deflate-magic-numbers.go
@@ -35,13 +42,15 @@ func main() {
func main1() error {
for i := 0; i < 2; i++ {
bias := uint32(0)
if i != 0 {
fmt.Println()
bias = 1 // See the "dcode base numbers" comment above.
}
for j := 0; j < 32; j++ {
x := uint32(0x08000000)
if baseNumbers[i][j] != bad {
x = 0x40000000 | (baseNumbers[i][j] << 8) | (extraBits[i][j] << 4)
if bn := baseNumbers[i][j]; bn != bad {
x = 0x40000000 | ((bn - bias) << 8) | (extraBits[i][j] << 4)
}
fmt.Printf("0x%08X,", x)
if j&7 == 7 {
+10 -9
View File
@@ -210,7 +210,7 @@ outer_loop:
goto outer_loop;
}
if ((table_entry >> 30) != 0) {
// Back-reference; length = base number + extra bits.
// Back-reference; length = base_number + extra_bits.
break;
}
if ((table_entry >> 29) != 0) {
@@ -229,7 +229,7 @@ outer_loop:
table_entry = self->private_impl.f_huffs[0][top + (bits & mask)];
}
// length = base number + extra bits.
// length = base_number + extra_bits.
uint32_t length = (table_entry >> 8) & 0xFFFF;
{
uint32_t n = (table_entry >> 4) & 0x0F;
@@ -263,7 +263,7 @@ outer_loop:
bits >>= n;
n_bits -= n;
if ((table_entry >> 30) != 0) {
// Back-reference; distance = base number + extra bits.
// Back-reference; distance = base_number + extra_bits.
break;
}
if ((table_entry >> 24) != 0x10) {
@@ -277,8 +277,9 @@ outer_loop:
table_entry = self->private_impl.f_huffs[1][top + (bits & mask)];
}
// distance = base number + extra bits.
uint32_t distance = (table_entry >> 8) & 0xFFFF;
// dist_minus_1 = base_number_minus_1 + extra_bits + 1.
// distance = dist_minus_1 + 1.
uint32_t dist_minus_1 = (table_entry >> 8) & 0xFFFF;
{
uint32_t n = (table_entry >> 4) & 0x0F;
if (n) {
@@ -291,19 +292,19 @@ outer_loop:
n_bits += 8;
}
#endif
distance += bits & wuffs_base__width_to_mask_table[n];
dist_minus_1 += bits & wuffs_base__width_to_mask_table[n];
bits >>= n;
n_bits -= n;
}
}
// TODO: look at a sliding window, not just output written so far to dst.
if ((ptrdiff_t)(distance) > (pdst - pdst0)) {
if ((ptrdiff_t)(dist_minus_1 + 1) > (pdst - pdst0)) {
status = WUFFS_DEFLATE__ERROR_BAD_ARGUMENT;
goto end;
}
uint8_t* pback = pdst - distance;
uint8_t* pback = pdst - (dist_minus_1 + 1);
#if defined(WUFFS_DEFLATE__HAVE_64_BIT_UNALIGNED_LITTLE_ENDIAN_LOADS)
// Back-copy fast path, copying 8 instead of 1 bytes at a time.
@@ -312,7 +313,7 @@ outer_loop:
// 8*N >= length, i.e. we round length up to a multiple of 8), instead of
// only length bytes, but that's OK, as subsequent iterations will fix up
// the overrun.
if (distance >= 8) {
if (dist_minus_1 + 1 >= 8) {
while (1) {
*((uint64_t*)(pdst)) = *((uint64_t*)(pback));
if (length <= 8) {
+4 -4
View File
@@ -49,10 +49,10 @@ pri const lcode_magic_numbers[32] u32 = $(
)
pri const dcode_magic_numbers[32] u32 = $(
0x40000100, 0x40000200, 0x40000300, 0x40000400, 0x40000510, 0x40000710, 0x40000920, 0x40000D20,
0x40001130, 0x40001930, 0x40002140, 0x40003140, 0x40004150, 0x40006150, 0x40008160, 0x4000C160,
0x40010170, 0x40018170, 0x40020180, 0x40030180, 0x40040190, 0x40060190, 0x400801A0, 0x400C01A0,
0x401001B0, 0x401801B0, 0x402001C0, 0x403001C0, 0x404001D0, 0x406001D0, 0x08000000, 0x08000000,
0x40000000, 0x40000100, 0x40000200, 0x40000300, 0x40000410, 0x40000610, 0x40000820, 0x40000C20,
0x40001030, 0x40001830, 0x40002040, 0x40003040, 0x40004050, 0x40006050, 0x40008060, 0x4000C060,
0x40010070, 0x40018070, 0x40020080, 0x40030080, 0x40040090, 0x40060090, 0x400800A0, 0x400C00A0,
0x401000B0, 0x401800B0, 0x402000C0, 0x403000C0, 0x404000D0, 0x406000D0, 0x08000000, 0x08000000,
)
pub struct decoder?(
+24 -17
View File
@@ -156,7 +156,7 @@ pri func decoder.decode_huffman_fast?(dst writer1, src reader1)() {
return error "internal error: inconsistent Huffman decoder state"
}
// length = base number + extra bits.
// length = base_number + extra_bits.
var length u32[..0x7FFF] = (table_entry >> 8) & 0x7FFF
table_entry_n_bits = (table_entry >> 4) & 0x0F
if table_entry_n_bits > 0 {
@@ -237,7 +237,7 @@ pri func decoder.decode_huffman_fast?(dst writer1, src reader1)() {
assert in.src.available() >= 2
}
// For H-D, all symbols should be base number + extra bits.
// For H-D, all symbols should be base_number + extra_bits.
if (table_entry >> 24) != 0x40 {
if (table_entry >> 24) == 0x08 {
return error "bad Huffman code"
@@ -245,9 +245,13 @@ pri func decoder.decode_huffman_fast?(dst writer1, src reader1)() {
return error "internal error: inconsistent Huffman decoder state"
}
// distance = base number + extra bits.
// distance = base_number_minus_1 + extra_bits.
//
// TODO: the upper bound isn't right. 0x8000 is a valid distance.
// TODO: rename distance to dist_minus_1.
//
// The -1 is from the bias in script/print-deflate-magic-numbers.go.
// That bias makes the "& 0x7FFF" 15-ish lines below correct and
// undoing that bias makes proving (distance + 1) > 0 trivial.
var distance u32[..0x7FFF] = (table_entry >> 8) & 0x7FFF
table_entry_n_bits = (table_entry >> 4) & 0x0F
if table_entry_n_bits > 0 {
@@ -267,9 +271,6 @@ pri func decoder.decode_huffman_fast?(dst writer1, src reader1)() {
bits >>= table_entry_n_bits
n_bits -= table_entry_n_bits
}
if distance <= 0 {
return error "internal error: inconsistent Huffman decoder state"
}
var n_copied u32
// The "while true { etc; break }" is a redundant version of "etc", but
@@ -277,18 +278,20 @@ pri func decoder.decode_huffman_fast?(dst writer1, src reader1)() {
// decode_huffman_slow.
while true,
pre length <= 258,
pre distance > 0,
pre in.dst.available() >= 258,
inv in.dst.is_marked(),
{
// Copy from this.history.
if (distance as u64) > in.dst.since_mark().length() {
if ((distance + 1) as u64) > in.dst.since_mark().length() {
// Set (hlen, hdist) to be the length-distance pair to copy
// from this.history, and (length, distance) to be the
// remaining length-distance pair to copy from in.dst.
var hlen u32[..0x7FFF]
var hdist u32[..0x7FFF] = ((distance as u64) - in.dst.since_mark().length()) as u32
var hdist u32 =
(((distance + 1) as u64) - in.dst.since_mark().length()) as u32
if length > hdist {
assert hdist < length via "a < b: b > a"()
assert hdist < 0x8000 via "a < b: a < c; c <= b"(c:length)
length -= hdist
hlen = hdist
// TODO: this if check should be redundant.
@@ -305,24 +308,28 @@ pri func decoder.decode_huffman_fast?(dst writer1, src reader1)() {
}
// Re-purpose the hdist variable as the this.history index to
// start copying from.
hdist = (this.history_index - hdist) & 0x7FFF
hdist = this.history_index - hdist
// Copy from hdist to the end of this.history, and then
// possibly wrap around the this.history ringbuffer.
//
// This copying is simpler than the decode_huffman_slow version
// because it cannot yield. We have already checked that
// in.dst.available() is large enough.
//
// The "while true { etc; break }" is a redundant version of
// "etc", but its presence minimizes the diff between
// decode_huffman_fast and decode_huffman_slow.
while true,
inv length <= 258,
inv distance > 0,
inv in.dst.available() >= 258,
inv in.dst.is_marked(),
{
// TODO: copy_from_slice32 should probably update the
// "in.dst.available() >= 258" fact. It should certainly
// invalidate any fact that was "<=" instead of ">=".
n_copied = in.dst.copy_from_slice32(s:this.history[hdist:], length:hlen)
n_copied = in.dst.copy_from_slice32(
s:this.history[hdist & 0x7FFF:], length:hlen)
if hlen <= n_copied {
break
}
@@ -336,23 +343,23 @@ pri func decoder.decode_huffman_fast?(dst writer1, src reader1)() {
continue:loop
}
if (distance as u64) > in.dst.since_mark().length() {
if ((distance + 1) as u64) > in.dst.since_mark().length() {
return error "internal error: inconsistent distance"
}
}
// Once again, redundant but explicit assertions.
assert length <= 258
assert distance > 0
assert (distance as u64) <= in.dst.since_mark().length()
assert ((distance + 1) as u64) <= in.dst.since_mark().length()
assert in.dst.available() >= 258
assert in.dst.is_marked()
// We can therefore prove:
assert (distance + 1) > 0
assert (length as u64) <= 258
assert (length as u64) <= in.dst.available() via "a <= b: a <= c; c <= b"(c:258)
// Copy from in.dst.
in.dst.copy_from_history32(distance:distance, length:length)
in.dst.copy_from_history32(distance:(distance + 1), length:length)
break
}
}
+20 -14
View File
@@ -99,7 +99,7 @@ pri func decoder.decode_huffman_slow?(dst writer1, src reader1)() {
return error "internal error: inconsistent Huffman decoder state"
}
// length = base number + extra bits.
// length = base_number + extra_bits.
var length u32[..0x7FFF] = (table_entry >> 8) & 0x7FFF
table_entry_n_bits = (table_entry >> 4) & 0x0F
if table_entry_n_bits > 0 {
@@ -149,7 +149,7 @@ pri func decoder.decode_huffman_slow?(dst writer1, src reader1)() {
}
}
// For H-D, all symbols should be base number + extra bits.
// For H-D, all symbols should be base_number + extra_bits.
if (table_entry >> 24) != 0x40 {
if (table_entry >> 24) == 0x08 {
return error "bad Huffman code"
@@ -157,9 +157,13 @@ pri func decoder.decode_huffman_slow?(dst writer1, src reader1)() {
return error "internal error: inconsistent Huffman decoder state"
}
// distance = base number + extra bits.
// distance = base_number_minus_1 + extra_bits.
//
// TODO: the upper bound isn't right. 0x8000 is a valid distance.
// TODO: rename distance to dist_minus_1.
//
// The -1 is from the bias in script/print-deflate-magic-numbers.go.
// That bias makes the "& 0x7FFF" 15-ish lines below correct and
// undoing that bias makes proving (distance + 1) > 0 trivial.
var distance u32[..0x7FFF] = (table_entry >> 8) & 0x7FFF
table_entry_n_bits = (table_entry >> 4) & 0x0F
if table_entry_n_bits > 0 {
@@ -174,20 +178,20 @@ pri func decoder.decode_huffman_slow?(dst writer1, src reader1)() {
bits >>= table_entry_n_bits
n_bits -= table_entry_n_bits
}
if distance <= 0 {
return error "internal error: inconsistent Huffman decoder state"
}
var n_copied u32
while true {
// Copy from this.history.
if (distance as u64) > in.dst.since_mark().length() {
if ((distance + 1) as u64) > in.dst.since_mark().length() {
// Set (hlen, hdist) to be the length-distance pair to copy
// from this.history, and (length, distance) to be the
// remaining length-distance pair to copy from in.dst.
var hlen u32[..0x7FFF]
var hdist u32[..0x7FFF] = ((distance as u64) - in.dst.since_mark().length()) as u32
var hdist u32 =
(((distance + 1) as u64) - in.dst.since_mark().length()) as u32
if length > hdist {
assert hdist < length via "a < b: b > a"()
assert hdist < 0x8000 via "a < b: a < c; c <= b"(c:length)
length -= hdist
hlen = hdist
} else {
@@ -199,11 +203,12 @@ pri func decoder.decode_huffman_slow?(dst writer1, src reader1)() {
}
// Re-purpose the hdist variable as the this.history index to
// start copying from.
hdist = (this.history_index - hdist) & 0x7FFF
hdist = this.history_index - hdist
// Copy from hdist to the end of this.history.
while true {
n_copied = in.dst.copy_from_slice32(s:this.history[hdist:], length:hlen)
n_copied = in.dst.copy_from_slice32(
s:this.history[hdist & 0x7FFF:], length:hlen)
if hlen <= n_copied {
hlen = 0
break
@@ -222,13 +227,14 @@ pri func decoder.decode_huffman_slow?(dst writer1, src reader1)() {
// Copy from the start of this.history, if we wrapped around.
if hlen > 0 {
while true {
n_copied = in.dst.copy_from_slice32(s:this.history[hdist:], length:hlen)
n_copied = in.dst.copy_from_slice32(
s:this.history[hdist & 0x7FFF:], length:hlen)
if hlen <= n_copied {
hlen = 0
break
}
hlen -= n_copied
hdist = (hdist + (n_copied & 0x7FFF)) & 0x7FFF
hdist = hdist ~+ n_copied
// TODO: "closed for writes" instead?
yield suspension "short write"
}
@@ -241,7 +247,7 @@ pri func decoder.decode_huffman_slow?(dst writer1, src reader1)() {
}
// Copy from in.dst.
n_copied = in.dst.copy_from_history32(distance:distance, length:length)
n_copied = in.dst.copy_from_history32(distance:(distance + 1), length:length)
if length <= n_copied {
length = 0
break
+1 -1
View File
@@ -684,7 +684,7 @@ proc tests[] = {
test_wuffs_deflate_decode_256_bytes, //
test_wuffs_deflate_decode_deflate_backref_crosses_blocks, //
// test_wuffs_deflate_decode_deflate_distance_32768, // TODO: uncomment.
test_wuffs_deflate_decode_deflate_distance_32768, //
test_wuffs_deflate_decode_midsummer, //
test_wuffs_deflate_decode_pi, //
test_wuffs_deflate_decode_pi_many_big_reads, //