Decode flate's clcode symbols.

This commit is contained in:
Nigel Tao committed 2017-07-21 17:47:58 +10:00
1 parent 74400247f2
commit fd07fa1f27
5 files changed
+299 -30

No files matched your search

+137 -14
View File
@@ -102,18 +102,19 @@ typedef int32_t puffs_flate_status;
#define PUFFS_FLATE_SUSPENSION_LIMITED_READ 9 // 0x00000009
#define PUFFS_FLATE_SUSPENSION_LIMITED_WRITE 10 // 0x0000000a
#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040128 // 0xbb08f800
#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040127 // 0xbb08f801
#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE -1157040128 // 0xbb08f800
#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040127 // 0xbb08f801
#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040126 // 0xbb08f802
#define PUFFS_FLATE_ERROR_BAD_LITERAL_LENGTH_CODE_COUNT \
-1157040126 // 0xbb08f802
#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \
-1157040125 // 0xbb08f803
#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_INIT_HUFF_STATE \
#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \
-1157040124 // 0xbb08f804
#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \
-1157040123 // 0xbb08f805
#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_N_BITS \
-1157040123 // 0xbb08f805
#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040122 // 0xbb08f806
#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040121 // 0xbb08f807
-1157040122 // 0xbb08f806
#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040121 // 0xbb08f807
#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040120 // 0xbb08f808
bool puffs_flate_status_is_error(puffs_flate_status s);
@@ -177,6 +178,13 @@ typedef struct {
uint32_t v_n_dist;
uint32_t v_n_clen;
uint32_t v_i;
uint32_t v_symbol;
uint32_t v_code;
uint32_t v_count;
uint32_t v_first;
uint32_t v_index;
uint32_t v_length;
uint32_t v_n_extra_bits;
} c_init_huffs[1];
struct {
uint32_t coro_susp_point;
@@ -267,12 +275,13 @@ const char* puffs_flate_status_strings0[11] = {
"flate: limited write",
};
const char* puffs_flate_status_strings1[8] = {
const char* puffs_flate_status_strings1[9] = {
"flate: bad Huffman code",
"flate: bad distance code count",
"flate: bad flate block",
"flate: bad literal/length code count",
"flate: inconsistent stored block length",
"flate: internal error: inconsistent init_huff state",
"flate: internal error: inconsistent Huffman decoder state",
"flate: internal error: inconsistent n_bits",
"flate: no Huffman codes",
"flate: TODO: fixed Huffman blocks",
@@ -288,7 +297,7 @@ const char* puffs_flate_status_string(puffs_flate_status s) {
break;
case puffs_flate_packageid:
a = puffs_flate_status_strings1;
n = 8;
n = 9;
break;
}
uint32_t i = s & 0xff;
@@ -701,6 +710,13 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self,
uint32_t v_n_dist;
uint32_t v_n_clen;
uint32_t v_i;
uint32_t v_symbol;
uint32_t v_code;
uint32_t v_count;
uint32_t v_first;
uint32_t v_index;
uint32_t v_length;
uint32_t v_n_extra_bits;
uint8_t* b_rptr_src = NULL;
uint8_t* b_rend_src = NULL;
@@ -724,6 +740,13 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self,
v_n_dist = self->private_impl.c_init_huffs[0].v_n_dist;
v_n_clen = self->private_impl.c_init_huffs[0].v_n_clen;
v_i = self->private_impl.c_init_huffs[0].v_i;
v_symbol = self->private_impl.c_init_huffs[0].v_symbol;
v_code = self->private_impl.c_init_huffs[0].v_code;
v_count = self->private_impl.c_init_huffs[0].v_count;
v_first = self->private_impl.c_init_huffs[0].v_first;
v_index = self->private_impl.c_init_huffs[0].v_index;
v_length = self->private_impl.c_init_huffs[0].v_length;
v_n_extra_bits = self->private_impl.c_init_huffs[0].v_n_extra_bits;
}
switch (coro_susp_point) {
PUFFS_COROUTINE_SUSPENSION_POINT(0);
@@ -780,6 +803,97 @@ puffs_flate_status puffs_flate_decoder_init_huffs(puffs_flate_decoder* self,
if (status) {
goto suspend;
}
v_i = 0;
while (v_i < 31) {
v_symbol = 0;
v_code = 0;
v_count = 0;
v_first = 0;
v_index = 0;
v_length = 1;
while (true) {
if (v_length > 15) {
status = PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE;
goto exit;
}
while (v_n_bits <= 0) {
PUFFS_COROUTINE_SUSPENSION_POINT(4);
if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) {
goto short_read_src;
}
uint8_t t_2 = *b_rptr_src++;
v_bits |= (((uint32_t)(t_2)) << v_n_bits);
v_n_bits += 8;
}
v_code |= (v_bits & 1);
if (v_code > 65535) {
status =
PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
v_bits >>= 1;
v_n_bits -= 1;
v_count = ((uint32_t)(
self->private_impl.f_huffs[0].private_impl.f_counts[v_length]));
if (v_count > 65535) {
status =
PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
if (v_code < (v_first + v_count)) {
v_index += v_code;
if (v_index < v_first) {
status =
PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
v_index -= v_first;
if (v_index >= 290) {
status =
PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
v_symbol = ((uint32_t)(
self->private_impl.f_huffs[0].private_impl.f_symbols[v_index]));
goto label_0_break;
}
v_index += v_count;
if (v_index > 65535) {
status =
PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
v_first = ((v_first + v_count) << 1);
if (v_first > 65535) {
status =
PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
v_code <<= 1;
v_length += 1;
}
label_0_break:;
v_n_extra_bits = 0;
if (v_symbol == 16) {
v_n_extra_bits = 2;
} else if (v_symbol == 17) {
v_n_extra_bits = 3;
} else if (v_symbol == 18) {
v_n_extra_bits = 7;
}
while (v_n_bits < v_n_extra_bits) {
PUFFS_COROUTINE_SUSPENSION_POINT(5);
if (PUFFS_UNLIKELY(b_rptr_src == b_rend_src)) {
goto short_read_src;
}
uint8_t t_3 = *b_rptr_src++;
v_bits |= (((uint32_t)(t_3)) << v_n_bits);
v_n_bits += 8;
}
v_bits >>= v_n_extra_bits;
v_n_bits -= v_n_extra_bits;
v_i += 1;
}
self->private_impl.f_wip0 = v_n_lit;
self->private_impl.f_wip1 = v_n_dist;
self->private_impl.f_bits = v_bits;
@@ -797,6 +911,13 @@ suspend:
self->private_impl.c_init_huffs[0].v_n_dist = v_n_dist;
self->private_impl.c_init_huffs[0].v_n_clen = v_n_clen;
self->private_impl.c_init_huffs[0].v_i = v_i;
self->private_impl.c_init_huffs[0].v_symbol = v_symbol;
self->private_impl.c_init_huffs[0].v_code = v_code;
self->private_impl.c_init_huffs[0].v_count = v_count;
self->private_impl.c_init_huffs[0].v_first = v_first;
self->private_impl.c_init_huffs[0].v_index = v_index;
self->private_impl.c_init_huffs[0].v_length = v_length;
self->private_impl.c_init_huffs[0].v_n_extra_bits = v_n_extra_bits;
exit:
if (a_src.buf) {
@@ -854,7 +975,8 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self,
if (self->private_impl.f_huffs[a_which]
.private_impl.f_counts[self->private_impl.f_code_lengths[v_i]] >=
289) {
status = PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_INIT_HUFF_STATE;
status =
PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
self->private_impl.f_huffs[a_which]
@@ -879,7 +1001,8 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self,
v_offsets[v_i] = v_total;
v_count = self->private_impl.f_huffs[a_which].private_impl.f_counts[v_i];
if (v_total > (289 - v_count)) {
status = PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_INIT_HUFF_STATE;
status =
PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
v_total = (v_total + v_count);
@@ -894,7 +1017,7 @@ puffs_flate_status puffs_flate_decoder_init_huff(puffs_flate_decoder* self,
((uint16_t)(v_i));
if (v_offsets[self->private_impl.f_code_lengths[v_i]] >= 289) {
status =
PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_INIT_HUFF_STATE;
PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE;
goto exit;
}
v_offsets[self->private_impl.f_code_lengths[v_i]] += 1;
+16 -8
View File
@@ -102,18 +102,19 @@ typedef int32_t puffs_flate_status;
#define PUFFS_FLATE_SUSPENSION_LIMITED_READ 9 // 0x00000009
#define PUFFS_FLATE_SUSPENSION_LIMITED_WRITE 10 // 0x0000000a
#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040128 // 0xbb08f800
#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040127 // 0xbb08f801
#define PUFFS_FLATE_ERROR_BAD_HUFFMAN_CODE -1157040128 // 0xbb08f800
#define PUFFS_FLATE_ERROR_BAD_DISTANCE_CODE_COUNT -1157040127 // 0xbb08f801
#define PUFFS_FLATE_ERROR_BAD_FLATE_BLOCK -1157040126 // 0xbb08f802
#define PUFFS_FLATE_ERROR_BAD_LITERAL_LENGTH_CODE_COUNT \
-1157040126 // 0xbb08f802
#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \
-1157040125 // 0xbb08f803
#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_INIT_HUFF_STATE \
#define PUFFS_FLATE_ERROR_INCONSISTENT_STORED_BLOCK_LENGTH \
-1157040124 // 0xbb08f804
#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_HUFFMAN_DECODER_STATE \
-1157040123 // 0xbb08f805
#define PUFFS_FLATE_ERROR_INTERNAL_ERROR_INCONSISTENT_N_BITS \
-1157040123 // 0xbb08f805
#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040122 // 0xbb08f806
#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040121 // 0xbb08f807
-1157040122 // 0xbb08f806
#define PUFFS_FLATE_ERROR_NO_HUFFMAN_CODES -1157040121 // 0xbb08f807
#define PUFFS_FLATE_ERROR_TODO_FIXED_HUFFMAN_BLOCKS -1157040120 // 0xbb08f808
bool puffs_flate_status_is_error(puffs_flate_status s);
@@ -177,6 +178,13 @@ typedef struct {
uint32_t v_n_dist;
uint32_t v_n_clen;
uint32_t v_i;
uint32_t v_symbol;
uint32_t v_code;
uint32_t v_count;
uint32_t v_first;
uint32_t v_index;
uint32_t v_length;
uint32_t v_n_extra_bits;
} c_init_huffs[1];
struct {
uint32_t coro_susp_point;
+4 -1
View File
@@ -67,7 +67,10 @@ func (z *facts) appendFact(fact *a.Expr) {
z.appendFact(fact.RHS().Expr())
return
case t.KeyXAssociativeAnd:
// TODO.
for _, a := range fact.Args() {
z.appendFact(a.Expr())
}
return
}
*z = append(*z, fact)
+16 -2
View File
@@ -101,7 +101,7 @@ func invert(tm *t.Map, n *a.Expr) (*a.Expr, error) {
if cv := n.ConstValue(); cv != nil {
return nil, fmt.Errorf("check: invert(%q) called on constant expression", n.String(tm))
}
id0, lhs, rhs := n.ID0(), n.LHS().Expr(), n.RHS().Expr()
id0, lhs, rhs, args := n.ID0(), n.LHS().Expr(), n.RHS().Expr(), []*a.Node(nil)
switch id0.Key() {
case t.KeyXUnaryNot:
return rhs, nil
@@ -132,10 +132,24 @@ func invert(tm *t.Map, n *a.Expr) (*a.Expr, error) {
} else {
id0 = t.IDXBinaryAnd
}
case t.KeyXAssociativeAnd, t.KeyXAssociativeOr:
args = make([]*a.Node, 0, len(n.Args()))
for _, a := range n.Args() {
v, err := invert(tm, a.Expr())
if err != nil {
return nil, err
}
args = append(args, v.Node())
}
if id0.Key() == t.KeyXAssociativeAnd {
id0 = t.IDXAssociativeOr
} else {
id0 = t.IDXAssociativeAnd
}
default:
id0, lhs, rhs = t.IDXUnaryNot, nil, n
}
o := a.NewExpr(n.Node().Raw().Flags(), id0, 0, lhs.Node(), nil, rhs.Node(), nil)
o := a.NewExpr(n.Node().Raw().Flags(), id0, 0, lhs.Node(), nil, rhs.Node(), args)
o.SetMType(n.MType())
return o, nil
}
+126 -5
View File
@@ -5,11 +5,12 @@
packageid "flte"
pub error "bad Huffman code"
pub error "bad distance code count"
pub error "bad flate block"
pub error "bad literal/length code count"
pub error "inconsistent stored block length"
pub error "internal error: inconsistent init_huff state"
pub error "internal error: inconsistent Huffman decoder state"
pub error "internal error: inconsistent n_bits"
pub error "no Huffman codes"
@@ -125,7 +126,127 @@ pri func decoder.init_huffs?(src reader1)() {
}
this.init_huff?(which:0, n_codes:19)
// TODO: implement.
i = 0
while i < 31 { // TODO: replace 31 with n_lit + n_dist.
// Decode a clcode symbol from H-CL.
var symbol u32[..289]
var code u32
var count u32
var first u32
var index u32
var length u32 = 1
while true,
pre first <= 0xFFFF,
pre index <= 0xFFFF,
inv i < 31,
{
if length > 15 {
return error "bad Huffman code"
}
// Read the next bit.
while n_bits <= 0,
inv first <= 0xFFFF,
inv index <= 0xFFFF,
inv i < 31,
inv length <= 15,
post n_bits > 0,
{
bits |= (in.src.read_u8?() as u32) << n_bits
n_bits += 8
}
code |= bits & 1
// TODO: improve the proof engine's bounds checking so that any
// check returning "internal error: inconsistent Huffman decoder
// state" can be removed. There is one here and more below.
if code > 0xFFFF {
return error "internal error: inconsistent Huffman decoder state"
}
bits >>= 1
n_bits -= 1
count = this.huffs[0].counts[length] as u32
if count > 0xFFFF {
return error "internal error: inconsistent Huffman decoder state"
}
// Compare the code so far to the counts.
//
// For the clcode example in this package's README.md:
// - counts[1] = 0
// - counts[2] = 0
// - counts[3] = 7
// - counts[4] = 1
// - counts[5] = 2
//
// Recall that the map from bits to clcode symbols is:
// bits clcode
// 0b000 0
// 0b001 4
// 0b010 5
// 0b011 6
// 0b100 7
// 0b101 8
// 0b110 9
// 0b1110 17
// 0b11110 3
// 0b11111 18
//
// The loop iterations at this point have:
// - length=1, count=0, first= 0, index=0, code in [0.. 1].
// - length=2, count=0, first= 0, index=0, code in [0.. 3].
// - length=3, count=7, first= 0, index=0, code in [0.. 7].
// - length=4, count=1, first=14, index=7, code in [0..15].
// - length=5, count=2, first=30, index=8, code in [0..31].
if code < (first + count) {
index += code
if index < first {
return error "internal error: inconsistent Huffman decoder state"
}
index -= first
if index >= 290 {
return error "internal error: inconsistent Huffman decoder state"
}
symbol = this.huffs[0].symbols[index] as u32
break
}
index += count
if index > 0xFFFF {
return error "internal error: inconsistent Huffman decoder state"
}
first = (first + count) << 1
if first > 0xFFFF {
return error "internal error: inconsistent Huffman decoder state"
}
code <<= 1
length += 1
}
// Read any extra bits, as per the RFC 1951 section 3.2.7.
var n_extra_bits u32[..7]
if symbol == 16 {
n_extra_bits = 2
} else if symbol == 17 {
n_extra_bits = 3
} else if symbol == 18 {
n_extra_bits = 7
}
while n_bits < n_extra_bits,
inv i < 31,
post n_bits >= n_extra_bits,
{
assert n_bits < 7 via "a < b: a < c; c <= b"(c:n_extra_bits)
bits |= (in.src.read_u8?() as u32) << n_bits
n_bits += 8
}
// TODO: do something with those extra bits.
bits >>= n_extra_bits
n_bits -= n_extra_bits
i += 1
}
this.wip0 = n_lit
this.wip1 = n_dist
@@ -176,7 +297,7 @@ pri func decoder.init_huff?(which u32[..1], n_codes u32[..289])() {
// TODO: this if should be unnecessary. Have some way to assert that,
// for all j, counts[j] <= i, and thus counts[j]++ will not overflow.
if this.huffs[in.which].counts[this.code_lengths[i]] >= 289 {
return error "internal error: inconsistent init_huff state"
return error "internal error: inconsistent Huffman decoder state"
}
this.huffs[in.which].counts[this.code_lengths[i]] += 1
i += 1
@@ -203,7 +324,7 @@ pri func decoder.init_huff?(which u32[..1], n_codes u32[..289])() {
offsets[i] = total
var count u16[..289] = this.huffs[in.which].counts[i]
if total > (289 - count) {
return error "internal error: inconsistent init_huff state"
return error "internal error: inconsistent Huffman decoder state"
}
assert (total + count) <= 289 via "(a + b) <= c: a <= (c - b)"()
// TODO: change this to total += count, once the proof engine's bounds
@@ -228,7 +349,7 @@ pri func decoder.init_huff?(which u32[..1], n_codes u32[..289])() {
if this.code_lengths[i] != 0 {
this.huffs[in.which].symbols[offsets[this.code_lengths[i]]] = i as u16
if offsets[this.code_lengths[i]] >= 289 {
return error "internal error: inconsistent init_huff state"
return error "internal error: inconsistent Huffman decoder state"
}
offsets[this.code_lengths[i]] += 1
}