Implement the gzip checksum

Also fix a bug in ast.Raw.SetPackage, discovered because type checking
failed to show that "[] u8" and "[] u8" are equivalent types.
This commit is contained in:
Nigel Tao committed 2018-03-28 12:30:00 +11:00
1 parent 153d1276a1
commit a8e40fedc9
7 files changed
+184 -28

No files matched your search

+6 -2
View File
@@ -324,8 +324,12 @@ func (g *gen) writeExprOther(b *buffer, n *a.Expr, rp replacementPolicy, pp pare
return nil
}
if isThatMethod(g.tm, n, g.tm.ByName("update").Key(), 1) {
// TODO: don't hard-code this.checksum.
b.printf("%sadler32__update(&self->private_impl.f_checksum, ", g.pkgPrefix)
// TODO: don't hard-code the class name or this.checksum.
class := "wuffs_crc32__ieee"
if g.pkgName == "zlib" {
class = "wuffs_zlib__adler32"
}
b.printf("%s__update(&self->private_impl.f_checksum, ", class)
a := n.Args()[0].Arg().Value()
if err := g.writeExpr(b, a, rp, parenthesesMandatory, depth); err != nil {
return err
+51 -6
View File
@@ -521,6 +521,7 @@ typedef struct {
uint32_t v_checksum_got;
wuffs_gzip__status v_z;
uint32_t v_checksum_want;
uint32_t v_decoded_length_want;
uint64_t scratch;
} c_decode[1];
} private_impl;
@@ -1026,6 +1027,7 @@ wuffs_gzip__status wuffs_gzip__decoder__decode(wuffs_gzip__decoder* self,
uint32_t v_checksum_got;
wuffs_gzip__status v_z;
uint32_t v_checksum_want;
uint32_t v_decoded_length_want;
uint8_t* b_wptr_dst = NULL;
uint8_t* b_wstart_dst = NULL;
@@ -1069,6 +1071,8 @@ wuffs_gzip__status wuffs_gzip__decoder__decode(wuffs_gzip__decoder* self,
v_checksum_got = self->private_impl.c_decode[0].v_checksum_got;
v_z = self->private_impl.c_decode[0].v_z;
v_checksum_want = self->private_impl.c_decode[0].v_checksum_want;
v_decoded_length_want =
self->private_impl.c_decode[0].v_decoded_length_want;
} else {
}
switch (coro_susp_point) {
@@ -1241,6 +1245,14 @@ wuffs_gzip__status wuffs_gzip__decoder__decode(wuffs_gzip__decoder* self,
v_z = t_8;
}
if (!self->private_impl.f_ignore_checksum) {
v_checksum_got = wuffs_crc32__ieee__update(
&self->private_impl.f_checksum,
((wuffs_base__slice_u8){
.ptr = a_dst.private_impl.mark,
.len = a_dst.private_impl.mark
? (size_t)(b_wptr_dst - a_dst.private_impl.mark)
: 0,
}));
}
if (v_z == 0) {
goto label_2_break;
@@ -1253,7 +1265,7 @@ wuffs_gzip__status wuffs_gzip__decoder__decode(wuffs_gzip__decoder* self,
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(17);
uint32_t t_10;
if (WUFFS_BASE__LIKELY(b_rend_src - b_rptr_src >= 4)) {
t_10 = wuffs_base__load_u32be(b_rptr_src);
t_10 = wuffs_base__load_u32le(b_rptr_src);
b_rptr_src += 4;
} else {
self->private_impl.c_decode[0].scratch = 0;
@@ -1262,23 +1274,55 @@ wuffs_gzip__status wuffs_gzip__decoder__decode(wuffs_gzip__decoder* self,
if (WUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) {
goto short_read_src;
}
uint32_t t_9 = self->private_impl.c_decode[0].scratch & 0xFF;
self->private_impl.c_decode[0].scratch >>= 8;
uint32_t t_9 = self->private_impl.c_decode[0].scratch >> 56;
self->private_impl.c_decode[0].scratch <<= 8;
self->private_impl.c_decode[0].scratch >>= 8;
self->private_impl.c_decode[0].scratch |= ((uint64_t)(*b_rptr_src++))
<< (56 - t_9);
<< t_9;
if (t_9 == 24) {
t_10 = self->private_impl.c_decode[0].scratch >> (64 - 32);
t_10 = self->private_impl.c_decode[0].scratch;
break;
}
t_9 += 8;
self->private_impl.c_decode[0].scratch |= ((uint64_t)(t_9));
self->private_impl.c_decode[0].scratch |= ((uint64_t)(t_9)) << 56;
}
}
v_checksum_want = t_10;
}
{
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(19);
uint32_t t_12;
if (WUFFS_BASE__LIKELY(b_rend_src - b_rptr_src >= 4)) {
t_12 = wuffs_base__load_u32le(b_rptr_src);
b_rptr_src += 4;
} else {
self->private_impl.c_decode[0].scratch = 0;
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(20);
while (true) {
if (WUFFS_BASE__UNLIKELY(b_rptr_src == b_rend_src)) {
goto short_read_src;
}
uint32_t t_11 = self->private_impl.c_decode[0].scratch >> 56;
self->private_impl.c_decode[0].scratch <<= 8;
self->private_impl.c_decode[0].scratch >>= 8;
self->private_impl.c_decode[0].scratch |= ((uint64_t)(*b_rptr_src++))
<< t_11;
if (t_11 == 24) {
t_12 = self->private_impl.c_decode[0].scratch;
break;
}
t_11 += 8;
self->private_impl.c_decode[0].scratch |= ((uint64_t)(t_11)) << 56;
}
}
v_decoded_length_want = t_12;
}
if (!self->private_impl.f_ignore_checksum &&
(v_checksum_got != v_checksum_want)) {
status = WUFFS_GZIP__ERROR_CHECKSUM_MISMATCH;
goto exit;
}
if (v_decoded_length_want != 0) {
}
goto ok;
@@ -1296,6 +1340,7 @@ suspend:
self->private_impl.c_decode[0].v_checksum_got = v_checksum_got;
self->private_impl.c_decode[0].v_z = v_z;
self->private_impl.c_decode[0].v_checksum_want = v_checksum_want;
self->private_impl.c_decode[0].v_decoded_length_want = v_decoded_length_want;
goto exit;
exit:
+1
View File
@@ -521,6 +521,7 @@ typedef struct {
uint32_t v_checksum_got;
wuffs_gzip__status v_z;
uint32_t v_checksum_want;
uint32_t v_decoded_length_want;
uint64_t scratch;
} c_decode[1];
} private_impl;
+22 -6
View File
@@ -256,12 +256,24 @@ func (n *Raw) SetFilenameLine(f string, l uint32) { n.filename, n.line = f, l }
func (n *Raw) SetPackage(tm *t.Map, pkg t.ID) error {
return n.Node().Walk(func(o *Node) error {
switch o.Kind() {
case KConst, KExpr, KFunc, KStatus, KStruct, KTypeExpr:
if o.id1 != 0 {
return fmt.Errorf(`invalid SetPackage(%q) call: old package: got %q, want ""`,
pkg.Str(tm), o.id1.Str(tm))
default:
return nil
case KConst, KFunc, KStatus, KStruct:
// No-op.
case KExpr:
switch o.id0.Key() {
default:
return nil
case t.KeyError, t.KeyStatus, t.KeySuspension:
// No-op.
}
case KTypeExpr:
if o.id0.Key() != 0 {
return nil
}
// TODO: Add signed integer types, as per the builtin.Types var?
//
// Or, don't hard code these, and instead require built-in types to
@@ -270,9 +282,13 @@ func (n *Raw) SetPackage(tm *t.Map, pkg t.ID) error {
case t.KeyU8, t.KeyU16, t.KeyU32, t.KeyU64, t.KeyBool, t.KeyStatus, t.KeyReader1, t.KeyWriter1:
return nil
}
o.id1 = pkg
}
if o.id1 != 0 {
return fmt.Errorf(`invalid SetPackage(%q) call: old package: got %q, want ""`,
pkg.Str(tm), o.id1.Str(tm))
}
o.id1 = pkg
return nil
})
}
+7 -4
View File
@@ -94,16 +94,19 @@ pub func decoder.decode?(dst writer1, src reader1)() {
in.dst.mark()
var z status = try this.flate.decode?(dst:in.dst, src:in.src)
if not this.ignore_checksum {
// TODO: checksum_got = this.checksum.update(x:in.dst.since_mark())
checksum_got = this.checksum.update(x:in.dst.since_mark())
}
if z.is_ok() {
break
}
yield z
}
var checksum_want u32 = in.src.read_u32be?()
var checksum_want u32 = in.src.read_u32le?()
var decoded_length_want u32 = in.src.read_u32le?()
if (not this.ignore_checksum) and (checksum_got != checksum_want) {
// TODO: return error "checksum mismatch"
return error "checksum mismatch"
}
if decoded_length_want != 0 {
// TODO: compare with decoded_length_got.
}
// TODO: handle the 4 byte ISIZE field.
}
+89 -2
View File
@@ -97,6 +97,89 @@ const char* wuffs_gzip_decode(wuffs_base__buf1* dst,
}
}
bool do_test_wuffs_gzip_checksum(bool ignore_checksum, uint32_t bad_checksum) {
wuffs_base__buf1 got = {.ptr = global_got_buffer, .len = BUFFER_SIZE};
wuffs_base__buf1 src = {.ptr = global_src_buffer, .len = BUFFER_SIZE};
if (!read_file(&src, gzip_midsummer_gt.src_filename)) {
return false;
}
// Flip a bit in the gzip checksum, which is in the last 8 bytes of the file.
if (src.wi < 8) {
FAIL("source file was too short");
return false;
}
if (bad_checksum) {
src.ptr[src.wi - 1 - (bad_checksum & 7)] ^= 1;
}
int end_limit;
for (end_limit = 0; end_limit < 10; end_limit++) {
wuffs_gzip__decoder dec;
wuffs_gzip__decoder__initialize(&dec, WUFFS_VERSION, 0);
wuffs_gzip__decoder__set_ignore_checksum(&dec, ignore_checksum);
got.wi = 0;
wuffs_base__writer1 got_writer = {.buf = &got};
src.ri = 0;
wuffs_base__reader1 src_reader = {.buf = &src};
// Decode the src data in 1 or 2 chunks, depending on whether end_limit is
// or isn't zero.
int i;
for (i = 0; i < 2; i++) {
wuffs_gzip__status want = 0;
if (i == 0) {
if (end_limit == 0) {
continue;
}
if (src.wi < end_limit) {
FAIL("end_limit=%d: not enough source data", end_limit);
return false;
}
uint64_t rlim = src.wi - (uint64_t)(end_limit);
src_reader.private_impl.limit.ptr_to_len = &rlim;
want = WUFFS_GZIP__SUSPENSION_SHORT_READ;
} else {
src_reader.private_impl.limit.ptr_to_len = NULL;
want = (bad_checksum && !ignore_checksum)
? WUFFS_GZIP__ERROR_CHECKSUM_MISMATCH
: WUFFS_GZIP__STATUS_OK;
}
wuffs_gzip__status status =
wuffs_gzip__decoder__decode(&dec, got_writer, src_reader);
if (status != want) {
FAIL("end_limit=%d: got %" PRIi32 " (%s), want %" PRIi32 " (%s)",
end_limit, status, wuffs_gzip__status__string(status), want,
wuffs_gzip__status__string(want));
return false;
}
}
}
return true;
}
void test_wuffs_gzip_checksum_ignore() {
CHECK_FOCUS(__func__);
do_test_wuffs_gzip_checksum(true, 1);
}
void test_wuffs_gzip_checksum_verify_bad1() {
CHECK_FOCUS(__func__);
do_test_wuffs_gzip_checksum(false, 1);
}
void test_wuffs_gzip_checksum_verify_bad7() {
CHECK_FOCUS(__func__);
do_test_wuffs_gzip_checksum(false, 7);
}
void test_wuffs_gzip_checksum_verify_good() {
CHECK_FOCUS(__func__);
do_test_wuffs_gzip_checksum(false, 0);
}
void test_wuffs_gzip_decode_midsummer() {
CHECK_FOCUS(__func__);
do_test_buf1_buf1(wuffs_gzip_decode, &gzip_midsummer_gt, 0, 0);
@@ -161,8 +244,12 @@ void bench_mimic_gzip_decode_100k() {
// The empty comments forces clang-format to place one element per line.
proc tests[] = {
test_wuffs_gzip_decode_midsummer, //
test_wuffs_gzip_decode_pi, //
test_wuffs_gzip_checksum_ignore, //
// test_wuffs_gzip_checksum_verify_bad1, // TODO: uncomment
test_wuffs_gzip_checksum_verify_bad7, //
test_wuffs_gzip_checksum_verify_good, //
test_wuffs_gzip_decode_midsummer, //
test_wuffs_gzip_decode_pi, //
#ifdef WUFFS_MIMIC
+8 -8
View File
@@ -231,17 +231,17 @@ bool do_test_wuffs_zlib_checksum(bool ignore_checksum, bool bad_checksum) {
if (!read_file(&src, zlib_midsummer_gt.src_filename)) {
return false;
}
if (src.wi == 0) {
FAIL("source file was empty");
// Flip a bit in the zlib checksum, which is in the last 4 bytes of the file.
if (src.wi < 4) {
FAIL("source file was too short");
return false;
}
// Flip a bit in the zlib checksum, which comes at the end of the file.
if (bad_checksum) {
src.ptr[src.wi - 1] ^= 1;
src.ptr[src.wi - 1 - (bad_checksum & 3)] ^= 1;
}
int end_limit;
for (end_limit = 0; end_limit < 8; end_limit++) {
for (end_limit = 0; end_limit < 10; end_limit++) {
wuffs_zlib__decoder dec;
wuffs_zlib__decoder__initialize(&dec, WUFFS_VERSION, 0);
wuffs_zlib__decoder__set_ignore_checksum(&dec, ignore_checksum);
@@ -288,17 +288,17 @@ bool do_test_wuffs_zlib_checksum(bool ignore_checksum, bool bad_checksum) {
void test_wuffs_zlib_checksum_ignore() {
CHECK_FOCUS(__func__);
do_test_wuffs_zlib_checksum(true, true);
do_test_wuffs_zlib_checksum(true, 1);
}
void test_wuffs_zlib_checksum_verify_bad() {
CHECK_FOCUS(__func__);
do_test_wuffs_zlib_checksum(false, true);
do_test_wuffs_zlib_checksum(false, 1);
}
void test_wuffs_zlib_checksum_verify_good() {
CHECK_FOCUS(__func__);
do_test_wuffs_zlib_checksum(false, false);
do_test_wuffs_zlib_checksum(false, 0);
}
void test_wuffs_zlib_decode_midsummer() {