diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 99daeb75..689111ba 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -13942,6 +13942,11 @@ struct wuffs_xz__decoder__struct { uint32_t f_bcj_x86_prev_mask; uint64_t f_block_compressed_size; uint64_t f_block_uncompressed_size; + uint64_t f_compressed_size_for_index; + uint32_t f_verification_have_hashed_sizes[2]; + uint32_t f_verification_want_hashed_sizes[2]; + uint64_t f_verification_have_total_sizes[2]; + uint64_t f_verification_want_total_sizes[2]; uint64_t f_num_actual_blocks; uint64_t f_num_index_blocks; uint64_t f_index_block_compressed_size; @@ -67855,6 +67860,7 @@ wuffs_xz__decoder__do_transform_io( wuffs_base__bitvec256 v_checksum256_have = {0}; uint64_t v_compressed_size = 0; uint64_t v_uncompressed_size = 0; + uint32_t v_hash = 0; uint8_t v_c8 = 0; uint16_t v_footer_magic = 0; @@ -67992,6 +67998,7 @@ wuffs_xz__decoder__do_transform_io( wuffs_private_impl__ignore_status(wuffs_crc32__ieee_hasher__initialize(&self->private_data.f_crc32, sizeof (wuffs_crc32__ieee_hasher), WUFFS_VERSION, WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED)); } + self->private_impl.f_compressed_size_for_index = 4u; while (true) { v_smark = ((uint64_t)(iop_a_src - io0_a_src)); { @@ -68004,6 +68011,7 @@ wuffs_xz__decoder__do_transform_io( iop_a_src = a_src->data.ptr + a_src->meta.ri; } } + wuffs_private_impl__u64__sat_add_indirect(&self->private_impl.f_compressed_size_for_index, wuffs_private_impl__io__count_since(v_smark, ((uint64_t)(iop_a_src - io0_a_src)))); if ( ! self->private_impl.f_ignore_checksum) { v_checksum32_have = wuffs_crc32__ieee_hasher__update_u32(&self->private_data.f_crc32, wuffs_private_impl__io__since(v_smark, ((uint64_t)(iop_a_src - io0_a_src)), io0_a_src)); } @@ -68168,6 +68176,24 @@ wuffs_xz__decoder__do_transform_io( status = wuffs_base__make_status(wuffs_xz__error__bad_block_header); goto exit; } + wuffs_private_impl__u64__sat_add_indirect(&self->private_impl.f_compressed_size_for_index, v_compressed_size); + wuffs_private_impl__u64__sat_add_indirect(&self->private_impl.f_compressed_size_for_index, ((uint64_t)(WUFFS_XZ__CHECKSUM_LENGTH[self->private_impl.f_checksummer]))); + self->private_impl.f_verification_have_total_sizes[0u] += self->private_impl.f_compressed_size_for_index; + v_hash = ((uint32_t)((self->private_impl.f_compressed_size_for_index ^ (self->private_impl.f_compressed_size_for_index >> 32u)))); + v_hash *= 3432918353u; + v_hash = (((uint32_t)(v_hash << 15u)) | (v_hash >> 17u)); + v_hash *= 461845907u; + v_hash ^= self->private_impl.f_verification_have_hashed_sizes[0u]; + v_hash = (((uint32_t)(v_hash << 13u)) | (v_hash >> 19u)); + self->private_impl.f_verification_have_hashed_sizes[0u] = ((uint32_t)(((uint32_t)(v_hash * 5u)) + 3864292196u)); + self->private_impl.f_verification_have_total_sizes[1u] += v_uncompressed_size; + v_hash = ((uint32_t)((v_uncompressed_size ^ (v_uncompressed_size >> 32u)))); + v_hash *= 3432918353u; + v_hash = (((uint32_t)(v_hash << 15u)) | (v_hash >> 17u)); + v_hash *= 461845907u; + v_hash ^= self->private_impl.f_verification_have_hashed_sizes[1u]; + v_hash = (((uint32_t)(v_hash << 13u)) | (v_hash >> 19u)); + self->private_impl.f_verification_have_hashed_sizes[1u] = ((uint32_t)(((uint32_t)(v_hash * 5u)) + 3864292196u)); while ((v_compressed_size & 3u) != 0u) { { WUFFS_BASE__COROUTINE_SUSPENSION_POINT(11); @@ -69026,6 +69052,7 @@ wuffs_xz__decoder__verify_index( uint8_t v_c8 = 0; uint32_t v_shift = 0; + uint32_t v_hash = 0; const uint8_t* iop_a_src = NULL; const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; @@ -69077,12 +69104,12 @@ wuffs_xz__decoder__verify_index( v_shift += 7u; continue; } else if ((v_c8 == 0u) && (v_shift > 0u)) { - status = wuffs_base__make_status(wuffs_xz__error__bad_block_header); + status = wuffs_base__make_status(wuffs_xz__error__bad_index); goto exit; } break; } else if (v_c8 != 1u) { - status = wuffs_base__make_status(wuffs_xz__error__bad_block_header); + status = wuffs_base__make_status(wuffs_xz__error__bad_index); goto exit; } self->private_impl.f_num_index_blocks |= (((uint64_t)(1u)) << 63u); @@ -69113,12 +69140,12 @@ wuffs_xz__decoder__verify_index( v_shift += 7u; continue; } else if ((v_c8 == 0u) && (v_shift > 0u)) { - status = wuffs_base__make_status(wuffs_xz__error__bad_block_header); + status = wuffs_base__make_status(wuffs_xz__error__bad_index); goto exit; } break; } else if (v_c8 != 1u) { - status = wuffs_base__make_status(wuffs_xz__error__bad_block_header); + status = wuffs_base__make_status(wuffs_xz__error__bad_index); goto exit; } self->private_impl.f_index_block_compressed_size |= (((uint64_t)(1u)) << 63u); @@ -69142,17 +69169,40 @@ wuffs_xz__decoder__verify_index( v_shift += 7u; continue; } else if ((v_c8 == 0u) && (v_shift > 0u)) { - status = wuffs_base__make_status(wuffs_xz__error__bad_block_header); + status = wuffs_base__make_status(wuffs_xz__error__bad_index); goto exit; } break; } else if (v_c8 != 1u) { - status = wuffs_base__make_status(wuffs_xz__error__bad_block_header); + status = wuffs_base__make_status(wuffs_xz__error__bad_index); goto exit; } self->private_impl.f_index_block_uncompressed_size |= (((uint64_t)(1u)) << 63u); break; } + self->private_impl.f_verification_want_total_sizes[0u] += self->private_impl.f_index_block_compressed_size; + v_hash = ((uint32_t)((self->private_impl.f_index_block_compressed_size ^ (self->private_impl.f_index_block_compressed_size >> 32u)))); + v_hash *= 3432918353u; + v_hash = (((uint32_t)(v_hash << 15u)) | (v_hash >> 17u)); + v_hash *= 461845907u; + v_hash ^= self->private_impl.f_verification_want_hashed_sizes[0u]; + v_hash = (((uint32_t)(v_hash << 13u)) | (v_hash >> 19u)); + self->private_impl.f_verification_want_hashed_sizes[0u] = ((uint32_t)(((uint32_t)(v_hash * 5u)) + 3864292196u)); + self->private_impl.f_verification_want_total_sizes[1u] += self->private_impl.f_index_block_uncompressed_size; + v_hash = ((uint32_t)((self->private_impl.f_index_block_uncompressed_size ^ (self->private_impl.f_index_block_uncompressed_size >> 32u)))); + v_hash *= 3432918353u; + v_hash = (((uint32_t)(v_hash << 15u)) | (v_hash >> 17u)); + v_hash *= 461845907u; + v_hash ^= self->private_impl.f_verification_want_hashed_sizes[1u]; + v_hash = (((uint32_t)(v_hash << 13u)) | (v_hash >> 19u)); + self->private_impl.f_verification_want_hashed_sizes[1u] = ((uint32_t)(((uint32_t)(v_hash * 5u)) + 3864292196u)); + } + if ((self->private_impl.f_verification_have_hashed_sizes[0u] != self->private_impl.f_verification_want_hashed_sizes[0u]) || + (self->private_impl.f_verification_have_hashed_sizes[1u] != self->private_impl.f_verification_want_hashed_sizes[1u]) || + (self->private_impl.f_verification_have_total_sizes[0u] != self->private_impl.f_verification_want_total_sizes[0u]) || + (self->private_impl.f_verification_have_total_sizes[1u] != self->private_impl.f_verification_want_total_sizes[1u])) { + status = wuffs_base__make_status(wuffs_xz__error__bad_index); + goto exit; } goto ok; diff --git a/std/xz/decode_xz.wuffs b/std/xz/decode_xz.wuffs index 88af8838..4a523f7d 100644 --- a/std/xz/decode_xz.wuffs +++ b/std/xz/decode_xz.wuffs @@ -53,8 +53,28 @@ pub struct decoder? implements base.io_transformer( bcj_pos : base.u32, bcj_x86_prev_mask : base.u32, - block_compressed_size : base.u64, - block_uncompressed_size : base.u64, + block_compressed_size : base.u64, + block_uncompressed_size : base.u64, + compressed_size_for_index : base.u64, + + // Strictly speaking, we should verify that each block's compressed and + // uncompressed sizes (as it is encountered during decoding the stream) + // equals the per-block pair of sizes listed in the index at the end of + // the file. + // + // This requires O(N) memory, where N is the number of blocks. Instead + // (and the spec's "4.3. List of Records" suggests doing this), we can + // use O(1) memory by just comparing a hash (and a total) of the + // compressed and uncompressed sizes. + // + // This implementation uses a Murmur inspired hash function. It's given + // u64 values (not u8) and the final mixing step is not required. + // + // For the array indexes: 0 is compressed, 1 is uncompressed. + verification_have_hashed_sizes : array[2] base.u32, + verification_want_hashed_sizes : array[2] base.u32, + verification_have_total_sizes : array[2] base.u64, + verification_want_total_sizes : array[2] base.u64, num_actual_blocks : base.u64, num_index_blocks : base.u64, @@ -127,6 +147,7 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work var checksum256_have : base.bitvec256 var compressed_size : base.u64 var uncompressed_size : base.u64 + var hash : base.u32 var c8 : base.u8 var footer_magic : base.u16 @@ -172,9 +193,11 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work this.crc32.reset!() } + this.compressed_size_for_index = 4 // For the upcoming 4-byte CRC-32. while true { smark = args.src.mark() status =? this.decode_block_header_with_padding?(src: args.src) + this.compressed_size_for_index ~sat+= args.src.count_since(mark: smark) if not this.ignore_checksum { checksum32_have = this.crc32.update_u32!(x: args.src.since(mark: smark)) } @@ -259,6 +282,43 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work return "#bad block header" } + // A block's 'compressed size' (as measured here) needs to match what's + // listed in the index at the end of the XZ file. Per the XZ file + // format specification, a block is: + // + // +==============+=================+===============+=======+ + // | Block Header | Compressed Data | Block Padding | Check | + // +==============+=================+===============+=======+ + // + // The compressed_size variable only measures the "Compressed Data" + // portion. The 'compressed size for indexing purposes' needs to add + // the size of the Block Header and the Check (but not the Block + // Padding). this.compressed_size_for_index was initialized and + // accumulated above to, at this point in time, hold the size of the + // Block Header. We need to add the other two parts. + this.compressed_size_for_index ~sat+= compressed_size + this.compressed_size_for_index ~sat+= CHECKSUM_LENGTH[this.checksummer] as base.u64 + + // Update the verification_have_etc values. The hash function is + // loosely based on https://en.wikipedia.org/wiki/MurmurHash#Algorithm + this.verification_have_total_sizes[0] ~mod+= this.compressed_size_for_index + hash = ((this.compressed_size_for_index ^ (this.compressed_size_for_index >> 32)) & 0xFFFF_FFFF) as base.u32 + hash ~mod*= 0xCC9E_2D51 + hash = (hash ~mod<< 15) | (hash >> 17) + hash ~mod*= 0x1B87_3593 + hash ^= this.verification_have_hashed_sizes[0] + hash = (hash ~mod<< 13) | (hash >> 19) + this.verification_have_hashed_sizes[0] = (hash ~mod* 5) ~mod+ 0xE654_6B64 + this.verification_have_total_sizes[1] ~mod+= uncompressed_size + hash = ((uncompressed_size ^ (uncompressed_size >> 32)) & 0xFFFF_FFFF) as base.u32 + hash ~mod*= 0xCC9E_2D51 + hash = (hash ~mod<< 15) | (hash >> 17) + hash ~mod*= 0x1B87_3593 + hash ^= this.verification_have_hashed_sizes[1] + hash = (hash ~mod<< 13) | (hash >> 19) + this.verification_have_hashed_sizes[1] = (hash ~mod* 5) ~mod+ 0xE654_6B64 + + // Consume the "Block Padding". while (compressed_size & 3) <> 0 { c8 = args.src.read_u8?() if c8 <> 0x00 { @@ -563,6 +623,7 @@ pri func decoder.decode_block_header_sans_padding?(src: base.io_reader) { pri func decoder.verify_index?(src: base.io_reader) { var c8 : base.u8 var shift : base.u32 + var hash : base.u32 if not this.started_verify_index { this.started_verify_index = true @@ -580,11 +641,11 @@ pri func decoder.verify_index?(src: base.io_reader) { shift += 7 continue } else if (c8 == 0x00) and (shift > 0) { - return "#bad block header" + return "#bad index" } break } else if c8 <> 1 { - return "#bad block header" + return "#bad index" } this.num_index_blocks |= (1 as base.u64) << 63 break @@ -607,11 +668,11 @@ pri func decoder.verify_index?(src: base.io_reader) { shift += 7 continue } else if (c8 == 0x00) and (shift > 0) { - return "#bad block header" + return "#bad index" } break } else if c8 <> 1 { - return "#bad block header" + return "#bad index" } this.index_block_compressed_size |= (1 as base.u64) << 63 break @@ -627,16 +688,42 @@ pri func decoder.verify_index?(src: base.io_reader) { shift += 7 continue } else if (c8 == 0x00) and (shift > 0) { - return "#bad block header" + return "#bad index" } break } else if c8 <> 1 { - return "#bad block header" + return "#bad index" } this.index_block_uncompressed_size |= (1 as base.u64) << 63 break } endwhile + + // Update the verification_want_etc values. The hash function is + // loosely based on https://en.wikipedia.org/wiki/MurmurHash#Algorithm + this.verification_want_total_sizes[0] ~mod+= this.index_block_compressed_size + hash = ((this.index_block_compressed_size ^ (this.index_block_compressed_size >> 32)) & 0xFFFF_FFFF) as base.u32 + hash ~mod*= 0xCC9E_2D51 + hash = (hash ~mod<< 15) | (hash >> 17) + hash ~mod*= 0x1B87_3593 + hash ^= this.verification_want_hashed_sizes[0] + hash = (hash ~mod<< 13) | (hash >> 19) + this.verification_want_hashed_sizes[0] = (hash ~mod* 5) ~mod+ 0xE654_6B64 + this.verification_want_total_sizes[1] ~mod+= this.index_block_uncompressed_size + hash = ((this.index_block_uncompressed_size ^ (this.index_block_uncompressed_size >> 32)) & 0xFFFF_FFFF) as base.u32 + hash ~mod*= 0xCC9E_2D51 + hash = (hash ~mod<< 15) | (hash >> 17) + hash ~mod*= 0x1B87_3593 + hash ^= this.verification_want_hashed_sizes[1] + hash = (hash ~mod<< 13) | (hash >> 19) + this.verification_want_hashed_sizes[1] = (hash ~mod* 5) ~mod+ 0xE654_6B64 } endwhile + + if (this.verification_have_hashed_sizes[0] <> this.verification_want_hashed_sizes[0]) or + (this.verification_have_hashed_sizes[1] <> this.verification_want_hashed_sizes[1]) or + (this.verification_have_total_sizes[0] <> this.verification_want_total_sizes[0]) or + (this.verification_have_total_sizes[1] <> this.verification_want_total_sizes[1]) { + return "#bad index" + } } pri func decoder.verify_footer?(src: base.io_reader) { diff --git a/test/3pdata/mzcat-checksums-of-xzsuite.txt b/test/3pdata/mzcat-checksums-of-xzsuite.txt index 0b99159f..773b19e9 100644 --- a/test/3pdata/mzcat-checksums-of-xzsuite.txt +++ b/test/3pdata/mzcat-checksums-of-xzsuite.txt @@ -23,12 +23,12 @@ BAD 15a2a343 test/3pdata/xzsuite/bad-1-stream_flags-1.xz BAD 15a2a343 test/3pdata/xzsuite/bad-1-stream_flags-3.xz BAD 15a2a343 test/3pdata/xzsuite/bad-1-vli-2.xz BAD 31963516 test/3pdata/xzsuite/bad-2-compressed_data_padding.xz -OK. 15a2a343 test/3pdata/xzsuite/bad-2-index-1.xz -OK. 15a2a343 test/3pdata/xzsuite/bad-2-index-2.xz +BAD 15a2a343 test/3pdata/xzsuite/bad-2-index-1.xz +BAD 15a2a343 test/3pdata/xzsuite/bad-2-index-2.xz BAD 15a2a343 test/3pdata/xzsuite/bad-2-index-3.xz BAD 15a2a343 test/3pdata/xzsuite/bad-2-index-4.xz -OK. 15a2a343 test/3pdata/xzsuite/bad-2-index-5.xz -OK. 15a2a343 test/3pdata/xzsuite/bad-3-index-uncomp-overflow.xz +BAD 15a2a343 test/3pdata/xzsuite/bad-2-index-5.xz +BAD 15a2a343 test/3pdata/xzsuite/bad-3-index-uncomp-overflow.xz BAD 15a2a343 test/3pdata/xzsuite/bad-too_big_size-with_eopm.lzma BAD 77e47a71 test/3pdata/xzsuite/bad-too_small_size-without_eopm-1.lzma BAD dd46b7fb test/3pdata/xzsuite/bad-too_small_size-without_eopm-2.lzma