From d2a63a7e3436b636ab47eff5830aaaadb2cdfde3 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sun, 11 Feb 2024 12:59:29 +1100 Subject: [PATCH] std/xz: implement apply_filter_04_x86 --- release/c/wuffs-unsupported-snapshot.c | 118 +++++++++++++++++++- std/xz/decode_filter.wuffs | 120 +++++++++++++++++++++ std/xz/decode_xz.wuffs | 10 +- test/3pdata/mzcat-checksums-of-xzsuite.txt | 1 + test/data/artificial-xz-filter/make.go | 3 + 5 files changed, 248 insertions(+), 4 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 8f3059e3..66821184 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -13925,6 +13925,7 @@ struct wuffs_xz__decoder__struct { bool f_block_has_uncompressed_size; uint8_t f_bcj_undo_index; uint32_t f_bcj_pos; + uint32_t f_bcj_x86_prev_mask; uint64_t f_block_compressed_size; uint64_t f_block_uncompressed_size; @@ -66758,6 +66759,21 @@ const char wuffs_xz__error__internal_error_inconsistent_bcj_filter_state[] = "#x // ---------------- Private Consts +static const bool +WUFFS_XZ__FILTER_04_X86_MASK_TO_ALLOWED_STATUS[8] WUFFS_BASE__POTENTIALLY_UNUSED = { + 1u, 1u, 1u, 0u, 1u, 0u, 0u, 0u, +}; + +static const uint8_t +WUFFS_XZ__FILTER_04_X86_MASK_TO_BIT_NUM[8] WUFFS_BASE__POTENTIALLY_UNUSED = { + 0u, 1u, 2u, 2u, 3u, 3u, 3u, 3u, +}; + +static const uint32_t +WUFFS_XZ__FILTER_04_X86_MASK_TO_XOR_OPERAND[8] WUFFS_BASE__POTENTIALLY_UNUSED = { + 4294967295u, 16777215u, 65535u, 65535u, 255u, 255u, 255u, 255u, +}; + static const uint8_t WUFFS_XZ__CHECKSUM_LENGTH[4] WUFFS_BASE__POTENTIALLY_UNUSED = { 0u, 4u, 8u, 32u, @@ -66779,6 +66795,12 @@ wuffs_xz__decoder__apply_non_final_filters__choosy_default( wuffs_xz__decoder* self, wuffs_base__slice_u8 a_dst_slice); +WUFFS_BASE__GENERATED_C_CODE +static uint8_t +wuffs_xz__decoder__apply_filter_04_x86( + wuffs_xz__decoder* self, + wuffs_base__slice_u8 a_dst_slice); + WUFFS_BASE__GENERATED_C_CODE static uint8_t wuffs_xz__decoder__apply_filter_07_arm( @@ -66999,6 +67021,96 @@ wuffs_xz__decoder__apply_non_final_filters__choosy_default( return 0u; } +// -------- func xz.decoder.apply_filter_04_x86 + +WUFFS_BASE__GENERATED_C_CODE +static uint8_t +wuffs_xz__decoder__apply_filter_04_x86( + wuffs_xz__decoder* self, + wuffs_base__slice_u8 a_dst_slice) { + wuffs_base__slice_u8 v_s = {0}; + uint32_t v_p = 0; + uint64_t v_i = 0; + uint64_t v_prev_pos = 0; + uint32_t v_prev_mask = 0; + uint8_t v_c8 = 0; + uint32_t v_src = 0; + uint32_t v_dst = 0; + uint32_t v_bit_num = 0; + + v_s = a_dst_slice; + v_p = ((uint32_t)(self->private_impl.f_bcj_pos + 5u)); + v_prev_pos = 18446744073709551615u; + v_prev_mask = self->private_impl.f_bcj_x86_prev_mask; + while (((uint64_t)(v_s.len)) >= 5u) { + if (((uint8_t)(v_s.ptr[0u] & 254u)) != 232u) { + v_i += 1u; + v_p += 1u; + v_s = wuffs_base__slice_u8__subslice_i(v_s, 1u); + continue; + } + v_prev_pos = ((uint64_t)(v_i - v_prev_pos)); + if (v_prev_pos > 3u) { + v_prev_mask = 0u; + } else if (v_prev_pos > 0u) { + v_prev_mask = (((uint32_t)(v_prev_mask << (v_prev_pos - 1u))) & 7u); + if (v_prev_mask != 0u) { + v_c8 = v_s.ptr[((uint8_t)(4u - WUFFS_XZ__FILTER_04_X86_MASK_TO_BIT_NUM[(v_prev_mask & 7u)]))]; + if ( ! WUFFS_XZ__FILTER_04_X86_MASK_TO_ALLOWED_STATUS[(v_prev_mask & 7u)] || (v_c8 == 0u) || (v_c8 == 255u)) { + v_prev_pos = v_i; + v_prev_mask = (((uint32_t)(v_prev_mask << 1u)) | 1u); + v_i += 1u; + v_p += 1u; + v_s = wuffs_base__slice_u8__subslice_i(v_s, 1u); + continue; + } + } + } + v_prev_pos = v_i; + v_c8 = v_s.ptr[4u]; + if ((v_c8 != 0u) && (v_c8 != 255u)) { + v_prev_mask = (((uint32_t)(v_prev_mask << 1u)) | 1u); + v_i += 1u; + v_p += 1u; + v_s = wuffs_base__slice_u8__subslice_i(v_s, 1u); + continue; + } + v_src = ((((uint32_t)(v_s.ptr[1u])) << 0u) | + (((uint32_t)(v_s.ptr[2u])) << 8u) | + (((uint32_t)(v_s.ptr[3u])) << 16u) | + (((uint32_t)(v_s.ptr[4u])) << 24u)); + while (true) { + v_dst = ((uint32_t)(v_src - v_p)); + if (v_prev_mask == 0u) { + break; + } + v_bit_num = ((uint32_t)(WUFFS_XZ__FILTER_04_X86_MASK_TO_BIT_NUM[(v_prev_mask & 7u)])); + v_c8 = ((uint8_t)((v_dst >> (24u - (v_bit_num * 8u))))); + if ((v_c8 != 0u) && (v_c8 != 255u)) { + break; + } + v_src = (v_dst ^ WUFFS_XZ__FILTER_04_X86_MASK_TO_XOR_OPERAND[(v_prev_mask & 7u)]); + } + v_dst &= 33554431u; + v_dst |= ((uint32_t)(0u - (v_dst & 16777216u))); + v_s.ptr[1u] = ((uint8_t)((v_dst >> 0u))); + v_s.ptr[2u] = ((uint8_t)((v_dst >> 8u))); + v_s.ptr[3u] = ((uint8_t)((v_dst >> 16u))); + v_s.ptr[4u] = ((uint8_t)((v_dst >> 24u))); + v_i += 5u; + v_p += 5u; + v_s = wuffs_base__slice_u8__subslice_i(v_s, 5u); + } + v_prev_pos = ((uint64_t)(v_i - v_prev_pos)); + if (v_prev_pos > 3u) { + self->private_impl.f_bcj_x86_prev_mask = 0u; + } else if (v_prev_pos > 0u) { + self->private_impl.f_bcj_x86_prev_mask = ((uint32_t)(v_prev_mask << (v_prev_pos - 1u))); + } + self->private_impl.f_bcj_pos = ((uint32_t)(v_p - 5u)); + return ((uint8_t)(((uint64_t)(v_s.len)))); +} + // -------- func xz.decoder.apply_filter_07_arm WUFFS_BASE__GENERATED_C_CODE @@ -68102,6 +68214,7 @@ wuffs_xz__decoder__decode_block_header_sans_padding( } } self->private_impl.f_bcj_pos = 0u; + self->private_impl.f_bcj_x86_prev_mask = 0u; self->private_impl.choosy_apply_non_final_filters = ( &wuffs_xz__decoder__apply_non_final_filters__choosy_default); v_f = 0u; @@ -68155,7 +68268,10 @@ wuffs_xz__decoder__decode_block_header_sans_padding( goto exit; } else { self->private_impl.f_filters[v_f] = ((uint32_t)(v_filter_id)); - if (v_filter_id == 7u) { + if (v_filter_id == 4u) { + self->private_impl.choosy_apply_non_final_filters = ( + &wuffs_xz__decoder__apply_filter_04_x86); + } else if (v_filter_id == 7u) { self->private_impl.choosy_apply_non_final_filters = ( &wuffs_xz__decoder__apply_filter_07_arm); } else if (v_filter_id == 8u) { diff --git a/std/xz/decode_filter.wuffs b/std/xz/decode_filter.wuffs index 476362f0..5d3cbbfa 100644 --- a/std/xz/decode_filter.wuffs +++ b/std/xz/decode_filter.wuffs @@ -50,6 +50,126 @@ pri func decoder.apply_non_final_filters!(dst_slice: slice base.u8) base.u8, return 0 } +pri const FILTER_04_X86_MASK_TO_ALLOWED_STATUS : roarray[8] base.bool = [ + true, true, true, false, true, false, false, false, +] + +pri const FILTER_04_X86_MASK_TO_BIT_NUM : roarray[8] base.u8[..= 3] = [ + 0, 1, 2, 2, 3, 3, 3, 3, +] + +pri const FILTER_04_X86_MASK_TO_XOR_OPERAND : roarray[8] base.u32 = [ + 0xFFFF_FFFF, + 0x00FF_FFFF, + 0x0000_FFFF, + 0x0000_FFFF, + 0x0000_00FF, + 0x0000_00FF, + 0x0000_00FF, + 0x0000_00FF, +] + +pri func decoder.apply_filter_04_x86!(dst_slice: slice base.u8) base.u8 { + var s : slice base.u8 + var p : base.u32 + + var i : base.u64 + var prev_pos : base.u64 + var prev_mask : base.u32 + var c8 : base.u8 + var src : base.u32 + var dst : base.u32 + var bit_num : base.u32[..= 3] + + s = args.dst_slice + p = this.bcj_pos ~mod+ 5 + + prev_pos = 0xFFFF_FFFF_FFFF_FFFF + prev_mask = this.bcj_x86_prev_mask + + while s.length() >= 5, + post s.length() < 5, + { + if (s[0] & 0xFE) <> 0xE8 { + i ~mod+= 1 + p ~mod+= 1 + s = s[1 ..] + continue + } + + prev_pos = i ~mod- prev_pos + if prev_pos > 3 { + prev_mask = 0 + } else if prev_pos > 0 { + prev_mask = (prev_mask ~mod<< (prev_pos - 1)) & 7 + if prev_mask <> 0 { + c8 = s[4 - FILTER_04_X86_MASK_TO_BIT_NUM[prev_mask & 7]] + if (not FILTER_04_X86_MASK_TO_ALLOWED_STATUS[prev_mask & 7]) or + (c8 == 0x00) or (c8 == 0xFF) { + prev_pos = i + prev_mask = (prev_mask ~mod<< 1) | 1 + i ~mod+= 1 + p ~mod+= 1 + s = s[1 ..] + continue + } + } + } + + prev_pos = i + + c8 = s[4] + if (c8 <> 0x00) and (c8 <> 0xFF) { + prev_mask = (prev_mask ~mod<< 1) | 1 + i ~mod+= 1 + p ~mod+= 1 + s = s[1 ..] + continue + } + + src = ((s[1] as base.u32) << 0x00) | + ((s[2] as base.u32) << 0x08) | + ((s[3] as base.u32) << 0x10) | + ((s[4] as base.u32) << 0x18) + + while true, + inv s.length() >= 5, + { + dst = src ~mod- p + if prev_mask == 0 { + break + } + bit_num = FILTER_04_X86_MASK_TO_BIT_NUM[prev_mask & 7] as base.u32 + c8 = ((dst >> (24 - (bit_num * 8))) & 0xFF) as base.u8 + if (c8 <> 0x00) and (c8 <> 0xFF) { + break + } + src = dst ^ FILTER_04_X86_MASK_TO_XOR_OPERAND[prev_mask & 7] + } endwhile + + dst &= 0x01FF_FFFF + dst |= 0 ~mod- (dst & 0x0100_0000) + s[1] = ((dst >> 0x00) & 0xFF) as base.u8 + s[2] = ((dst >> 0x08) & 0xFF) as base.u8 + s[3] = ((dst >> 0x10) & 0xFF) as base.u8 + s[4] = ((dst >> 0x18) & 0xFF) as base.u8 + + i ~mod+= 5 + p ~mod+= 5 + s = s[5 ..] + } endwhile + + prev_pos = i ~mod- prev_pos + if prev_pos > 3 { + this.bcj_x86_prev_mask = 0 + } else if prev_pos > 0 { + this.bcj_x86_prev_mask = prev_mask ~mod<< (prev_pos - 1) + } + + this.bcj_pos = p ~mod- 5 + return s.length() as base.u8 +} + pri func decoder.apply_filter_07_arm!(dst_slice: slice base.u8) base.u8 { var s : slice base.u8 var p : base.u32 diff --git a/std/xz/decode_xz.wuffs b/std/xz/decode_xz.wuffs index 48595ff6..5887fbb7 100644 --- a/std/xz/decode_xz.wuffs +++ b/std/xz/decode_xz.wuffs @@ -47,8 +47,9 @@ pub struct decoder? implements base.io_transformer( block_has_compressed_size : base.bool, block_has_uncompressed_size : base.bool, - bcj_undo_index : base.u8, - bcj_pos : base.u32, + bcj_undo_index : base.u8, + bcj_pos : base.u32, + bcj_x86_prev_mask : base.u32, block_compressed_size : base.u64, block_uncompressed_size : base.u64, @@ -386,6 +387,7 @@ pri func decoder.decode_block_header_sans_padding?(src: base.io_reader) { // Configure the non-final filters. this.bcj_pos = 0 + this.bcj_x86_prev_mask = 0 choose apply_non_final_filters = [apply_non_final_filters] f = 0 while f < this.num_non_final_filters { @@ -421,7 +423,9 @@ pri func decoder.decode_block_header_sans_padding?(src: base.io_reader) { return "#unsupported filter combination" } else { this.filters[f] = filter_id as base.u32 - if filter_id == 0x07 { + if filter_id == 0x04 { + choose apply_non_final_filters = [apply_filter_04_x86] + } else if filter_id == 0x07 { choose apply_non_final_filters = [apply_filter_07_arm] } else if filter_id == 0x08 { choose apply_non_final_filters = [apply_filter_08_armthumb] diff --git a/test/3pdata/mzcat-checksums-of-xzsuite.txt b/test/3pdata/mzcat-checksums-of-xzsuite.txt index e03efb03..3af4cef4 100644 --- a/test/3pdata/mzcat-checksums-of-xzsuite.txt +++ b/test/3pdata/mzcat-checksums-of-xzsuite.txt @@ -61,6 +61,7 @@ OK. db4847fb test/3pdata/xzsuite/good-1-lzma2-3.xz OK. db4847fb test/3pdata/xzsuite/good-1-lzma2-4.xz OK. 00000000 test/3pdata/xzsuite/good-1-lzma2-5.xz OK. cc270da1 test/3pdata/xzsuite/good-1-sparc-lzma2.xz +OK. a3679091 test/3pdata/xzsuite/good-1-x86-lzma2.xz OK. 15a2a343 test/3pdata/xzsuite/good-2-lzma2.xz BAD 15a2a343 test/3pdata/xzsuite/good-known_size-with_eopm.lzma OK. 15a2a343 test/3pdata/xzsuite/good-known_size-without_eopm.lzma diff --git a/test/data/artificial-xz-filter/make.go b/test/data/artificial-xz-filter/make.go index 04cf8201..2a6a6b19 100644 --- a/test/data/artificial-xz-filter/make.go +++ b/test/data/artificial-xz-filter/make.go @@ -35,6 +35,9 @@ func main1() error { filterName string generator func() []byte }{ + // No test case generated for 04/x86, as the x86 CPU bytecode format is + // complicated (variable length instructions). Test coverage is instead + // provided by xz-tests-files's good-1-x86-lzma2.xz file. {"07", "arm", genArm}, {"08", "armthumb", genArmthumb}, {"09", "sparc", genSparc},