From 13c72db3508d33b9416a22a0ab8a8d4d8d5cd7be Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Fri, 1 Mar 2024 21:06:56 +1100 Subject: [PATCH] std/lzma: add QUIRK_ALLOW_NON_ZERO_INITIAL_BYTE --- release/c/wuffs-unsupported-snapshot.c | 18 ++++++++++++++---- std/lzip/decode_lzip.wuffs | 6 ++++++ std/lzma/decode_lzma.wuffs | 16 +++++++++++++--- std/lzma/decode_quirks.wuffs | 8 +++++++- test/3pdata/mzcat-checksums-of-xzsuite.txt | 2 +- 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index fc2efa17..c707ccb9 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -10353,7 +10353,9 @@ extern const char wuffs_lzma__error__unsupported_properties[]; #define WUFFS_LZMA__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE 4294967568u -#define WUFFS_LZMA__QUIRK_FORMAT_EXTENSION 1348001792u +#define WUFFS_LZMA__QUIRK_ALLOW_NON_ZERO_INITIAL_BYTE 1348001792u + +#define WUFFS_LZMA__QUIRK_FORMAT_EXTENSION 1348001793u // ---------------- Struct Declarations @@ -10479,6 +10481,7 @@ struct wuffs_lzma__decoder__struct { bool f_lzma2_need_properties; bool f_lzma2_need_dict_reset; bool f_prev_lzma2_chunk_was_uncompressed; + bool f_allow_non_zero_initial_byte; uint8_t f_stashed_bytes[2]; uint32_t f_stashed_bits; uint32_t f_stashed_range; @@ -52530,6 +52533,10 @@ wuffs_lzma__decoder__get_quirk( } if (a_key == 1348001792u) { + if (self->private_impl.f_allow_non_zero_initial_byte) { + return 1u; + } + } else if (a_key == 1348001793u) { return ((uint64_t)(self->private_impl.f_format_extension)); } return 0u; @@ -52557,6 +52564,8 @@ wuffs_lzma__decoder__set_quirk( uint32_t v_n = 0; if (a_key == 1348001792u) { + self->private_impl.f_allow_non_zero_initial_byte = (a_value > 0u); + } else if (a_key == 1348001793u) { if (a_value == 0u) { self->private_impl.f_format_extension = 0u; return wuffs_base__make_status(NULL); @@ -53105,7 +53114,7 @@ wuffs_lzma__decoder__do_transform_io( uint8_t t_8 = *iop_a_src++; v_c8 = t_8; } - if (v_c8 != 0u) { + if ((v_c8 != 0u) && ! self->private_impl.f_allow_non_zero_initial_byte) { status = wuffs_base__make_status(wuffs_lzma__error__bad_code); goto exit; } @@ -53927,7 +53936,7 @@ wuffs_lzip__decoder__do_transform_io( uint8_t t_1 = *iop_a_src++; v_c8 = t_1; } - v_status = wuffs_lzma__decoder__set_quirk(&self->private_data.f_lzma, 1348001792u, (1u | (((uint64_t)(v_c8)) << 8u))); + v_status = wuffs_lzma__decoder__set_quirk(&self->private_data.f_lzma, 1348001793u, (1u | (((uint64_t)(v_c8)) << 8u))); if ( ! wuffs_base__status__is_ok(&v_status)) { if (v_status.repr == wuffs_base__error__bad_argument) { status = wuffs_base__make_status(wuffs_lzip__error__bad_header); @@ -53944,6 +53953,7 @@ wuffs_lzip__decoder__do_transform_io( } self->private_impl.f_ssize_have = 0u; self->private_impl.f_dsize_have = 0u; + wuffs_lzma__decoder__set_quirk(&self->private_data.f_lzma, 1348001792u, 1u); while (true) { v_dmark = ((uint64_t)(iop_a_dst - io0_a_dst)); v_smark = ((uint64_t)(iop_a_src - io0_a_src)); @@ -69991,7 +70001,7 @@ wuffs_xz__decoder__decode_block_header_sans_padding( uint8_t t_10 = *iop_a_src++; v_c8 = t_10; } - v_status = wuffs_lzma__decoder__set_quirk(&self->private_data.f_lzma, 1348001792u, (2u | (((uint64_t)(v_c8)) << 8u))); + v_status = wuffs_lzma__decoder__set_quirk(&self->private_data.f_lzma, 1348001793u, (2u | (((uint64_t)(v_c8)) << 8u))); if ( ! wuffs_base__status__is_ok(&v_status)) { status = wuffs_base__make_status(wuffs_xz__error__bad_filter); goto exit; diff --git a/std/lzip/decode_lzip.wuffs b/std/lzip/decode_lzip.wuffs index 8c5b4898..ce797c95 100644 --- a/std/lzip/decode_lzip.wuffs +++ b/std/lzip/decode_lzip.wuffs @@ -106,8 +106,14 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work } // Decode LZMA. + // + // Re lzma.QUIRK_ALLOW_NON_ZERO_INITIAL_BYTE, as exercised by + // test/3pdata/xzsuite/lzip-testsuite/fox6_mark.lz, the LZMA spec says + // that the decoder must check that the byte is zero, but the Lzip spec + // says that the byte "must be ignored by the range decoder". this.ssize_have = 0 this.dsize_have = 0 + this.lzma.set_quirk!(key: lzma.QUIRK_ALLOW_NON_ZERO_INITIAL_BYTE, value: 1) while true { dmark = args.dst.mark() smark = args.src.mark() diff --git a/std/lzma/decode_lzma.wuffs b/std/lzma/decode_lzma.wuffs index cb085fb5..4b346842 100644 --- a/std/lzma/decode_lzma.wuffs +++ b/std/lzma/decode_lzma.wuffs @@ -60,6 +60,8 @@ pub struct decoder? implements base.io_transformer( prev_lzma2_chunk_was_uncompressed : base.bool, + allow_non_zero_initial_byte : base.bool, + stashed_bytes : array[2] base.u8, // 0 is prev_byte, 1 is match_byte. stashed_bits : base.u32, stashed_range : base.u32, @@ -265,7 +267,12 @@ pri func decoder.add_history!(hist: slice base.u8, workbuf: slice base.u8) base. } pub func decoder.get_quirk(key: base.u32) base.u64 { - if args.key == QUIRK_FORMAT_EXTENSION { + if args.key == QUIRK_ALLOW_NON_ZERO_INITIAL_BYTE { + if this.allow_non_zero_initial_byte { + return 1 + } + + } else if args.key == QUIRK_FORMAT_EXTENSION { return this.format_extension as base.u64 } return 0 @@ -275,7 +282,10 @@ pub func decoder.set_quirk!(key: base.u32, value: base.u64) base.status { var v : base.u32 var n : base.u32 - if args.key == QUIRK_FORMAT_EXTENSION { + if args.key == QUIRK_ALLOW_NON_ZERO_INITIAL_BYTE { + this.allow_non_zero_initial_byte = args.value > 0 + + } else if args.key == QUIRK_FORMAT_EXTENSION { if args.value == 0 { this.format_extension = 0 return ok @@ -493,7 +503,7 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work } c8 = args.src.read_u8?() - if c8 <> 0x00 { + if (c8 <> 0x00) and not this.allow_non_zero_initial_byte { return "#bad code" } this.stashed_bits = args.src.read_u32be?() diff --git a/std/lzma/decode_quirks.wuffs b/std/lzma/decode_quirks.wuffs index 2c7a1105..78140d36 100644 --- a/std/lzma/decode_quirks.wuffs +++ b/std/lzma/decode_quirks.wuffs @@ -18,6 +18,12 @@ pri const QUIRKS_BASE : base.u32 = 0x5058_E000 // -------- +// When this quirk is set, a positive or zero value ignores or enforces the "If +// initial byte [after properties or other headers] is not equal to ZERO, the +// LZMA Decoder must stop decoding and report error" from the spec +// https://raw.githubusercontent.com/jljusten/LZMA-SDK/781863cdf592da3e97420f50de5dac056ad352a5/DOC/lzma-specification.txt +pub const QUIRK_ALLOW_NON_ZERO_INITIAL_BYTE : base.u32 = 0x5058_E000 | 0x00 + // When this quirk is set, a positive value indicates an extension to the LZMA // format. Zero means to use the default LZMA format. // @@ -33,4 +39,4 @@ pri const QUIRKS_BASE : base.u32 = 0x5058_E000 // For LZMA2, the high 7 bytes (shifted right by 8) must be less than or equal // to 40 and indicates the dictionary size. See section "5.3.1. LZMA2" of // https://tukaani.org/xz/xz-file-format.txt -pub const QUIRK_FORMAT_EXTENSION : base.u32 = 0x5058_E000 | 0x00 +pub const QUIRK_FORMAT_EXTENSION : base.u32 = 0x5058_E000 | 0x01 diff --git a/test/3pdata/mzcat-checksums-of-xzsuite.txt b/test/3pdata/mzcat-checksums-of-xzsuite.txt index e70f4e4e..ebac0d20 100644 --- a/test/3pdata/mzcat-checksums-of-xzsuite.txt +++ b/test/3pdata/mzcat-checksums-of-xzsuite.txt @@ -70,7 +70,7 @@ OK. 15a2a343 test/3pdata/xzsuite/good-known_size-without_eopm.lzma OK. 15a2a343 test/3pdata/xzsuite/good-unknown_size-with_eopm.lzma OK. eb50cc6a test/3pdata/xzsuite/lzip-testsuite/fox.lz OK. 889dda54 test/3pdata/xzsuite/lzip-testsuite/fox6.lz -BAD 340878c7 test/3pdata/xzsuite/lzip-testsuite/fox6_mark.lz +OK. 889dda54 test/3pdata/xzsuite/lzip-testsuite/fox6_mark.lz BAD eb50cc6a test/3pdata/xzsuite/lzip-testsuite/fox_bcrc.lz BAD eb50cc6a test/3pdata/xzsuite/lzip-testsuite/fox_crc0.lz BAD eb50cc6a test/3pdata/xzsuite/lzip-testsuite/fox_das46.lz