From 22b4ff4842365e3dd9090e6477cf960f32227a4a Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Mon, 26 Feb 2024 22:30:23 +1100 Subject: [PATCH] std/xz: decode concatenated streams --- doc/note/quirks.md | 2 + example/mzcat/mzcat.c | 3 ++ release/c/wuffs-unsupported-snapshot.c | 54 ++++++++++++++++++++-- std/xz/decode_quirks.wuffs | 40 ++++++++++++++++ std/xz/decode_xz.wuffs | 49 ++++++++++++++++++-- test/3pdata/mzcat-checksums-of-xzsuite.txt | 4 -- test/mzcat-checksums-of-data.txt | 2 +- 7 files changed, 141 insertions(+), 13 deletions(-) create mode 100644 std/xz/decode_quirks.wuffs diff --git a/doc/note/quirks.md b/doc/note/quirks.md index 193f3bdf..7ff1b263 100644 --- a/doc/note/quirks.md +++ b/doc/note/quirks.md @@ -67,5 +67,7 @@ Package-specific quirks: - [GIF image decoder quirks](/std/gif/decode_quirks.wuffs) - [JSON decoder quirks](/std/json/decode_quirks.wuffs) +- [LZMA decoder quirks](/std/lzma/decode_quirks.wuffs) - [LZW decoder quirks](/std/lzw/decode_quirks.wuffs) +- [XZ decoder quirks](/std/xz/decode_quirks.wuffs) - [ZLIB decoder quirks](/std/zlib/decode_quirks.wuffs) diff --git a/example/mzcat/mzcat.c b/example/mzcat/mzcat.c index 30b002f3..3fce5043 100644 --- a/example/mzcat/mzcat.c +++ b/example/mzcat/mzcat.c @@ -221,6 +221,9 @@ initialize_io_transformer(uint8_t input_first_byte) { WUFFS_VERSION, WUFFS_INITIALIZE__DEFAULT_OPTIONS); io_transformer = wuffs_xz__decoder__upcast_as__wuffs_base__io_transformer( &g_potential_decoders.xz); + wuffs_base__io_transformer__set_quirk( + io_transformer, + WUFFS_XZ__QUIRK_DECODE_STANDALONE_CONCATENATED_STREAMS, 1); break; } diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 6a33eea2..defd6231 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -13808,6 +13808,7 @@ extern const char wuffs_xz__error__bad_checksum[]; extern const char wuffs_xz__error__bad_filter[]; extern const char wuffs_xz__error__bad_footer[]; extern const char wuffs_xz__error__bad_header[]; +extern const char wuffs_xz__error__bad_header_concatenated_stream[]; extern const char wuffs_xz__error__bad_index[]; extern const char wuffs_xz__error__bad_padding[]; extern const char wuffs_xz__error__truncated_input[]; @@ -13817,6 +13818,8 @@ extern const char wuffs_xz__error__unsupported_filter_combination[]; // ---------------- Public Consts +#define WUFFS_XZ__QUIRK_DECODE_STANDALONE_CONCATENATED_STREAMS 2021322752u + #define WUFFS_XZ__DECODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE 0u #define WUFFS_XZ__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE 4294967568u @@ -13935,6 +13938,7 @@ struct wuffs_xz__decoder__struct { uint32_t f_num_non_final_filters; uint8_t f_checksummer; bool f_ignore_checksum; + bool f_standalone_format; bool f_lzma_needs_reset; bool f_block_has_compressed_size; bool f_block_has_uncompressed_size; @@ -66941,6 +66945,7 @@ const char wuffs_xz__error__bad_checksum[] = "#xz: bad checksum"; const char wuffs_xz__error__bad_filter[] = "#xz: bad filter"; const char wuffs_xz__error__bad_footer[] = "#xz: bad footer"; const char wuffs_xz__error__bad_header[] = "#xz: bad header"; +const char wuffs_xz__error__bad_header_concatenated_stream[] = "#xz: bad header (concatenated stream)"; const char wuffs_xz__error__bad_index[] = "#xz: bad index"; const char wuffs_xz__error__bad_padding[] = "#xz: bad padding"; const char wuffs_xz__error__truncated_input[] = "#xz: truncated input"; @@ -66974,6 +66979,8 @@ WUFFS_XZ__FILTER_06_IA64_BRANCH_TABLE[32] WUFFS_BASE__POTENTIALLY_UNUSED = { 4u, 4u, 0u, 0u, 4u, 4u, 0u, 0u, }; +#define WUFFS_XZ__QUIRKS_BASE 2021322752u + static const uint8_t WUFFS_XZ__CHECKSUM_LENGTH[4] WUFFS_BASE__POTENTIALLY_UNUSED = { 0u, 4u, 8u, 32u, @@ -67706,8 +67713,14 @@ wuffs_xz__decoder__get_quirk( return 0; } - if ((a_key == 1u) && self->private_impl.f_ignore_checksum) { - return 1u; + if (a_key == 1u) { + if (self->private_impl.f_ignore_checksum) { + return 1u; + } + } else if (a_key == 2021322752u) { + if (self->private_impl.f_standalone_format) { + return 1u; + } } return 0u; } @@ -67733,6 +67746,9 @@ wuffs_xz__decoder__set_quirk( if (a_key == 1u) { self->private_impl.f_ignore_checksum = (a_value > 0u); return wuffs_base__make_status(NULL); + } else if (a_key == 2021322752u) { + self->private_impl.f_standalone_format = (a_value > 0u); + return wuffs_base__make_status(NULL); } return wuffs_base__make_status(wuffs_base__error__unsupported_option); } @@ -67863,6 +67879,7 @@ wuffs_xz__decoder__do_transform_io( uint64_t v_uncompressed_size = 0; uint32_t v_hash = 0; uint8_t v_c8 = 0; + uint32_t v_c32 = 0; uint16_t v_footer_magic = 0; uint8_t* iop_a_dst = NULL; @@ -67900,7 +67917,7 @@ wuffs_xz__decoder__do_transform_io( switch (coro_susp_point) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; - do { + while (true) { { WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); uint64_t t_0; @@ -67987,6 +68004,7 @@ wuffs_xz__decoder__do_transform_io( goto exit; } self->private_impl.f_flags = ((uint16_t)(v_header_magic)); + self->private_impl.f_num_actual_blocks = 0u; while (true) { if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); @@ -68604,7 +68622,34 @@ wuffs_xz__decoder__do_transform_io( status = wuffs_base__make_status(wuffs_xz__error__bad_footer); goto exit; } - } while (0); + if ( ! self->private_impl.f_standalone_format) { + break; + } + while (true) { + while (((uint64_t)(io2_a_src - iop_a_src)) < 4u) { + if (a_src && a_src->meta.closed) { + if (((uint64_t)(io2_a_src - iop_a_src)) == 0u) { + goto label__streams__break; + } else { + status = wuffs_base__make_status(wuffs_xz__error__truncated_input); + goto exit; + } + } + status = wuffs_base__make_status(wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(34); + } + v_c32 = wuffs_base__peek_u32le__no_bounds_check(iop_a_src); + if (v_c32 == 1484404733u) { + break; + } else if (v_c32 != 0u) { + status = wuffs_base__make_status(wuffs_xz__error__bad_header_concatenated_stream); + goto exit; + } + iop_a_src += 4u; + } + self->private_impl.f_started_verify_index = false; + } + label__streams__break:; ok: self->private_impl.p_do_transform_io = 0; @@ -69095,6 +69140,7 @@ wuffs_xz__decoder__verify_index( status = wuffs_base__make_status(wuffs_xz__error__bad_index); goto exit; } + self->private_impl.f_num_index_blocks = 0u; v_shift = 0u; while (true) { { diff --git a/std/xz/decode_quirks.wuffs b/std/xz/decode_quirks.wuffs new file mode 100644 index 00000000..d0ba2f5b --- /dev/null +++ b/std/xz/decode_quirks.wuffs @@ -0,0 +1,40 @@ +// Copyright 2024 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// -------- + +// Quirks are discussed in (/doc/note/quirks.md). +// +// The base38 encoding of "xz " is 0x1E1EBC. Left shifting by 10 gives +// 0x787A_F000. +pri const QUIRKS_BASE : base.u32 = 0x787A_F000 + +// -------- + +// When this quirk is set, a positive value means to allow one or more Streams +// (which can have trailing Stream Padding). Zero means exactly one Stream (and +// decoding stops at the end of that Stream; no Stream Padding is consumed). +// +// The XZ spec (section 2. Overall Structure of .xz File) says: +// +// While a typical file contains only one Stream and no Stream Padding, a +// decoder handling standalone .xz files SHOULD support files that have more +// than one Stream or Stream Padding. +// +// In contrast to standalone .xz files, when the .xz file format is used as +// an internal part of some other file format or communication protocol, it +// usually is expected that the decoder stops after the first Stream, and +// doesn't look for Stream Padding or possibly other Streams. +// +// For Wuffs, the quirk value is zero by default, which means to decode the +// XZ-as-part-of-something-else format, not the XZ-standalone format. Like most +// of Wuffs' other std/foobar quirks, decoding a more lenient variant of the +// foobar format is opt-in, not opt-out. +pub const QUIRK_DECODE_STANDALONE_CONCATENATED_STREAMS : base.u32 = 0x787A_F000 | 0x00 diff --git a/std/xz/decode_xz.wuffs b/std/xz/decode_xz.wuffs index cfb42a46..113f748f 100644 --- a/std/xz/decode_xz.wuffs +++ b/std/xz/decode_xz.wuffs @@ -22,6 +22,7 @@ pub status "#bad checksum" pub status "#bad filter" pub status "#bad footer" pub status "#bad header" +pub status "#bad header (concatenated stream)" pub status "#bad index" pub status "#bad padding" pub status "#truncated input" @@ -44,7 +45,8 @@ pub struct decoder? implements base.io_transformer( // 3: SHA-256. checksummer : base.u8[..= 3], - ignore_checksum : base.bool, + ignore_checksum : base.bool, + standalone_format : base.bool, lzma_needs_reset : base.bool, @@ -102,8 +104,14 @@ pub struct decoder? implements base.io_transformer( ) pub func decoder.get_quirk(key: base.u32) base.u64 { - if (args.key == base.QUIRK_IGNORE_CHECKSUM) and this.ignore_checksum { - return 1 + if args.key == base.QUIRK_IGNORE_CHECKSUM { + if this.ignore_checksum { + return 1 + } + } else if args.key == QUIRK_DECODE_STANDALONE_CONCATENATED_STREAMS { + if this.standalone_format { + return 1 + } } return 0 } @@ -112,6 +120,9 @@ pub func decoder.set_quirk!(key: base.u32, value: base.u64) base.status { if args.key == base.QUIRK_IGNORE_CHECKSUM { this.ignore_checksum = args.value > 0 return ok + } else if args.key == QUIRK_DECODE_STANDALONE_CONCATENATED_STREAMS { + this.standalone_format = args.value > 0 + return ok } return base."#unsupported option" } @@ -151,6 +162,7 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work var uncompressed_size : base.u64 var hash : base.u32 var c8 : base.u8 + var c32 : base.u32 var footer_magic : base.u16 while.streams true { @@ -182,6 +194,7 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work } this.flags = (header_magic & 0xFFFF) as base.u16 + this.num_actual_blocks = 0 while.blocks true { if args.src.length() <= 0 { yield? base."$short read" @@ -431,7 +444,34 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work return "#bad footer" } - break.streams + if not this.standalone_format { + break.streams + } + + // Consume any Stream Padding. + while.stream_padding true { + while args.src.length() < 4, + post args.src.length() >= 4, + { + if args.src.is_closed() { + if args.src.length() == 0 { + break.streams + } else { + return "#truncated input" + } + } + yield? base."$short read" + } endwhile + + c32 = args.src.peek_u32le() + if c32 == '\xFD\x37\x7A\x58'le { + break.stream_padding + } else if c32 <> 0 { + return "#bad header (concatenated stream)" + } + args.src.skip_u32_fast!(actual: 4, worst_case: 4) + } endwhile.stream_padding + this.started_verify_index = false } endwhile.streams } @@ -643,6 +683,7 @@ pri func decoder.verify_index?(src: base.io_reader) { return "#bad index" } + this.num_index_blocks = 0 shift = 0 while true { c8 = args.src.read_u8?() diff --git a/test/3pdata/mzcat-checksums-of-xzsuite.txt b/test/3pdata/mzcat-checksums-of-xzsuite.txt index 773b19e9..4294bb62 100644 --- a/test/3pdata/mzcat-checksums-of-xzsuite.txt +++ b/test/3pdata/mzcat-checksums-of-xzsuite.txt @@ -1,8 +1,4 @@ # Generated by script/print-mzcat-checksums.sh -OK. 00000000 test/3pdata/xzsuite/bad-0cat-alone.xz -OK. 00000000 test/3pdata/xzsuite/bad-0cat-header_magic.xz -OK. 00000000 test/3pdata/xzsuite/bad-0catpad-empty.xz -OK. 00000000 test/3pdata/xzsuite/bad-0pad-empty.xz BAD 15a2a343 test/3pdata/xzsuite/bad-1-block_header-4.xz BAD 15a2a343 test/3pdata/xzsuite/bad-1-block_header-5.xz BAD 15a2a343 test/3pdata/xzsuite/bad-1-check-crc32-2.xz diff --git a/test/mzcat-checksums-of-data.txt b/test/mzcat-checksums-of-data.txt index d80e131e..fc5dc86b 100644 --- a/test/mzcat-checksums-of-data.txt +++ b/test/mzcat-checksums-of-data.txt @@ -1,7 +1,7 @@ # Generated by script/print-mzcat-checksums.sh OK. 29058c73 test/data/256.bytes.bz2 OK. 29058c73 test/data/256.bytes.gz -OK. 24650d57 test/data/256.bytes.two-concatenated-streams.xz +OK. 29058c73 test/data/256.bytes.two-concatenated-streams.xz OK. 2b581525 test/data/DCI-P3-D65.icc.zlib OK. 6ddef9db test/data/abraca.txt.bz2 OK. b19df306 test/data/archive.tar.bz2