std/xz: decode concatenated streams

This commit is contained in:
Nigel Tao committed 2024-02-26 23:02:40 +11:00
1 parent 5b62289c63
commit 22b4ff4842
7 files changed
+141 -13

No files matched your search

+2
View File
@@ -67,5 +67,7 @@ Package-specific quirks:
- [GIF image decoder quirks](/std/gif/decode_quirks.wuffs)
- [JSON decoder quirks](/std/json/decode_quirks.wuffs)
- [LZMA decoder quirks](/std/lzma/decode_quirks.wuffs)
- [LZW decoder quirks](/std/lzw/decode_quirks.wuffs)
- [XZ decoder quirks](/std/xz/decode_quirks.wuffs)
- [ZLIB decoder quirks](/std/zlib/decode_quirks.wuffs)
+3
View File
@@ -221,6 +221,9 @@ initialize_io_transformer(uint8_t input_first_byte) {
WUFFS_VERSION, WUFFS_INITIALIZE__DEFAULT_OPTIONS);
io_transformer = wuffs_xz__decoder__upcast_as__wuffs_base__io_transformer(
&g_potential_decoders.xz);
wuffs_base__io_transformer__set_quirk(
io_transformer,
WUFFS_XZ__QUIRK_DECODE_STANDALONE_CONCATENATED_STREAMS, 1);
break;
}
+50 -4
View File
@@ -13808,6 +13808,7 @@ extern const char wuffs_xz__error__bad_checksum[];
extern const char wuffs_xz__error__bad_filter[];
extern const char wuffs_xz__error__bad_footer[];
extern const char wuffs_xz__error__bad_header[];
extern const char wuffs_xz__error__bad_header_concatenated_stream[];
extern const char wuffs_xz__error__bad_index[];
extern const char wuffs_xz__error__bad_padding[];
extern const char wuffs_xz__error__truncated_input[];
@@ -13817,6 +13818,8 @@ extern const char wuffs_xz__error__unsupported_filter_combination[];
// ---------------- Public Consts
#define WUFFS_XZ__QUIRK_DECODE_STANDALONE_CONCATENATED_STREAMS 2021322752u
#define WUFFS_XZ__DECODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE 0u
#define WUFFS_XZ__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE 4294967568u
@@ -13935,6 +13938,7 @@ struct wuffs_xz__decoder__struct {
uint32_t f_num_non_final_filters;
uint8_t f_checksummer;
bool f_ignore_checksum;
bool f_standalone_format;
bool f_lzma_needs_reset;
bool f_block_has_compressed_size;
bool f_block_has_uncompressed_size;
@@ -66941,6 +66945,7 @@ const char wuffs_xz__error__bad_checksum[] = "#xz: bad checksum";
const char wuffs_xz__error__bad_filter[] = "#xz: bad filter";
const char wuffs_xz__error__bad_footer[] = "#xz: bad footer";
const char wuffs_xz__error__bad_header[] = "#xz: bad header";
const char wuffs_xz__error__bad_header_concatenated_stream[] = "#xz: bad header (concatenated stream)";
const char wuffs_xz__error__bad_index[] = "#xz: bad index";
const char wuffs_xz__error__bad_padding[] = "#xz: bad padding";
const char wuffs_xz__error__truncated_input[] = "#xz: truncated input";
@@ -66974,6 +66979,8 @@ WUFFS_XZ__FILTER_06_IA64_BRANCH_TABLE[32] WUFFS_BASE__POTENTIALLY_UNUSED = {
4u, 4u, 0u, 0u, 4u, 4u, 0u, 0u,
};
#define WUFFS_XZ__QUIRKS_BASE 2021322752u
static const uint8_t
WUFFS_XZ__CHECKSUM_LENGTH[4] WUFFS_BASE__POTENTIALLY_UNUSED = {
0u, 4u, 8u, 32u,
@@ -67706,8 +67713,14 @@ wuffs_xz__decoder__get_quirk(
return 0;
}
if ((a_key == 1u) && self->private_impl.f_ignore_checksum) {
return 1u;
if (a_key == 1u) {
if (self->private_impl.f_ignore_checksum) {
return 1u;
}
} else if (a_key == 2021322752u) {
if (self->private_impl.f_standalone_format) {
return 1u;
}
}
return 0u;
}
@@ -67733,6 +67746,9 @@ wuffs_xz__decoder__set_quirk(
if (a_key == 1u) {
self->private_impl.f_ignore_checksum = (a_value > 0u);
return wuffs_base__make_status(NULL);
} else if (a_key == 2021322752u) {
self->private_impl.f_standalone_format = (a_value > 0u);
return wuffs_base__make_status(NULL);
}
return wuffs_base__make_status(wuffs_base__error__unsupported_option);
}
@@ -67863,6 +67879,7 @@ wuffs_xz__decoder__do_transform_io(
uint64_t v_uncompressed_size = 0;
uint32_t v_hash = 0;
uint8_t v_c8 = 0;
uint32_t v_c32 = 0;
uint16_t v_footer_magic = 0;
uint8_t* iop_a_dst = NULL;
@@ -67900,7 +67917,7 @@ wuffs_xz__decoder__do_transform_io(
switch (coro_susp_point) {
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0;
do {
while (true) {
{
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1);
uint64_t t_0;
@@ -67987,6 +68004,7 @@ wuffs_xz__decoder__do_transform_io(
goto exit;
}
self->private_impl.f_flags = ((uint16_t)(v_header_magic));
self->private_impl.f_num_actual_blocks = 0u;
while (true) {
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) {
status = wuffs_base__make_status(wuffs_base__suspension__short_read);
@@ -68604,7 +68622,34 @@ wuffs_xz__decoder__do_transform_io(
status = wuffs_base__make_status(wuffs_xz__error__bad_footer);
goto exit;
}
} while (0);
if ( ! self->private_impl.f_standalone_format) {
break;
}
while (true) {
while (((uint64_t)(io2_a_src - iop_a_src)) < 4u) {
if (a_src && a_src->meta.closed) {
if (((uint64_t)(io2_a_src - iop_a_src)) == 0u) {
goto label__streams__break;
} else {
status = wuffs_base__make_status(wuffs_xz__error__truncated_input);
goto exit;
}
}
status = wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(34);
}
v_c32 = wuffs_base__peek_u32le__no_bounds_check(iop_a_src);
if (v_c32 == 1484404733u) {
break;
} else if (v_c32 != 0u) {
status = wuffs_base__make_status(wuffs_xz__error__bad_header_concatenated_stream);
goto exit;
}
iop_a_src += 4u;
}
self->private_impl.f_started_verify_index = false;
}
label__streams__break:;
ok:
self->private_impl.p_do_transform_io = 0;
@@ -69095,6 +69140,7 @@ wuffs_xz__decoder__verify_index(
status = wuffs_base__make_status(wuffs_xz__error__bad_index);
goto exit;
}
self->private_impl.f_num_index_blocks = 0u;
v_shift = 0u;
while (true) {
{
+40
View File
@@ -0,0 +1,40 @@
// Copyright 2024 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// SPDX-License-Identifier: Apache-2.0 OR MIT
// --------
// Quirks are discussed in (/doc/note/quirks.md).
//
// The base38 encoding of "xz " is 0x1E1EBC. Left shifting by 10 gives
// 0x787A_F000.
pri const QUIRKS_BASE : base.u32 = 0x787A_F000
// --------
// When this quirk is set, a positive value means to allow one or more Streams
// (which can have trailing Stream Padding). Zero means exactly one Stream (and
// decoding stops at the end of that Stream; no Stream Padding is consumed).
//
// The XZ spec (section 2. Overall Structure of .xz File) says:
//
// While a typical file contains only one Stream and no Stream Padding, a
// decoder handling standalone .xz files SHOULD support files that have more
// than one Stream or Stream Padding.
//
// In contrast to standalone .xz files, when the .xz file format is used as
// an internal part of some other file format or communication protocol, it
// usually is expected that the decoder stops after the first Stream, and
// doesn't look for Stream Padding or possibly other Streams.
//
// For Wuffs, the quirk value is zero by default, which means to decode the
// XZ-as-part-of-something-else format, not the XZ-standalone format. Like most
// of Wuffs' other std/foobar quirks, decoding a more lenient variant of the
// foobar format is opt-in, not opt-out.
pub const QUIRK_DECODE_STANDALONE_CONCATENATED_STREAMS : base.u32 = 0x787A_F000 | 0x00
+45 -4
View File
@@ -22,6 +22,7 @@ pub status "#bad checksum"
pub status "#bad filter"
pub status "#bad footer"
pub status "#bad header"
pub status "#bad header (concatenated stream)"
pub status "#bad index"
pub status "#bad padding"
pub status "#truncated input"
@@ -44,7 +45,8 @@ pub struct decoder? implements base.io_transformer(
// 3: SHA-256.
checksummer : base.u8[..= 3],
ignore_checksum : base.bool,
ignore_checksum : base.bool,
standalone_format : base.bool,
lzma_needs_reset : base.bool,
@@ -102,8 +104,14 @@ pub struct decoder? implements base.io_transformer(
)
pub func decoder.get_quirk(key: base.u32) base.u64 {
if (args.key == base.QUIRK_IGNORE_CHECKSUM) and this.ignore_checksum {
return 1
if args.key == base.QUIRK_IGNORE_CHECKSUM {
if this.ignore_checksum {
return 1
}
} else if args.key == QUIRK_DECODE_STANDALONE_CONCATENATED_STREAMS {
if this.standalone_format {
return 1
}
}
return 0
}
@@ -112,6 +120,9 @@ pub func decoder.set_quirk!(key: base.u32, value: base.u64) base.status {
if args.key == base.QUIRK_IGNORE_CHECKSUM {
this.ignore_checksum = args.value > 0
return ok
} else if args.key == QUIRK_DECODE_STANDALONE_CONCATENATED_STREAMS {
this.standalone_format = args.value > 0
return ok
}
return base."#unsupported option"
}
@@ -151,6 +162,7 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work
var uncompressed_size : base.u64
var hash : base.u32
var c8 : base.u8
var c32 : base.u32
var footer_magic : base.u16
while.streams true {
@@ -182,6 +194,7 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work
}
this.flags = (header_magic & 0xFFFF) as base.u16
this.num_actual_blocks = 0
while.blocks true {
if args.src.length() <= 0 {
yield? base."$short read"
@@ -431,7 +444,34 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work
return "#bad footer"
}
break.streams
if not this.standalone_format {
break.streams
}
// Consume any Stream Padding.
while.stream_padding true {
while args.src.length() < 4,
post args.src.length() >= 4,
{
if args.src.is_closed() {
if args.src.length() == 0 {
break.streams
} else {
return "#truncated input"
}
}
yield? base."$short read"
} endwhile
c32 = args.src.peek_u32le()
if c32 == '\xFD\x37\x7A\x58'le {
break.stream_padding
} else if c32 <> 0 {
return "#bad header (concatenated stream)"
}
args.src.skip_u32_fast!(actual: 4, worst_case: 4)
} endwhile.stream_padding
this.started_verify_index = false
} endwhile.streams
}
@@ -643,6 +683,7 @@ pri func decoder.verify_index?(src: base.io_reader) {
return "#bad index"
}
this.num_index_blocks = 0
shift = 0
while true {
c8 = args.src.read_u8?()
@@ -1,8 +1,4 @@
# Generated by script/print-mzcat-checksums.sh
OK. 00000000 test/3pdata/xzsuite/bad-0cat-alone.xz
OK. 00000000 test/3pdata/xzsuite/bad-0cat-header_magic.xz
OK. 00000000 test/3pdata/xzsuite/bad-0catpad-empty.xz
OK. 00000000 test/3pdata/xzsuite/bad-0pad-empty.xz
BAD 15a2a343 test/3pdata/xzsuite/bad-1-block_header-4.xz
BAD 15a2a343 test/3pdata/xzsuite/bad-1-block_header-5.xz
BAD 15a2a343 test/3pdata/xzsuite/bad-1-check-crc32-2.xz
+1 -1
View File
@@ -1,7 +1,7 @@
# Generated by script/print-mzcat-checksums.sh
OK. 29058c73 test/data/256.bytes.bz2
OK. 29058c73 test/data/256.bytes.gz
OK. 24650d57 test/data/256.bytes.two-concatenated-streams.xz
OK. 29058c73 test/data/256.bytes.two-concatenated-streams.xz
OK. 2b581525 test/data/DCI-P3-D65.icc.zlib
OK. 6ddef9db test/data/abraca.txt.bz2
OK. b19df306 test/data/archive.tar.bz2