From 41a83f2e29139e66a84ccc76421e5246149f8aaa Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sun, 24 Dec 2023 22:33:42 +1100 Subject: [PATCH] std/lzma: add LZMA2 support --- release/c/wuffs-unsupported-snapshot.c | 444 ++++++++++++++++++------- std/lzma/decode_lzma.wuffs | 176 ++++++++-- std/lzma/decode_quirks.wuffs | 30 ++ test/c/std/lzma.c | 46 ++- 4 files changed, 535 insertions(+), 161 deletions(-) create mode 100644 std/lzma/decode_quirks.wuffs diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index f023304f..54610190 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -10329,6 +10329,7 @@ struct wuffs_json__decoder__struct { // ---------------- Status Codes +extern const char wuffs_lzma__error__bad_lzma2_header[]; extern const char wuffs_lzma__error__bad_bitstream_trailer[]; extern const char wuffs_lzma__error__bad_code[]; extern const char wuffs_lzma__error__bad_decoded_length[]; @@ -10344,6 +10345,8 @@ extern const char wuffs_lzma__error__unsupported_properties[]; #define WUFFS_LZMA__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE 0u +#define WUFFS_LZMA__QUIRK_FORMAT_EXTENSION 1348001792u + // ---------------- Struct Declarations typedef struct wuffs_lzma__decoder__struct wuffs_lzma__decoder; @@ -10457,13 +10460,18 @@ struct wuffs_lzma__decoder__struct { uint32_t f_lc; uint32_t f_lp; uint32_t f_pb; + uint32_t f_format_extension; uint32_t f_dict_size; uint64_t f_decoded_length; + uint64_t f_lzma2_encoded_length_have; + uint64_t f_lzma2_encoded_length_want; + bool f_lzma2_need_prob_reset; + bool f_lzma2_need_properties; uint32_t p_decode_bitstream_slow; uint32_t p_transform_io; uint32_t p_do_transform_io; - uint32_t p_decode_properties; + uint32_t p_decode_bitstream; } private_impl; struct { @@ -10514,8 +10522,9 @@ struct wuffs_lzma__decoder__struct { uint64_t scratch; } s_decode_bitstream_slow; struct { + uint8_t v_header_byte; uint64_t scratch; - } s_decode_properties; + } s_do_transform_io; } private_data; #ifdef __cplusplus @@ -50573,6 +50582,7 @@ wuffs_json__decoder__decode_trailer( // ---------------- Status Codes Implementations +const char wuffs_lzma__error__bad_lzma2_header[] = "#lzma: bad LZMA2 header"; const char wuffs_lzma__error__bad_bitstream_trailer[] = "#lzma: bad bitstream trailer"; const char wuffs_lzma__error__bad_code[] = "#lzma: bad code"; const char wuffs_lzma__error__bad_decoded_length[] = "#lzma: bad decoded length"; @@ -50581,6 +50591,7 @@ const char wuffs_lzma__error__bad_header[] = "#lzma: bad header"; const char wuffs_lzma__error__truncated_input[] = "#lzma: truncated input"; const char wuffs_lzma__error__unsupported_decoded_length[] = "#lzma: unsupported decoded length"; const char wuffs_lzma__error__unsupported_properties[] = "#lzma: unsupported properties"; +const char wuffs_lzma__error__todo[] = "#lzma: TODO"; // ---------------- Private Consts @@ -50613,6 +50624,8 @@ WUFFS_LZMA__CLAMP_NO_MORE_THAN_3[8] WUFFS_BASE__POTENTIALLY_UNUSED = { 0u, 1u, 2u, 3u, 3u, 3u, 3u, 3u, }; +#define WUFFS_LZMA__QUIRKS_BASE 1348001792u + // ---------------- Private Initializer Prototypes // ---------------- Private Function Prototypes @@ -50629,13 +50642,13 @@ static wuffs_base__status wuffs_lzma__decoder__do_transform_io( wuffs_lzma__decoder* self, wuffs_base__io_buffer* a_dst, - wuffs_base__io_buffer* a_src, - wuffs_base__slice_u8 a_workbuf); + wuffs_base__io_buffer* a_src); WUFFS_BASE__GENERATED_C_CODE static wuffs_base__status -wuffs_lzma__decoder__decode_properties( +wuffs_lzma__decoder__decode_bitstream( wuffs_lzma__decoder* self, + wuffs_base__io_buffer* a_dst, wuffs_base__io_buffer* a_src); WUFFS_BASE__GENERATED_C_CODE @@ -51820,6 +51833,9 @@ wuffs_lzma__decoder__get_quirk( return 0; } + if (a_key == 1348001792u) { + return ((uint64_t)(self->private_impl.f_format_extension)); + } return 0u; } @@ -51841,6 +51857,22 @@ wuffs_lzma__decoder__set_quirk( : wuffs_base__error__initialize_not_called); } + uint32_t v_v = 0; + + if (a_key == 1348001792u) { + if (a_value == 0u) { + } else if (((a_value & 255u) != 2u) || ((a_value >> 8u) > 40u)) { + return wuffs_base__make_status(wuffs_base__error__bad_argument); + } + self->private_impl.f_format_extension = ((uint32_t)(a_value)); + v_v = (self->private_impl.f_format_extension >> 8u); + if (v_v < 40u) { + self->private_impl.f_dict_size = ((2u | (v_v & 1u)) << ((v_v >> 1u) + 11u)); + } else { + self->private_impl.f_dict_size = 4294967295u; + } + return wuffs_base__make_status(NULL); + } return wuffs_base__make_status(wuffs_base__error__unsupported_option); } @@ -51919,7 +51951,7 @@ wuffs_lzma__decoder__transform_io( while (true) { { - wuffs_base__status t_0 = wuffs_lzma__decoder__do_transform_io(self, a_dst, a_src, a_workbuf); + wuffs_base__status t_0 = wuffs_lzma__decoder__do_transform_io(self, a_dst, a_src); v_status = t_0; } if ((v_status.repr == wuffs_base__suspension__short_read) && (a_src && a_src->meta.closed)) { @@ -51955,55 +51987,17 @@ static wuffs_base__status wuffs_lzma__decoder__do_transform_io( wuffs_lzma__decoder* self, wuffs_base__io_buffer* a_dst, - wuffs_base__io_buffer* a_src, - wuffs_base__slice_u8 a_workbuf) { - wuffs_base__status status = wuffs_base__make_status(NULL); - - uint32_t coro_susp_point = self->private_impl.p_do_transform_io; - switch (coro_susp_point) { - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; - - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); - status = wuffs_lzma__decoder__decode_properties(self, a_src); - if (status.repr) { - goto suspend; - } - wuffs_lzma__decoder__initialize_probs(self); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); - status = wuffs_lzma__decoder__decode_bitstream_slow(self, a_dst, a_src); - if (status.repr) { - goto suspend; - } - - goto ok; - ok: - self->private_impl.p_do_transform_io = 0; - goto exit; - } - - goto suspend; - suspend: - self->private_impl.p_do_transform_io = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; - - goto exit; - exit: - return status; -} - -// -------- func lzma.decoder.decode_properties - -WUFFS_BASE__GENERATED_C_CODE -static wuffs_base__status -wuffs_lzma__decoder__decode_properties( - wuffs_lzma__decoder* self, wuffs_base__io_buffer* a_src) { wuffs_base__status status = wuffs_base__make_status(NULL); - uint8_t v_c8 = 0; + uint8_t v_header_byte = 0; uint32_t v_c32 = 0; + uint8_t v_prop_byte = 0; uint32_t v_lc = 0; uint32_t v_lp = 0; uint32_t v_pb = 0; + uint64_t v_smark = 0; + wuffs_base__status v_status = wuffs_base__make_status(NULL); const uint8_t* iop_a_src = NULL; const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; @@ -52016,114 +52010,277 @@ wuffs_lzma__decoder__decode_properties( io2_a_src = io0_a_src + a_src->meta.wi; } - uint32_t coro_susp_point = self->private_impl.p_decode_properties; + uint32_t coro_susp_point = self->private_impl.p_do_transform_io; + if (coro_susp_point) { + v_header_byte = self->private_data.s_do_transform_io.v_header_byte; + } switch (coro_susp_point) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; - { - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); - if (WUFFS_BASE__UNLIKELY(iop_a_src == io2_a_src)) { - status = wuffs_base__make_status(wuffs_base__suspension__short_read); - goto suspend; - } - uint8_t t_0 = *iop_a_src++; - v_c8 = t_0; - } - if (v_c8 >= 225u) { - status = wuffs_base__make_status(wuffs_lzma__error__bad_header); - goto exit; - } - v_lc = ((uint32_t)(((uint8_t)(v_c8 % 9u)))); + self->private_impl.f_lzma2_need_prob_reset = true; + self->private_impl.f_lzma2_need_properties = true; + while (true) { + if (self->private_impl.f_format_extension == 0u) { + { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); + if (WUFFS_BASE__UNLIKELY(iop_a_src == io2_a_src)) { + status = wuffs_base__make_status(wuffs_base__suspension__short_read); + goto suspend; + } + uint8_t t_0 = *iop_a_src++; + v_prop_byte = t_0; + } + if (v_prop_byte >= 225u) { + status = wuffs_base__make_status(wuffs_lzma__error__bad_header); + goto exit; + } + v_lc = ((uint32_t)(((uint8_t)(v_prop_byte % 9u)))); #if defined(__GNUC__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wconversion" #endif - v_c8 /= 9u; + v_prop_byte /= 9u; #if defined(__GNUC__) #pragma GCC diagnostic pop #endif - v_lp = ((uint32_t)(((uint8_t)(v_c8 % 5u)))); - v_pb = ((uint32_t)(((uint8_t)(v_c8 / 5u)))); - if ((v_lc + v_lp) > 4u) { - status = wuffs_base__make_status(wuffs_lzma__error__unsupported_properties); - goto exit; - } - self->private_impl.f_lc = wuffs_base__u32__min(v_lc, 4u); - self->private_impl.f_lp = v_lp; - self->private_impl.f_pb = v_pb; - { - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); - uint32_t t_1; - if (WUFFS_BASE__LIKELY(io2_a_src - iop_a_src >= 4)) { - t_1 = wuffs_base__peek_u32le__no_bounds_check(iop_a_src); - iop_a_src += 4; + v_lp = ((uint32_t)(((uint8_t)(v_prop_byte % 5u)))); + v_pb = ((uint32_t)(((uint8_t)(v_prop_byte / 5u)))); + if ((v_lc + v_lp) > 4u) { + status = wuffs_base__make_status(wuffs_lzma__error__unsupported_properties); + goto exit; + } + self->private_impl.f_lc = wuffs_base__u32__min(v_lc, 4u); + self->private_impl.f_lp = v_lp; + self->private_impl.f_pb = v_pb; + { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); + uint32_t t_1; + if (WUFFS_BASE__LIKELY(io2_a_src - iop_a_src >= 4)) { + t_1 = wuffs_base__peek_u32le__no_bounds_check(iop_a_src); + iop_a_src += 4; + } else { + self->private_data.s_do_transform_io.scratch = 0; + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(3); + while (true) { + if (WUFFS_BASE__UNLIKELY(iop_a_src == io2_a_src)) { + status = wuffs_base__make_status(wuffs_base__suspension__short_read); + goto suspend; + } + uint64_t* scratch = &self->private_data.s_do_transform_io.scratch; + uint32_t num_bits_1 = ((uint32_t)(*scratch >> 56)); + *scratch <<= 8; + *scratch >>= 8; + *scratch |= ((uint64_t)(*iop_a_src++)) << num_bits_1; + if (num_bits_1 == 24) { + t_1 = ((uint32_t)(*scratch)); + break; + } + num_bits_1 += 8u; + *scratch |= ((uint64_t)(num_bits_1)) << 56; + } + } + v_c32 = t_1; + } + self->private_impl.f_dict_size = wuffs_base__u32__max(v_c32, 4096u); + { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(4); + uint64_t t_2; + if (WUFFS_BASE__LIKELY(io2_a_src - iop_a_src >= 8)) { + t_2 = wuffs_base__peek_u64le__no_bounds_check(iop_a_src); + iop_a_src += 8; + } else { + self->private_data.s_do_transform_io.scratch = 0; + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(5); + while (true) { + if (WUFFS_BASE__UNLIKELY(iop_a_src == io2_a_src)) { + status = wuffs_base__make_status(wuffs_base__suspension__short_read); + goto suspend; + } + uint64_t* scratch = &self->private_data.s_do_transform_io.scratch; + uint32_t num_bits_2 = ((uint32_t)(*scratch >> 56)); + *scratch <<= 8; + *scratch >>= 8; + *scratch |= ((uint64_t)(*iop_a_src++)) << num_bits_2; + if (num_bits_2 == 56) { + t_2 = ((uint64_t)(*scratch)); + break; + } + num_bits_2 += 8u; + *scratch |= ((uint64_t)(num_bits_2)) << 56; + } + } + self->private_impl.f_decoded_length = t_2; + } + if ((self->private_impl.f_decoded_length >= 9223372036854775808u) && (self->private_impl.f_decoded_length != 18446744073709551615u)) { + status = wuffs_base__make_status(wuffs_lzma__error__unsupported_decoded_length); + goto exit; + } + wuffs_lzma__decoder__initialize_probs(self); } else { - self->private_data.s_decode_properties.scratch = 0; - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(3); - while (true) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(6); + continue; + } else if (wuffs_base__peek_u8be__no_bounds_check(iop_a_src) == 0u) { + iop_a_src += 1u; + break; + } + { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(7); if (WUFFS_BASE__UNLIKELY(iop_a_src == io2_a_src)) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); goto suspend; } - uint64_t* scratch = &self->private_data.s_decode_properties.scratch; - uint32_t num_bits_1 = ((uint32_t)(*scratch >> 56)); - *scratch <<= 8; - *scratch >>= 8; - *scratch |= ((uint64_t)(*iop_a_src++)) << num_bits_1; - if (num_bits_1 == 24) { - t_1 = ((uint32_t)(*scratch)); - break; + uint8_t t_3 = *iop_a_src++; + v_header_byte = t_3; + } + if (v_header_byte < 128u) { + status = wuffs_base__make_status(wuffs_lzma__error__todo); + goto exit; + } + self->private_impl.f_decoded_length = ((uint64_t)(v_header_byte)); + { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(8); + uint32_t t_4; + if (WUFFS_BASE__LIKELY(io2_a_src - iop_a_src >= 2)) { + t_4 = ((uint32_t)(wuffs_base__peek_u16be__no_bounds_check(iop_a_src))); + iop_a_src += 2; + } else { + self->private_data.s_do_transform_io.scratch = 0; + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(9); + while (true) { + if (WUFFS_BASE__UNLIKELY(iop_a_src == io2_a_src)) { + status = wuffs_base__make_status(wuffs_base__suspension__short_read); + goto suspend; + } + uint64_t* scratch = &self->private_data.s_do_transform_io.scratch; + uint32_t num_bits_4 = ((uint32_t)(*scratch & 0xFFu)); + *scratch >>= 8; + *scratch <<= 8; + *scratch |= ((uint64_t)(*iop_a_src++)) << (56 - num_bits_4); + if (num_bits_4 == 8) { + t_4 = ((uint32_t)(*scratch >> 48)); + break; + } + num_bits_4 += 8u; + *scratch |= ((uint64_t)(num_bits_4)); + } } - num_bits_1 += 8u; - *scratch |= ((uint64_t)(num_bits_1)) << 56; + v_c32 = t_4; + } + self->private_impl.f_decoded_length = (((self->private_impl.f_decoded_length & 31u) << 16u) + ((uint64_t)((1u + v_c32)))); + { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(10); + uint32_t t_5; + if (WUFFS_BASE__LIKELY(io2_a_src - iop_a_src >= 2)) { + t_5 = ((uint32_t)(wuffs_base__peek_u16be__no_bounds_check(iop_a_src))); + iop_a_src += 2; + } else { + self->private_data.s_do_transform_io.scratch = 0; + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(11); + while (true) { + if (WUFFS_BASE__UNLIKELY(iop_a_src == io2_a_src)) { + status = wuffs_base__make_status(wuffs_base__suspension__short_read); + goto suspend; + } + uint64_t* scratch = &self->private_data.s_do_transform_io.scratch; + uint32_t num_bits_5 = ((uint32_t)(*scratch & 0xFFu)); + *scratch >>= 8; + *scratch <<= 8; + *scratch |= ((uint64_t)(*iop_a_src++)) << (56 - num_bits_5); + if (num_bits_5 == 8) { + t_5 = ((uint32_t)(*scratch >> 48)); + break; + } + num_bits_5 += 8u; + *scratch |= ((uint64_t)(num_bits_5)); + } + } + v_c32 = t_5; + } + self->private_impl.f_lzma2_encoded_length_want = ((uint64_t)((1u + v_c32))); + if (v_header_byte >= 160u) { + wuffs_lzma__decoder__initialize_probs(self); + self->private_impl.f_lzma2_need_prob_reset = false; + } + if (v_header_byte >= 192u) { + { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(12); + if (WUFFS_BASE__UNLIKELY(iop_a_src == io2_a_src)) { + status = wuffs_base__make_status(wuffs_base__suspension__short_read); + goto suspend; + } + uint8_t t_6 = *iop_a_src++; + v_prop_byte = t_6; + } + if (v_prop_byte >= 225u) { + status = wuffs_base__make_status(wuffs_lzma__error__bad_lzma2_header); + goto exit; + } + v_lc = ((uint32_t)(((uint8_t)(v_prop_byte % 9u)))); +#if defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + v_prop_byte /= 9u; +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + v_lp = ((uint32_t)(((uint8_t)(v_prop_byte % 5u)))); + v_pb = ((uint32_t)(((uint8_t)(v_prop_byte / 5u)))); + if ((v_lc + v_lp) > 4u) { + status = wuffs_base__make_status(wuffs_lzma__error__bad_lzma2_header); + goto exit; + } + self->private_impl.f_lc = wuffs_base__u32__min(v_lc, 4u); + self->private_impl.f_lp = v_lp; + self->private_impl.f_pb = v_pb; + self->private_impl.f_lzma2_need_properties = false; + } + if (v_header_byte >= 224u) { + } + if (self->private_impl.f_lzma2_need_prob_reset || self->private_impl.f_lzma2_need_properties) { + status = wuffs_base__make_status(wuffs_lzma__error__bad_lzma2_header); + goto exit; } } - v_c32 = t_1; - } - self->private_impl.f_dict_size = wuffs_base__u32__max(v_c32, 4096u); - { - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(4); - uint64_t t_2; - if (WUFFS_BASE__LIKELY(io2_a_src - iop_a_src >= 8)) { - t_2 = wuffs_base__peek_u64le__no_bounds_check(iop_a_src); - iop_a_src += 8; - } else { - self->private_data.s_decode_properties.scratch = 0; - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(5); - while (true) { - if (WUFFS_BASE__UNLIKELY(iop_a_src == io2_a_src)) { - status = wuffs_base__make_status(wuffs_base__suspension__short_read); - goto suspend; + self->private_impl.f_lzma2_encoded_length_have = 0u; + while (true) { + v_smark = ((uint64_t)(iop_a_src - io0_a_src)); + { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); } - uint64_t* scratch = &self->private_data.s_decode_properties.scratch; - uint32_t num_bits_2 = ((uint32_t)(*scratch >> 56)); - *scratch <<= 8; - *scratch >>= 8; - *scratch |= ((uint64_t)(*iop_a_src++)) << num_bits_2; - if (num_bits_2 == 56) { - t_2 = ((uint64_t)(*scratch)); - break; + wuffs_base__status t_7 = wuffs_lzma__decoder__decode_bitstream(self, a_dst, a_src); + v_status = t_7; + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; } - num_bits_2 += 8u; - *scratch |= ((uint64_t)(num_bits_2)) << 56; } + wuffs_private_impl__u64__sat_add_indirect(&self->private_impl.f_lzma2_encoded_length_have, wuffs_private_impl__io__count_since(v_smark, ((uint64_t)(iop_a_src - io0_a_src)))); + if (wuffs_base__status__is_ok(&v_status)) { + break; + } + status = v_status; + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13); + } + if (self->private_impl.f_format_extension == 0u) { + break; + } else if (self->private_impl.f_lzma2_encoded_length_have != self->private_impl.f_lzma2_encoded_length_want) { + status = wuffs_base__make_status(wuffs_lzma__error__bad_lzma2_header); + goto exit; } - self->private_impl.f_decoded_length = t_2; - } - if ((self->private_impl.f_decoded_length >= 9223372036854775808u) && (self->private_impl.f_decoded_length != 18446744073709551615u)) { - status = wuffs_base__make_status(wuffs_lzma__error__unsupported_decoded_length); - goto exit; } - goto ok; ok: - self->private_impl.p_decode_properties = 0; + self->private_impl.p_do_transform_io = 0; goto exit; } goto suspend; suspend: - self->private_impl.p_decode_properties = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; + self->private_impl.p_do_transform_io = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; + self->private_data.s_do_transform_io.v_header_byte = v_header_byte; goto exit; exit: @@ -52134,6 +52291,41 @@ wuffs_lzma__decoder__decode_properties( return status; } +// -------- func lzma.decoder.decode_bitstream + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_lzma__decoder__decode_bitstream( + wuffs_lzma__decoder* self, + wuffs_base__io_buffer* a_dst, + wuffs_base__io_buffer* a_src) { + wuffs_base__status status = wuffs_base__make_status(NULL); + + uint32_t coro_susp_point = self->private_impl.p_decode_bitstream; + switch (coro_susp_point) { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; + + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); + status = wuffs_lzma__decoder__decode_bitstream_slow(self, a_dst, a_src); + if (status.repr) { + goto suspend; + } + + goto ok; + ok: + self->private_impl.p_decode_bitstream = 0; + goto exit; + } + + goto suspend; + suspend: + self->private_impl.p_decode_bitstream = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; + + goto exit; + exit: + return status; +} + // -------- func lzma.decoder.initialize_probs WUFFS_BASE__GENERATED_C_CODE diff --git a/std/lzma/decode_lzma.wuffs b/std/lzma/decode_lzma.wuffs index 663be059..6f50e4e5 100644 --- a/std/lzma/decode_lzma.wuffs +++ b/std/lzma/decode_lzma.wuffs @@ -8,6 +8,7 @@ // // SPDX-License-Identifier: Apache-2.0 OR MIT +pub status "#bad LZMA2 header" pub status "#bad bitstream trailer" pub status "#bad code" pub status "#bad decoded length" @@ -17,6 +18,8 @@ pub status "#truncated input" pub status "#unsupported decoded length" pub status "#unsupported properties" +pri status "#TODO" + pub const DECODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE : base.u64 = 0xFFFF_FFFF pub const DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE : base.u64 = 0 @@ -33,10 +36,17 @@ pub struct decoder? implements base.io_transformer( lp : base.u32[..= 4], // The number of Literal Position bits. Default: 0. pb : base.u32[..= 4], // The number of Position Bits. Default: 2. + format_extension : base.u32, + dict_size : base.u32, decoded_length : base.u64, + lzma2_encoded_length_have : base.u64, + lzma2_encoded_length_want : base.u64, + lzma2_need_prob_reset : base.bool, + lzma2_need_properties : base.bool, + util : base.utility, ) + ( // 12 is the number of states. 4 is the pb inclusive maximum. @@ -96,10 +106,31 @@ pub struct decoder? implements base.io_transformer( ) pub func decoder.get_quirk(key: base.u32) base.u64 { + if args.key == QUIRK_FORMAT_EXTENSION { + return this.format_extension as base.u64 + } return 0 } pub func decoder.set_quirk!(key: base.u32, value: base.u64) base.status { + var v : base.u32 + + if args.key == QUIRK_FORMAT_EXTENSION { + if args.value == 0 { + // No-op. + } else if ((args.value & 0xFF) <> 0x02) or ((args.value >> 8) > 40) { + return base."#bad argument" + } + this.format_extension = (args.value & 0xFFFF_FFFF) as base.u32 + + v = this.format_extension >> 8 + if v < 40 { + this.dict_size = (2 | (v & 1)) << ((v >> 1) + 11) + } else { + this.dict_size = 0xFFFF_FFFF + } + return ok + } return base."#unsupported option" } @@ -118,7 +149,7 @@ pub func decoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf var status : base.status while true { - status =? this.do_transform_io?(dst: args.dst, src: args.src, workbuf: args.workbuf) + status =? this.do_transform_io?(dst: args.dst, src: args.src) if (status == base."$short read") and args.src.is_closed() { return "#truncated input" } @@ -126,43 +157,126 @@ pub func decoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf } endwhile } -pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, workbuf: slice base.u8) { - this.decode_properties?(src: args.src) - this.initialize_probs!() - this.decode_bitstream_slow?(dst: args.dst, src: args.src) -} - -pri func decoder.decode_properties?(src: base.io_reader) { - var c8 : base.u8 - var c32 : base.u32 +pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader) { + var header_byte : base.u8 + var c32 : base.u32 + var prop_byte : base.u8 var lc : base.u32[..= 8] var lp : base.u32[..= 4] var pb : base.u32[..= 4] - c8 = args.src.read_u8?() - if c8 >= 225 { - return "#bad header" - } - lc = (c8 % 9) as base.u32 - c8 /= 9 - lp = (c8 % 5) as base.u32 - pb = (c8 / 5) as base.u32 - if (lc + lp) > 4 { - return "#unsupported properties" - } - this.lc = lc.min(no_more_than: 4) - this.lp = lp - this.pb = pb + var smark : base.u64 + var status : base.status - c32 = args.src.read_u32le?() - this.dict_size = c32.max(no_less_than: 4096) + this.lzma2_need_prob_reset = true + this.lzma2_need_properties = true - this.decoded_length = args.src.read_u64le?() - if (this.decoded_length >= 0x8000_0000_0000_0000) and - (this.decoded_length <> 0xFFFF_FFFF_FFFF_FFFF) { - return "#unsupported decoded length" - } + // Loop over chunks. LZMA1 has only one (implicit) chunk. For LZMA2's chunk + // header, see + // https://github.com/jljusten/LZMA-SDK/blob/781863cdf592da3e97420f50de5dac056ad352a5/C/Lzma2Dec.c#L17-L23 + while true { + if this.format_extension == 0 { // LZMA1. + prop_byte = args.src.read_u8?() + if prop_byte >= 225 { + return "#bad header" + } + lc = (prop_byte % 9) as base.u32 + prop_byte /= 9 + lp = (prop_byte % 5) as base.u32 + pb = (prop_byte / 5) as base.u32 + if (lc + lp) > 4 { + return "#unsupported properties" + } + this.lc = lc.min(no_more_than: 4) + this.lp = lp + this.pb = pb + + c32 = args.src.read_u32le?() + this.dict_size = c32.max(no_less_than: 4096) + + this.decoded_length = args.src.read_u64le?() + if (this.decoded_length >= 0x8000_0000_0000_0000) and + (this.decoded_length <> 0xFFFF_FFFF_FFFF_FFFF) { + return "#unsupported decoded length" + } + + this.initialize_probs!() + + } else { // LZMA2. + if args.src.length() <= 0 { + yield? base."$short read" + continue + } else if args.src.peek_u8() == 0x00 { + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + break + } + + header_byte = args.src.read_u8?() + if header_byte < 0x80 { + return "#TODO" + } + this.decoded_length = header_byte as base.u64 + + c32 = args.src.read_u16be_as_u32?() + this.decoded_length = ((this.decoded_length & 0x1F) << 16) + ((1 + c32) as base.u64) + + c32 = args.src.read_u16be_as_u32?() + this.lzma2_encoded_length_want = (1 + c32) as base.u64 + + if header_byte >= 0xA0 { + this.initialize_probs!() + this.lzma2_need_prob_reset = false + } + + if header_byte >= 0xC0 { + prop_byte = args.src.read_u8?() + if prop_byte >= 225 { + return "#bad LZMA2 header" + } + lc = (prop_byte % 9) as base.u32 + prop_byte /= 9 + lp = (prop_byte % 5) as base.u32 + pb = (prop_byte / 5) as base.u32 + if (lc + lp) > 4 { + return "#bad LZMA2 header" + } + this.lc = lc.min(no_more_than: 4) + this.lp = lp + this.pb = pb + this.lzma2_need_properties = false + } + + if header_byte >= 0xE0 { + // TODO: enforce "reset dictionary". + } + + if this.lzma2_need_prob_reset or this.lzma2_need_properties { + return "#bad LZMA2 header" + } + } + + this.lzma2_encoded_length_have = 0 + while true { + smark = args.src.mark() + status =? this.decode_bitstream?(dst: args.dst, src: args.src) + this.lzma2_encoded_length_have ~sat+= args.src.count_since(mark: smark) + if status.is_ok() { + break + } + yield? status + } endwhile + + if this.format_extension == 0 { + break + } else if this.lzma2_encoded_length_have <> this.lzma2_encoded_length_want { + return "#bad LZMA2 header" + } + } endwhile +} + +pri func decoder.decode_bitstream?(dst: base.io_writer, src: base.io_reader) { + this.decode_bitstream_slow?(dst: args.dst, src: args.src) } // initialize_probs sets all of the probs_foo[i][j] values to 1024. diff --git a/std/lzma/decode_quirks.wuffs b/std/lzma/decode_quirks.wuffs new file mode 100644 index 00000000..1a10a9cf --- /dev/null +++ b/std/lzma/decode_quirks.wuffs @@ -0,0 +1,30 @@ +// Copyright 2023 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// -------- + +// Quirks are discussed in (/doc/note/quirks.md). +// +// The base38 encoding of "lzma" is 0x141638. Left shifting by 10 gives +// 0x5058_E000. +pri const QUIRKS_BASE : base.u32 = 0x5058_E000 + +// -------- + +// When this quirk is set, a positive value indicates an extension to the LZMA +// format. Zero means to use the default LZMA format. +// +// When positive, the low byte of the u64 value indicates the extension. +// Currently, the only valid extension is 0x02, indicating LZMA2. +// +// For LZMA2, the high 7 bytes (shifted right by 8) must be less than or equal +// to 40 and indicates the dictionary size. See section "5.3.1. LZMA2" of +// https://tukaani.org/xz/xz-file-format.txt +pub const QUIRK_FORMAT_EXTENSION : base.u32 = 0x5058_E000 | 0x00 diff --git a/test/c/std/lzma.c b/test/c/std/lzma.c index 7c7e1d46..d35be411 100644 --- a/test/c/std/lzma.c +++ b/test/c/std/lzma.c @@ -129,12 +129,49 @@ test_wuffs_lzma_decode_enwik5() { } const char* // -test_wuffs_lzma_decode_romeo() { +test_wuffs_lzma_decode_romeo_lzma1() { CHECK_FOCUS(__func__); return do_test_io_buffers(wuffs_lzma_decode, &g_lzma_romeo_gt, UINT64_MAX, UINT64_MAX); } +const char* // +test_wuffs_lzma_decode_romeo_lzma2() { + CHECK_FOCUS(__func__); + + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = g_src_slice_u8, + }); + wuffs_base__io_buffer have = ((wuffs_base__io_buffer){ + .data = g_have_slice_u8, + }); + wuffs_base__io_buffer want = ((wuffs_base__io_buffer){ + .data = g_want_slice_u8, + }); + + CHECK_STRING(read_file(&src, "test/data/romeo.txt.xz")); + if (src.meta.wi < 0x263) { + RETURN_FAIL("source file was too short"); + } + src.meta.ri = 0x018; + src.meta.wi = 0x263; + + wuffs_lzma__decoder dec; + CHECK_STATUS("initialize", wuffs_lzma__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__DEFAULT_OPTIONS)); + wuffs_lzma__decoder__set_quirk(&dec, WUFFS_LZMA__QUIRK_FORMAT_EXTENSION, 2); + CHECK_STATUS("transform_io", wuffs_lzma__decoder__transform_io( + &dec, &have, &src, g_work_slice_u8)); + + CHECK_STRING(read_file(&want, "test/data/romeo.txt")); + CHECK_STRING(check_io_buffers_equal("", &have, &want)); + if (src.meta.ri != src.meta.wi) { + RETURN_FAIL("ri=0x%zX is not equal to wi=0x%zX", src.meta.ri, src.meta.wi); + } + return NULL; +} + // ---------------- Mimic Tests #ifdef WUFFS_MIMIC @@ -147,7 +184,7 @@ test_mimic_lzma_decode_enwik5() { } const char* // -test_mimic_lzma_decode_romeo() { +test_mimic_lzma_decode_romeo_lzma1() { CHECK_FOCUS(__func__); return do_test_io_buffers(mimic_lzma_decode, &g_lzma_romeo_gt, UINT64_MAX, UINT64_MAX); @@ -185,12 +222,13 @@ proc g_tests[] = { test_wuffs_lzma_decode_enwik5, test_wuffs_lzma_decode_interface, - test_wuffs_lzma_decode_romeo, + test_wuffs_lzma_decode_romeo_lzma1, + test_wuffs_lzma_decode_romeo_lzma2, #ifdef WUFFS_MIMIC test_mimic_lzma_decode_enwik5, - test_mimic_lzma_decode_romeo, + test_mimic_lzma_decode_romeo_lzma1, #endif // WUFFS_MIMIC