std/lzma: use args.workbuf for io_buffer history

This commit is contained in:
Nigel Tao committed 2024-02-08 21:38:30 +11:00
1 parent d9bb6d4b4f
commit c21cc95713
7 files changed
+429 -67

No files matched your search

+9 -4
View File
@@ -78,18 +78,21 @@ Supported compression formats:
#define WUFFS_EXAMPLE_USE_SECCOMP
#endif
// LZMA's default dictionary size is 16 MiB. This program's destination-buffer
// size defaults to twice that.
#ifndef DST_BUFFER_ARRAY_SIZE
#define DST_BUFFER_ARRAY_SIZE (32 * 1024 * 1024)
#define DST_BUFFER_ARRAY_SIZE (128 * 1024)
#endif
#ifndef SRC_BUFFER_ARRAY_SIZE
#define SRC_BUFFER_ARRAY_SIZE (128 * 1024)
#endif
#ifndef WORKBUF_ARRAY_SIZE
#define WORKBUF_ARRAY_SIZE (64 * 1024 * 1024)
#endif
uint8_t g_dst_buffer_array[DST_BUFFER_ARRAY_SIZE];
uint8_t g_src_buffer_array[SRC_BUFFER_ARRAY_SIZE];
uint8_t g_workbuf_array[WORKBUF_ARRAY_SIZE];
// ----
@@ -269,7 +272,9 @@ main1(int argc, char** argv) {
while (true) {
wuffs_base__status status = wuffs_base__io_transformer__transform_io(
g_io_transformer, &dst, &src, wuffs_base__empty_slice_u8());
g_io_transformer, &dst, &src,
wuffs_base__make_slice_u8(&g_workbuf_array[0],
sizeof(g_workbuf_array)));
if (dst.meta.ri < dst.meta.wi) {
// TODO: handle EINTR and other write errors; see "man 2 write".
+2
View File
@@ -245,6 +245,7 @@ var Statuses = [...]string{
`"$mispositioned read"`,
`"$mispositioned write"`,
`"$short read"`,
`"$short workbuf"`,
`"$short write"`,
// Errors.
@@ -1024,6 +1025,7 @@ var InterfaceFuncs = []string{
// ---- io_transformer
// TODO: "io_transformer.add_history!(hist: slice u8)"? Should it take a workbuf arg?
"io_transformer.dst_history_retain_length() optional_u63",
"io_transformer.get_quirk(key: u32) u64",
"io_transformer.set_quirk!(key: u32, value: u64) status",
+212 -31
View File
@@ -535,6 +535,7 @@ extern const char wuffs_base__suspension__even_more_information[];
extern const char wuffs_base__suspension__mispositioned_read[];
extern const char wuffs_base__suspension__mispositioned_write[];
extern const char wuffs_base__suspension__short_read[];
extern const char wuffs_base__suspension__short_workbuf[];
extern const char wuffs_base__suspension__short_write[];
extern const char wuffs_base__error__bad_i_o_position[];
extern const char wuffs_base__error__bad_argument_length_too_short[];
@@ -10345,9 +10346,9 @@ extern const char wuffs_lzma__error__unsupported_properties[];
// ---------------- Public Consts
#define WUFFS_LZMA__DECODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE 4294967295u
#define WUFFS_LZMA__DECODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE 0u
#define WUFFS_LZMA__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE 0u
#define WUFFS_LZMA__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE 4294967568u
#define WUFFS_LZMA__QUIRK_FORMAT_EXTENSION 1348001792u
@@ -10466,6 +10467,8 @@ struct wuffs_lzma__decoder__struct {
uint32_t f_pb;
uint32_t f_format_extension;
uint32_t f_dict_size;
uint32_t f_dict_workbuf_index;
uint32_t f_dict_seen;
uint64_t f_decoded_length;
uint64_t f_lzma2_prior_decoded_length;
uint64_t f_lzma2_encoded_length_have;
@@ -10524,6 +10527,7 @@ struct wuffs_lzma__decoder__struct {
uint32_t v_index_small_dist_base;
uint32_t v_index_small_dist_extra;
uint32_t v_index_large_dist;
uint32_t v_dist;
uint64_t scratch;
} s_decode_bitstream_slow;
struct {
@@ -13798,9 +13802,9 @@ extern const char wuffs_xz__error__unsupported_filter[];
// ---------------- Public Consts
#define WUFFS_XZ__DECODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE 4294967295u
#define WUFFS_XZ__DECODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE 0u
#define WUFFS_XZ__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE 0u
#define WUFFS_XZ__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE 4294967568u
// ---------------- Struct Declarations
@@ -15627,6 +15631,7 @@ const char wuffs_base__suspension__even_more_information[] = "$base: even more i
const char wuffs_base__suspension__mispositioned_read[] = "$base: mispositioned read";
const char wuffs_base__suspension__mispositioned_write[] = "$base: mispositioned write";
const char wuffs_base__suspension__short_read[] = "$base: short read";
const char wuffs_base__suspension__short_workbuf[] = "$base: short workbuf";
const char wuffs_base__suspension__short_write[] = "$base: short write";
const char wuffs_base__error__bad_i_o_position[] = "#base: bad I/O position";
const char wuffs_base__error__bad_argument_length_too_short[] = "#base: bad argument (length too short)";
@@ -50871,6 +50876,7 @@ const char wuffs_lzma__error__bad_header[] = "#lzma: bad header";
const char wuffs_lzma__error__truncated_input[] = "#lzma: truncated input";
const char wuffs_lzma__error__unsupported_decoded_length[] = "#lzma: unsupported decoded length";
const char wuffs_lzma__error__unsupported_properties[] = "#lzma: unsupported properties";
const char wuffs_lzma__error__internal_error_inconsistent_dictionary_state[] = "#lzma: internal error: inconsistent dictionary state";
// ---------------- Private Consts
@@ -50914,21 +50920,31 @@ static wuffs_base__status
wuffs_lzma__decoder__decode_bitstream_slow(
wuffs_lzma__decoder* self,
wuffs_base__io_buffer* a_dst,
wuffs_base__io_buffer* a_src);
wuffs_base__io_buffer* a_src,
wuffs_base__slice_u8 a_workbuf);
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__status
wuffs_lzma__decoder__add_history(
wuffs_lzma__decoder* self,
wuffs_base__slice_u8 a_hist,
wuffs_base__slice_u8 a_workbuf);
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__status
wuffs_lzma__decoder__do_transform_io(
wuffs_lzma__decoder* self,
wuffs_base__io_buffer* a_dst,
wuffs_base__io_buffer* a_src);
wuffs_base__io_buffer* a_src,
wuffs_base__slice_u8 a_workbuf);
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__status
wuffs_lzma__decoder__decode_bitstream(
wuffs_lzma__decoder* self,
wuffs_base__io_buffer* a_dst,
wuffs_base__io_buffer* a_src);
wuffs_base__io_buffer* a_src,
wuffs_base__slice_u8 a_workbuf);
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__empty_struct
@@ -51030,7 +51046,8 @@ static wuffs_base__status
wuffs_lzma__decoder__decode_bitstream_slow(
wuffs_lzma__decoder* self,
wuffs_base__io_buffer* a_dst,
wuffs_base__io_buffer* a_src) {
wuffs_base__io_buffer* a_src,
wuffs_base__slice_u8 a_workbuf) {
wuffs_base__status status = wuffs_base__make_status(NULL);
uint8_t v_c8 = 0;
@@ -51069,6 +51086,9 @@ wuffs_lzma__decoder__decode_bitstream_slow(
uint32_t v_index_small_dist_extra = 0;
uint32_t v_index_small_dist = 0;
uint32_t v_index_large_dist = 0;
uint32_t v_dist = 0;
uint32_t v_adj_dist = 0;
uint64_t v_wb_index = 0;
uint8_t* iop_a_dst = NULL;
uint8_t* io0_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
@@ -51122,6 +51142,7 @@ wuffs_lzma__decoder__decode_bitstream_slow(
v_index_small_dist_base = self->private_data.s_decode_bitstream_slow.v_index_small_dist_base;
v_index_small_dist_extra = self->private_data.s_decode_bitstream_slow.v_index_small_dist_extra;
v_index_large_dist = self->private_data.s_decode_bitstream_slow.v_index_large_dist;
v_dist = self->private_data.s_decode_bitstream_slow.v_dist;
}
switch (coro_susp_point) {
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0;
@@ -52023,21 +52044,73 @@ wuffs_lzma__decoder__decode_bitstream_slow(
v_state = ((uint32_t)(WUFFS_LZMA__STATE_TRANSITION_LONGREP[v_state]));
} while (0);
} while (0);
if ((((uint64_t)((v_rep0 + 1u))) > ((uint64_t)(v_pos + self->private_impl.f_lzma2_prior_decoded_length))) || (((uint64_t)((v_rep0 + 1u))) > ((uint64_t)(self->private_impl.f_dict_size)))) {
v_dist = (v_rep0 + 1u);
if ((((uint64_t)(v_dist)) > ((uint64_t)(v_pos + self->private_impl.f_lzma2_prior_decoded_length))) || (((uint64_t)(v_dist)) > ((uint64_t)(self->private_impl.f_dict_size)))) {
status = wuffs_base__make_status(wuffs_lzma__error__bad_distance);
goto exit;
}
v_pos += ((uint64_t)(v_len));
while (((uint64_t)(v_len)) > ((uint64_t)(io2_a_dst - iop_a_dst))) {
while (274u > ((uint64_t)(io2_a_dst - iop_a_dst))) {
status = wuffs_base__make_status(wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(38);
}
if (((uint64_t)((v_rep0 + 1u))) > ((uint64_t)(iop_a_dst - io0_a_dst))) {
status = wuffs_base__make_status(wuffs_base__error__insufficient_history);
goto exit;
if (((uint64_t)(v_dist)) > ((uint64_t)(iop_a_dst - io0_a_dst))) {
v_adj_dist = ((uint32_t)((((uint64_t)(v_dist)) - ((uint64_t)(iop_a_dst - io0_a_dst)))));
if (v_adj_dist > self->private_impl.f_dict_seen) {
status = wuffs_base__make_status(wuffs_lzma__error__bad_distance);
goto exit;
}
v_wb_index = ((uint64_t)(((uint64_t)(self->private_impl.f_dict_workbuf_index)) - ((uint64_t)(v_adj_dist))));
while (v_wb_index >= 9223372036854775808u) {
v_wb_index += ((uint64_t)(self->private_impl.f_dict_size));
}
if (v_wb_index > ((uint64_t)(a_workbuf.len))) {
status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_dictionary_state);
goto exit;
}
if (v_len < v_adj_dist) {
wuffs_private_impl__io_writer__limited_copy_u32_from_slice(
&iop_a_dst, io2_a_dst,(v_len + 1u), wuffs_base__slice_u8__subslice_i(a_workbuf, v_wb_index));
if ( ! (iop_a_dst > io1_a_dst)) {
status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_dictionary_state);
goto exit;
}
v_match_byte = iop_a_dst[-1];
iop_a_dst--;
if ( ! (iop_a_dst > io1_a_dst)) {
status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_dictionary_state);
goto exit;
}
v_prev_byte = iop_a_dst[-1];
continue;
} else if (v_len == v_adj_dist) {
wuffs_private_impl__io_writer__limited_copy_u32_from_slice(
&iop_a_dst, io2_a_dst,v_len, wuffs_base__slice_u8__subslice_i(a_workbuf, v_wb_index));
wuffs_private_impl__io_writer__limited_copy_u32_from_history(
&iop_a_dst, io0_a_dst, io2_a_dst, 1u, v_dist);
if ( ! (iop_a_dst > io1_a_dst)) {
status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_dictionary_state);
goto exit;
}
v_match_byte = iop_a_dst[-1];
iop_a_dst--;
if ( ! (iop_a_dst > io1_a_dst)) {
status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_dictionary_state);
goto exit;
}
v_prev_byte = iop_a_dst[-1];
continue;
}
wuffs_private_impl__io_writer__limited_copy_u32_from_slice(
&iop_a_dst, io2_a_dst,v_adj_dist, wuffs_base__slice_u8__subslice_i(a_workbuf, v_wb_index));
v_len -= v_adj_dist;
if ((((uint64_t)(v_len)) > ((uint64_t)(io2_a_dst - iop_a_dst))) || (((uint64_t)(v_dist)) > ((uint64_t)(iop_a_dst - io0_a_dst)))) {
status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_dictionary_state);
goto exit;
}
}
v_match_cusp = wuffs_private_impl__io_writer__limited_copy_u32_from_history_fast_return_cusp(
&iop_a_dst, io0_a_dst, io2_a_dst, v_len, (v_rep0 + 1u));
&iop_a_dst, io0_a_dst, io2_a_dst, v_len, v_dist);
v_match_byte = ((uint8_t)((v_match_cusp >> 8u)));
v_prev_byte = ((uint8_t)(v_match_cusp));
}
@@ -52084,6 +52157,7 @@ wuffs_lzma__decoder__decode_bitstream_slow(
self->private_data.s_decode_bitstream_slow.v_index_small_dist_base = v_index_small_dist_base;
self->private_data.s_decode_bitstream_slow.v_index_small_dist_extra = v_index_small_dist_extra;
self->private_data.s_decode_bitstream_slow.v_index_large_dist = v_index_large_dist;
self->private_data.s_decode_bitstream_slow.v_dist = v_dist;
goto exit;
exit:
@@ -52097,6 +52171,63 @@ wuffs_lzma__decoder__decode_bitstream_slow(
return status;
}
// -------- func lzma.decoder.add_history
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__status
wuffs_lzma__decoder__add_history(
wuffs_lzma__decoder* self,
wuffs_base__slice_u8 a_hist,
wuffs_base__slice_u8 a_workbuf) {
uint64_t v_dict_workbuf_index = 0;
uint64_t v_dict_size = 0;
uint64_t v_hist_length = 0;
wuffs_base__slice_u8 v_s = {0};
uint64_t v_n_copied = 0;
uint64_t v_n = 0;
v_dict_workbuf_index = ((uint64_t)(self->private_impl.f_dict_workbuf_index));
v_dict_size = ((uint64_t)(self->private_impl.f_dict_size));
if (((uint64_t)(a_hist.len)) == 0u) {
return wuffs_base__make_status(NULL);
}
if (((uint64_t)(a_workbuf.len)) < (v_dict_size + 273u)) {
return wuffs_base__make_status(wuffs_base__error__bad_workbuf_length);
}
v_hist_length = ((uint64_t)(a_hist.len));
if (v_hist_length > 4294967295u) {
self->private_impl.f_dict_seen = 4294967295u;
} else {
wuffs_private_impl__u32__sat_add_indirect(&self->private_impl.f_dict_seen, ((uint32_t)(v_hist_length)));
}
v_s = a_hist;
if (((uint64_t)(v_s.len)) >= v_dict_size) {
v_s = wuffs_private_impl__slice_u8__suffix(v_s, v_dict_size);
wuffs_private_impl__slice_u8__copy_from_slice(a_workbuf, v_s);
self->private_impl.f_dict_workbuf_index = 0u;
} else if (v_dict_workbuf_index > v_dict_size) {
return wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_dictionary_state);
} else {
v_n_copied = wuffs_private_impl__slice_u8__copy_from_slice(wuffs_base__slice_u8__subslice_ij(a_workbuf, v_dict_workbuf_index, v_dict_size), v_s);
if (v_n_copied < ((uint64_t)(v_s.len))) {
v_n = wuffs_private_impl__slice_u8__copy_from_slice(a_workbuf, wuffs_base__slice_u8__subslice_i(v_s, v_n_copied));
self->private_impl.f_dict_workbuf_index = ((uint32_t)(v_n));
} else {
v_n = ((uint64_t)(v_dict_workbuf_index + v_n_copied));
if (v_n < v_dict_size) {
self->private_impl.f_dict_workbuf_index = ((uint32_t)(v_n));
} else {
self->private_impl.f_dict_workbuf_index = 0u;
}
}
}
if ((273u > v_dict_size) || (v_dict_size > ((uint64_t)(a_workbuf.len)))) {
return wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_dictionary_state);
}
wuffs_private_impl__slice_u8__copy_from_slice(wuffs_base__slice_u8__subslice_i(a_workbuf, v_dict_size), wuffs_base__slice_u8__subslice_j(a_workbuf, 273u));
return wuffs_base__make_status(NULL);
}
// -------- func lzma.decoder.get_quirk
WUFFS_BASE__GENERATED_C_CODE
@@ -52169,10 +52300,7 @@ wuffs_lzma__decoder__dst_history_retain_length(
return wuffs_base__utility__make_optional_u63(false, 0u);
}
if (self->private_impl.f_dict_size == 0u) {
return wuffs_base__utility__make_optional_u63(false, 0u);
}
return wuffs_base__utility__make_optional_u63(true, ((uint64_t)(self->private_impl.f_dict_size)));
return wuffs_base__utility__make_optional_u63(true, 0u);
}
// -------- func lzma.decoder.workbuf_len
@@ -52189,7 +52317,13 @@ wuffs_lzma__decoder__workbuf_len(
return wuffs_base__utility__empty_range_ii_u64();
}
return wuffs_base__utility__make_range_ii_u64(0u, 0u);
uint64_t v_m = 0;
if (self->private_impl.f_dict_size == 0u) {
return wuffs_base__utility__make_range_ii_u64(0u, 0u);
}
v_m = (((uint64_t)(self->private_impl.f_dict_size)) + 273u);
return wuffs_base__utility__make_range_ii_u64(v_m, v_m);
}
// -------- func lzma.decoder.transform_io
@@ -52222,22 +52356,59 @@ wuffs_lzma__decoder__transform_io(
self->private_impl.active_coroutine = 0;
wuffs_base__status status = wuffs_base__make_status(NULL);
wuffs_base__status v_status = wuffs_base__make_status(NULL);
uint64_t v_mark = 0;
wuffs_base__status v_dti_status = wuffs_base__make_status(NULL);
wuffs_base__status v_ah_status = wuffs_base__make_status(NULL);
uint8_t* iop_a_dst = NULL;
uint8_t* io0_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
uint8_t* io1_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
uint8_t* io2_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
if (a_dst && a_dst->data.ptr) {
io0_a_dst = a_dst->data.ptr;
io1_a_dst = io0_a_dst + a_dst->meta.wi;
iop_a_dst = io1_a_dst;
io2_a_dst = io0_a_dst + a_dst->data.len;
if (a_dst->meta.closed) {
io2_a_dst = iop_a_dst;
}
}
uint32_t coro_susp_point = self->private_impl.p_transform_io;
switch (coro_susp_point) {
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0;
while (true) {
v_mark = ((uint64_t)(iop_a_dst - io0_a_dst));
{
wuffs_base__status t_0 = wuffs_lzma__decoder__do_transform_io(self, a_dst, a_src);
v_status = t_0;
if (a_dst) {
a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr));
}
wuffs_base__status t_0 = wuffs_lzma__decoder__do_transform_io(self, a_dst, a_src, a_workbuf);
v_dti_status = t_0;
if (a_dst) {
iop_a_dst = a_dst->data.ptr + a_dst->meta.wi;
}
}
if ((v_status.repr == wuffs_base__suspension__short_read) && (a_src && a_src->meta.closed)) {
if ( ! wuffs_base__status__is_suspension(&v_dti_status)) {
status = v_dti_status;
if (wuffs_base__status__is_error(&status)) {
goto exit;
} else if (wuffs_base__status__is_suspension(&status)) {
status = wuffs_base__make_status(wuffs_base__error__cannot_return_a_suspension);
goto exit;
}
goto ok;
} else if ((v_dti_status.repr == wuffs_base__suspension__short_read) && (a_src && a_src->meta.closed)) {
status = wuffs_base__make_status(wuffs_lzma__error__truncated_input);
goto exit;
}
status = v_status;
v_ah_status = wuffs_lzma__decoder__add_history(self, wuffs_private_impl__io__since(v_mark, ((uint64_t)(iop_a_dst - io0_a_dst)), io0_a_dst), a_workbuf);
if (wuffs_base__status__is_error(&v_ah_status)) {
status = v_ah_status;
goto exit;
}
status = v_dti_status;
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(1);
}
@@ -52253,6 +52424,10 @@ wuffs_lzma__decoder__transform_io(
goto exit;
exit:
if (a_dst && a_dst->data.ptr) {
a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr));
}
if (wuffs_base__status__is_error(&status)) {
self->private_impl.magic = WUFFS_BASE__DISABLED;
}
@@ -52266,7 +52441,8 @@ static wuffs_base__status
wuffs_lzma__decoder__do_transform_io(
wuffs_lzma__decoder* self,
wuffs_base__io_buffer* a_dst,
wuffs_base__io_buffer* a_src) {
wuffs_base__io_buffer* a_src,
wuffs_base__slice_u8 a_workbuf) {
wuffs_base__status status = wuffs_base__make_status(NULL);
uint8_t v_header_byte = 0;
@@ -52594,6 +52770,10 @@ wuffs_lzma__decoder__do_transform_io(
goto exit;
}
}
while (((uint64_t)(a_workbuf.len)) < (((uint64_t)(self->private_impl.f_dict_size)) + 273u)) {
status = wuffs_base__make_status(wuffs_base__suspension__short_workbuf);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(17);
}
self->private_impl.f_lzma2_encoded_length_have = 0u;
while (true) {
v_dmark = ((uint64_t)(iop_a_dst - io0_a_dst));
@@ -52605,7 +52785,7 @@ wuffs_lzma__decoder__do_transform_io(
if (a_src) {
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
}
wuffs_base__status t_8 = wuffs_lzma__decoder__decode_bitstream(self, a_dst, a_src);
wuffs_base__status t_8 = wuffs_lzma__decoder__decode_bitstream(self, a_dst, a_src, a_workbuf);
v_status = t_8;
if (a_dst) {
iop_a_dst = a_dst->data.ptr + a_dst->meta.wi;
@@ -52620,7 +52800,7 @@ wuffs_lzma__decoder__do_transform_io(
break;
}
status = v_status;
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(17);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(18);
}
if (self->private_impl.f_format_extension == 0u) {
break;
@@ -52660,7 +52840,8 @@ static wuffs_base__status
wuffs_lzma__decoder__decode_bitstream(
wuffs_lzma__decoder* self,
wuffs_base__io_buffer* a_dst,
wuffs_base__io_buffer* a_src) {
wuffs_base__io_buffer* a_src,
wuffs_base__slice_u8 a_workbuf) {
wuffs_base__status status = wuffs_base__make_status(NULL);
uint32_t coro_susp_point = self->private_impl.p_decode_bitstream;
@@ -52668,7 +52849,7 @@ wuffs_lzma__decoder__decode_bitstream(
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0;
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1);
status = wuffs_lzma__decoder__decode_bitstream_slow(self, a_dst, a_src);
status = wuffs_lzma__decoder__decode_bitstream_slow(self, a_dst, a_src, a_workbuf);
if (status.repr) {
goto suspend;
}
@@ -66797,7 +66978,7 @@ wuffs_xz__decoder__workbuf_len(
return wuffs_base__utility__empty_range_ii_u64();
}
return wuffs_base__utility__make_range_ii_u64(0u, 0u);
return wuffs_lzma__decoder__workbuf_len(&self->private_data.f_lzma);
}
// -------- func xz.decoder.transform_io
@@ -67097,7 +67278,7 @@ wuffs_xz__decoder__do_transform_io(
if (a_src) {
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
}
wuffs_base__status t_4 = wuffs_lzma__decoder__transform_io(&self->private_data.f_lzma, a_dst, a_src, wuffs_base__utility__empty_slice_u8());
wuffs_base__status t_4 = wuffs_lzma__decoder__transform_io(&self->private_data.f_lzma, a_dst, a_src, a_workbuf);
v_status = t_4;
if (a_dst) {
iop_a_dst = a_dst->data.ptr + a_dst->meta.wi;
+85 -9
View File
@@ -18,7 +18,7 @@
// its LZMA implementation looks relatively verbose and repetitive (even though
// it's equivalent to the C/C++ code, at the machine code level).
pri func decoder.decode_bitstream_slow?(dst: base.io_writer, src: base.io_reader) {
pri func decoder.decode_bitstream_slow?(dst: base.io_writer, src: base.io_reader, workbuf: roslice base.u8) {
var c8 : base.u8
var bits : base.u32
@@ -68,6 +68,17 @@ pri func decoder.decode_bitstream_slow?(dst: base.io_writer, src: base.io_reader
var index_large_dist : base.u32[..= 15]
// dist is part of the Lempel-Ziv (length, distance) pair. When dist is at
// or below the args.dst history length then the LZ copy can be resolved
// just from args.dst (and adj_dist and wb_index are unused). Otherwise, if
// e.g. dist was 100 and args.dst.history_length() was 40 then adj_dist
// would be 60: the LZ copy starts 60 bytes before the end of the
// args.workbuf ringbuffer. wb_index is that start index, equal to
// ((this.dict_workbuf_index ~mod- adj_dist) modulo this.dict_size).
var dist : base.u32
var adj_dist : base.u32
var wb_index : base.u64
// Initialize the bitstream.
c8 = args.src.read_u8?()
if c8 <> 0x00 {
@@ -804,24 +815,89 @@ pri func decoder.decode_bitstream_slow?(dst: base.io_writer, src: base.io_reader
// AlgOve90 labelDoTheLZCopy:
// AlgOve94 emitCopy(len, mrud[0])
assert len >= 1
if (((rep0 + 1) as base.u64) > (pos ~mod+ this.lzma2_prior_decoded_length)) or
(((rep0 + 1) as base.u64) > (this.dict_size as base.u64)) {
dist = rep0 + 1
assert dist >= 1
if ((dist as base.u64) > (pos ~mod+ this.lzma2_prior_decoded_length)) or
((dist as base.u64) > (this.dict_size as base.u64)) {
return "#bad distance"
}
pos ~mod+= len as base.u64
while (len as base.u64) > args.dst.length(),
// 274 is at least (len + 1). A limited_copy_u32_from_slice call below
// passes "up_to: len + 1". It's simpler if it's always a full copy.
while 274 > args.dst.length(),
inv len >= 1,
post (len as base.u64) <= args.dst.length(),
inv dist >= 1,
post 274 <= args.dst.length(),
{
yield? base."$short write"
} endwhile
if ((rep0 + 1) as base.u64) > args.dst.history_length() {
return base."#insufficient history"
assert (len as base.u64) <= args.dst.length() via "a <= b: a <= c; c <= b"(c: 274)
// Copy from the args.workbuf history, if necessary.
if (dist as base.u64) > args.dst.history_length() {
adj_dist = (((dist as base.u64) - args.dst.history_length()) & 0xFFFF_FFFF) as base.u32
if adj_dist > this.dict_seen {
return "#bad distance"
}
wb_index = (this.dict_workbuf_index as base.u64) ~mod- (adj_dist as base.u64)
while wb_index >= 0x8000_0000_0000_0000,
inv dist >= 1,
{
wb_index ~mod+= this.dict_size as base.u64
} endwhile
if wb_index > args.workbuf.length() {
return "#internal error: inconsistent dictionary state"
}
if len < adj_dist {
// Copy len bytes of history and one more byte for the later
// half of the two-byte cusp, all from args.workbuf.
args.dst.limited_copy_u32_from_slice!(up_to: len + 1, s: args.workbuf[wb_index ..])
if not args.dst.can_undo_byte() {
return "#internal error: inconsistent dictionary state"
}
match_byte = args.dst.peek_undo_byte()
args.dst.undo_byte!()
if not args.dst.can_undo_byte() {
return "#internal error: inconsistent dictionary state"
}
prev_byte = args.dst.peek_undo_byte()
continue.outer
} else if len == adj_dist {
// Like above, copy len bytes of history and one more byte for
// the later half of the two-byte cusp, but the copy-source
// crosses over from args.workbuf to args.dst.
args.dst.limited_copy_u32_from_slice!(up_to: len, s: args.workbuf[wb_index ..])
args.dst.limited_copy_u32_from_history!(up_to: 1, distance: dist)
if not args.dst.can_undo_byte() {
return "#internal error: inconsistent dictionary state"
}
match_byte = args.dst.peek_undo_byte()
args.dst.undo_byte!()
if not args.dst.can_undo_byte() {
return "#internal error: inconsistent dictionary state"
}
prev_byte = args.dst.peek_undo_byte()
continue.outer
}
args.dst.limited_copy_u32_from_slice!(up_to: adj_dist, s: args.workbuf[wb_index ..])
len -= adj_dist
assert len >= 1
if ((len as base.u64) > args.dst.length()) or
((dist as base.u64) > args.dst.history_length()) {
return "#internal error: inconsistent dictionary state"
}
}
assert (rep0 + 1) >= 1
// Copy from the args.dst history.
assert len >= 1
assert dist >= 1
assert (len as base.u64) <= args.dst.length()
assert (dist as base.u64) <= args.dst.history_length()
match_cusp = args.dst.limited_copy_u32_from_history_fast_return_cusp!(
up_to: len, distance: (rep0 + 1))
up_to: len, distance: dist)
match_byte = (match_cusp >> 8) as base.u8
prev_byte = (match_cusp & 0xFF) as base.u8
} endwhile.outer
+116 -15
View File
@@ -18,9 +18,10 @@ pub status "#truncated input"
pub status "#unsupported decoded length"
pub status "#unsupported properties"
pub const DECODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE : base.u64 = 0xFFFF_FFFF
pri status "#internal error: inconsistent dictionary state"
pub const DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE : base.u64 = 0
pub const DECODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE : base.u64 = 0
pub const DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE : base.u64 = 0xFFFF_FFFF + 273
pub struct decoder? implements base.io_transformer(
// In theory, lc ranges in [0 ..= 8]. In practice, to keep the
@@ -36,8 +37,19 @@ pub struct decoder? implements base.io_transformer(
format_extension : base.u32,
// dict_size is the nominal size of the args.workbuf ringbuffer that
// holds historical decoded bytes. The workbuf_len is a little larger:
// see "ML" below.
dict_size : base.u32,
// dict_workbuf_index indexes the args.workbuf ringbuffer, pointing to
// where add_history will next write to.
dict_workbuf_index : base.u32,
// dict_seen is the cumulative sum (saturating at 0xFFFF_FFFF) of the
// number of bytes passed to add_history.
dict_seen : base.u32,
decoded_length : base.u64,
lzma2_prior_decoded_length : base.u64,
@@ -166,6 +178,79 @@ pub struct decoder? implements base.io_transformer(
// https://github.com/jljusten/LZMA-SDK/blob/f287b63f6bb8b88e281f18a9295340c732245167/Source/LzmaDecode.h#L48
)
pri func decoder.add_history!(hist: slice base.u8, workbuf: slice base.u8) base.status {
var dict_workbuf_index : base.u64[..= 0xFFFF_FFFF]
var dict_size : base.u64[..= 0xFFFF_FFFF]
var hist_length : base.u64
var s : slice base.u8
var n_copied : base.u64
var n : base.u64
dict_workbuf_index = this.dict_workbuf_index as base.u64
dict_size = this.dict_size as base.u64
if args.hist.length() == 0 {
return ok
}
// 273 is ML, where ML is the Maximum Length of an LZMA Lempel-Ziv (length,
// distance) pair. After an add_history call, args.workbuf[dict_size ..
// (dict_size + ML)] duplicates args.workbuf[.. ML]. This simplifies
// copying up to (ML + 1) bytes from the ringbuffer, as there is no need to
// split the copy around the dict_size boundary.
//
// This technique is similar to the one used in the std/deflate package
// (although that package uses a different value of ML, 258). That package
// also extends the nominal dict_size by (ML - 1), not ML as is done here,
// because std/deflate copies up to ML bytes, but std/lzma copies up to (ML
// + 1) bytes. std/lzma also needs to know the later half of the two-byte
// cusp: the first byte after the copied history.
if args.workbuf.length() < (dict_size + 273) {
return base."#bad workbuf length"
}
assert args.workbuf.length() >= dict_size via "a >= b: a >= (b + c); 0 <= c"(c: 273)
assert dict_size <= args.workbuf.length() via "a <= b: b >= a"()
hist_length = args.hist.length()
if hist_length > 0xFFFF_FFFF {
this.dict_seen = 0xFFFF_FFFF
} else {
this.dict_seen ~sat+= hist_length as base.u32
}
s = args.hist
if s.length() >= dict_size {
s = s.suffix(up_to: dict_size)
args.workbuf.copy_from_slice!(s: s)
this.dict_workbuf_index = 0
} else if dict_workbuf_index > dict_size {
return "#internal error: inconsistent dictionary state"
} else {
n_copied = args.workbuf[dict_workbuf_index .. dict_size].copy_from_slice!(s: s)
if n_copied < s.length() {
n = args.workbuf.copy_from_slice!(s: s[n_copied ..])
this.dict_workbuf_index = (n & 0xFFFF_FFFF) as base.u32
} else {
n = dict_workbuf_index ~mod+ n_copied
if n < dict_size {
assert n < 0xFFFF_FFFF via "a < b: a < c; c <= b"(c: dict_size)
this.dict_workbuf_index = n as base.u32
} else {
this.dict_workbuf_index = 0
}
}
}
// Have the tail duplicate the head.
if (273 > dict_size) or (dict_size > args.workbuf.length()) {
return "#internal error: inconsistent dictionary state"
}
assert 273 <= args.workbuf.length() via "a <= b: a <= c; c <= b"(c: dict_size)
args.workbuf[dict_size ..].copy_from_slice!(s: args.workbuf[.. 273])
return ok
}
pub func decoder.get_quirk(key: base.u32) base.u64 {
if args.key == QUIRK_FORMAT_EXTENSION {
return this.format_extension as base.u64
@@ -196,29 +281,41 @@ pub func decoder.set_quirk!(key: base.u32, value: base.u64) base.status {
}
pub func decoder.dst_history_retain_length() base.optional_u63 {
if this.dict_size == 0 {
return this.util.make_optional_u63(has_value: false, value: 0)
}
return this.util.make_optional_u63(has_value: true, value: this.dict_size as base.u64)
return this.util.make_optional_u63(has_value: true, value: 0)
}
pub func decoder.workbuf_len() base.range_ii_u64 {
return this.util.make_range_ii_u64(min_incl: 0, max_incl: 0)
var m : base.u64
if this.dict_size == 0 {
return this.util.make_range_ii_u64(min_incl: 0, max_incl: 0)
}
m = (this.dict_size as base.u64) + 273
return this.util.make_range_ii_u64(min_incl: m, max_incl: m)
}
pub func decoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf: slice base.u8) {
var status : base.status
var mark : base.u64
var dti_status : base.status
var ah_status : base.status
while true {
status =? this.do_transform_io?(dst: args.dst, src: args.src)
if (status == base."$short read") and args.src.is_closed() {
mark = args.dst.mark()
dti_status =? this.do_transform_io?(dst: args.dst, src: args.src, workbuf: args.workbuf)
if not dti_status.is_suspension() {
return dti_status
} else if (dti_status == base."$short read") and args.src.is_closed() {
return "#truncated input"
}
yield? status
ah_status = this.add_history!(hist: args.dst.since(mark: mark), workbuf: args.workbuf)
if ah_status.is_error() {
return ah_status
}
yield? dti_status
} endwhile
}
pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader) {
pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, workbuf: roslice base.u8) {
var header_byte : base.u8
var c32 : base.u32
var prop_byte : base.u8
@@ -348,11 +445,15 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader) {
}
}
while args.workbuf.length() < ((this.dict_size as base.u64) + 273) {
yield? base."$short workbuf"
} endwhile
this.lzma2_encoded_length_have = 0
while true {
dmark = args.dst.mark()
smark = args.src.mark()
status =? this.decode_bitstream?(dst: args.dst, src: args.src)
status =? this.decode_bitstream?(dst: args.dst, src: args.src, workbuf: args.workbuf)
this.lzma2_encoded_length_have ~sat+= args.src.count_since(mark: smark)
this.lzma2_prior_decoded_length ~sat+= args.dst.count_since(mark: dmark)
if status.is_ok() {
@@ -369,8 +470,8 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader) {
} endwhile
}
pri func decoder.decode_bitstream?(dst: base.io_writer, src: base.io_reader) {
this.decode_bitstream_slow?(dst: args.dst, src: args.src)
pri func decoder.decode_bitstream?(dst: base.io_writer, src: base.io_reader, workbuf: roslice base.u8) {
this.decode_bitstream_slow?(dst: args.dst, src: args.src, workbuf: args.workbuf)
}
// initialize_probs sets all of the probs_foo[i][j] values to 1024.
+4 -5
View File
@@ -26,9 +26,8 @@ pub status "#truncated input"
pub status "#unsupported checksum algorithm"
pub status "#unsupported filter"
pub const DECODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE : base.u64 = 0xFFFF_FFFF
pub const DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE : base.u64 = 0
pub const DECODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE : base.u64 = 0
pub const DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE : base.u64 = 0xFFFF_FFFF + 273
pub struct decoder? implements base.io_transformer(
filters : array[3] base.u32,
@@ -79,7 +78,7 @@ pub func decoder.dst_history_retain_length() base.optional_u63 {
}
pub func decoder.workbuf_len() base.range_ii_u64 {
return this.util.make_range_ii_u64(min_incl: 0, max_incl: 0)
return this.lzma.workbuf_len()
}
pub func decoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf: slice base.u8) {
@@ -177,7 +176,7 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work
while true {
dmark = args.dst.mark()
smark = args.src.mark()
status =? this.lzma.transform_io?(dst: args.dst, src: args.src, workbuf: this.util.empty_slice_u8())
status =? this.lzma.transform_io?(dst: args.dst, src: args.src, workbuf: args.workbuf)
compressed_size ~mod+= args.src.count_since(mark: smark)
uncompressed_size ~mod+= args.dst.count_since(mark: dmark)
+1 -3
View File
@@ -137,14 +137,12 @@ test_wuffs_xz_decode_enwik5() {
UINT64_MAX);
}
/* TODO
const char* //
test_wuffs_xz_decode_one_byte_reads() {
CHECK_FOCUS(__func__);
return do_test_io_buffers(wuffs_xz_decode, &g_xz_romeo_delta1_gt, UINT64_MAX,
1);
}
*/
const char* //
test_wuffs_xz_decode_romeo() {
@@ -203,7 +201,7 @@ proc g_tests[] = {
test_wuffs_xz_decode_enwik5,
test_wuffs_xz_decode_interface,
// TODO test_wuffs_xz_decode_one_byte_reads,
test_wuffs_xz_decode_one_byte_reads,
test_wuffs_xz_decode_romeo,
#ifdef WUFFS_MIMIC