mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-17 16:32:11 +01:00
std/brotli: add QUIRK_REPORT_METADATA
The decoder can now report metadata blocks, like libbrotlidec's BrotliDecoderSetMetadataCallbacks: transform_io returns "@metadata reported", metadata_length gives the block's length and the block's contents are written to dst. The differential fuzzer compares the reported metadata with libbrotlidec's callbacks. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UZ978Fjdsyig4v3wYAZRg5
This commit is contained in:
5 files changed
+774
-27
No files matched your search
@@ -9253,6 +9253,8 @@ extern const char wuffs_brotli__error__unsupported_serialized_encoder_dictionary
|
||||
|
||||
#define WUFFS_BROTLI__QUIRK_LARGE_WINDOW 716673024u
|
||||
|
||||
#define WUFFS_BROTLI__QUIRK_REPORT_METADATA 716673034u
|
||||
|
||||
#define WUFFS_BROTLI__ENCODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE 0u
|
||||
|
||||
#define WUFFS_BROTLI__QUIRK_ENCODER_QUALITY 716673025u
|
||||
@@ -9383,6 +9385,11 @@ wuffs_brotli__decoder__set_quirk(
|
||||
uint32_t a_key,
|
||||
uint64_t a_value);
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
WUFFS_BASE__MAYBE_STATIC uint64_t
|
||||
wuffs_brotli__decoder__metadata_length(
|
||||
const wuffs_brotli__decoder* self);
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
WUFFS_BASE__MAYBE_STATIC wuffs_base__optional_u63
|
||||
wuffs_brotli__decoder__dst_history_retain_length(
|
||||
@@ -9612,6 +9619,10 @@ struct wuffs_brotli__decoder__struct {
|
||||
uint32_t f_cmd_state;
|
||||
bool f_quirk_large_window;
|
||||
bool f_large_window;
|
||||
bool f_quirk_report_metadata;
|
||||
uint32_t f_md_length;
|
||||
bool f_md_note_pending;
|
||||
bool f_md_copying;
|
||||
bool f_started;
|
||||
uint64_t f_dict_total;
|
||||
uint32_t f_num_chunks;
|
||||
@@ -9634,6 +9645,7 @@ struct wuffs_brotli__decoder__struct {
|
||||
|
||||
uint32_t p_transform_io;
|
||||
uint32_t p_do_transform_io;
|
||||
uint32_t p_report_metadata;
|
||||
uint32_t p_read_bits;
|
||||
uint32_t p_read_bits32;
|
||||
uint32_t p_read_symbol;
|
||||
@@ -9852,6 +9864,11 @@ struct wuffs_brotli__decoder__struct {
|
||||
return wuffs_brotli__decoder__set_quirk(this, a_key, a_value);
|
||||
}
|
||||
|
||||
inline uint64_t
|
||||
metadata_length() const {
|
||||
return wuffs_brotli__decoder__metadata_length(this);
|
||||
}
|
||||
|
||||
inline wuffs_base__optional_u63
|
||||
dst_history_retain_length() const {
|
||||
return wuffs_brotli__decoder__dst_history_retain_length(this);
|
||||
@@ -77384,10 +77401,18 @@ WUFFS_BASE__GENERATED_C_CODE
|
||||
static wuffs_base__status
|
||||
wuffs_brotli__decoder__do_transform_io(
|
||||
wuffs_brotli__decoder* self,
|
||||
wuffs_base__io_buffer* a_dst,
|
||||
wuffs_base__io_buffer* a_src,
|
||||
wuffs_base__slice_u8 a_workbuf,
|
||||
wuffs_base__slice_u8 a_dictbuf);
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
static wuffs_base__status
|
||||
wuffs_brotli__decoder__report_metadata(
|
||||
wuffs_brotli__decoder* self,
|
||||
wuffs_base__io_buffer* a_dst,
|
||||
wuffs_base__io_buffer* a_src);
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
static wuffs_base__empty_struct
|
||||
wuffs_brotli__decoder__initialize_codes(
|
||||
@@ -81135,6 +81160,8 @@ wuffs_brotli__decoder__get_quirk(
|
||||
|
||||
if ((a_key == 716673024u) && self->private_impl.f_quirk_large_window) {
|
||||
return 1u;
|
||||
} else if ((a_key == 716673034u) && self->private_impl.f_quirk_report_metadata) {
|
||||
return 1u;
|
||||
}
|
||||
return 0u;
|
||||
}
|
||||
@@ -81163,10 +81190,33 @@ wuffs_brotli__decoder__set_quirk(
|
||||
}
|
||||
self->private_impl.f_quirk_large_window = (a_value > 0u);
|
||||
return wuffs_base__make_status(NULL);
|
||||
} else if (a_key == 716673034u) {
|
||||
if (self->private_impl.f_started) {
|
||||
return wuffs_base__make_status(wuffs_base__error__bad_call_sequence);
|
||||
}
|
||||
self->private_impl.f_quirk_report_metadata = (a_value > 0u);
|
||||
return wuffs_base__make_status(NULL);
|
||||
}
|
||||
return wuffs_base__make_status(wuffs_base__error__unsupported_option);
|
||||
}
|
||||
|
||||
// -------- func brotli.decoder.metadata_length
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
WUFFS_BASE__MAYBE_STATIC uint64_t
|
||||
wuffs_brotli__decoder__metadata_length(
|
||||
const wuffs_brotli__decoder* self) {
|
||||
if (!self) {
|
||||
return 0;
|
||||
}
|
||||
if ((self->private_impl.magic != WUFFS_BASE__MAGIC) &&
|
||||
(self->private_impl.magic != WUFFS_BASE__DISABLED)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return ((uint64_t)(self->private_impl.f_md_length));
|
||||
}
|
||||
|
||||
// -------- func brotli.decoder.dst_history_retain_length
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
@@ -81249,7 +81299,11 @@ wuffs_brotli__decoder__transform_io(
|
||||
goto exit;
|
||||
}
|
||||
{
|
||||
wuffs_base__status t_0 = wuffs_brotli__decoder__do_transform_io(self, a_src, wuffs_base__slice_u8__subslice_i(a_workbuf, self->private_impl.f_dict_total), wuffs_base__slice_u8__subslice_j(a_workbuf, self->private_impl.f_dict_total));
|
||||
wuffs_base__status t_0 = wuffs_brotli__decoder__do_transform_io(self,
|
||||
a_dst,
|
||||
a_src,
|
||||
wuffs_base__slice_u8__subslice_i(a_workbuf, self->private_impl.f_dict_total),
|
||||
wuffs_base__slice_u8__subslice_j(a_workbuf, self->private_impl.f_dict_total));
|
||||
v_status = t_0;
|
||||
}
|
||||
if (self->private_impl.f_dict_total > ((uint64_t)(a_workbuf.len))) {
|
||||
@@ -81270,7 +81324,11 @@ wuffs_brotli__decoder__transform_io(
|
||||
status = wuffs_base__make_status(NULL);
|
||||
goto ok;
|
||||
} else if (v_status.repr == wuffs_base__suspension__short_write) {
|
||||
if (wuffs_brotli__decoder__is_fully_flushed(self)) {
|
||||
if (self->private_impl.f_md_note_pending) {
|
||||
self->private_impl.f_md_note_pending = false;
|
||||
status = wuffs_base__make_status(wuffs_base__note__metadata_reported);
|
||||
goto ok;
|
||||
} else if (wuffs_brotli__decoder__is_fully_flushed(self) && ! self->private_impl.f_md_copying) {
|
||||
continue;
|
||||
}
|
||||
} else if (v_status.repr == wuffs_base__suspension__short_read) {
|
||||
@@ -81367,6 +81425,7 @@ WUFFS_BASE__GENERATED_C_CODE
|
||||
static wuffs_base__status
|
||||
wuffs_brotli__decoder__do_transform_io(
|
||||
wuffs_brotli__decoder* self,
|
||||
wuffs_base__io_buffer* a_dst,
|
||||
wuffs_base__io_buffer* a_src,
|
||||
wuffs_base__slice_u8 a_workbuf,
|
||||
wuffs_base__slice_u8 a_dictbuf) {
|
||||
@@ -81435,15 +81494,29 @@ wuffs_brotli__decoder__do_transform_io(
|
||||
}
|
||||
}
|
||||
if (self->private_impl.f_is_metadata) {
|
||||
self->private_data.s_do_transform_io.scratch = ((uint64_t)(self->private_impl.f_mb_remaining));
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(3);
|
||||
if (self->private_data.s_do_transform_io.scratch > ((uint64_t)(io2_a_src - iop_a_src))) {
|
||||
self->private_data.s_do_transform_io.scratch -= ((uint64_t)(io2_a_src - iop_a_src));
|
||||
iop_a_src = io2_a_src;
|
||||
status = wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
goto suspend;
|
||||
if (self->private_impl.f_quirk_report_metadata) {
|
||||
if (a_src) {
|
||||
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
|
||||
}
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(3);
|
||||
status = wuffs_brotli__decoder__report_metadata(self, a_dst, a_src);
|
||||
if (a_src) {
|
||||
iop_a_src = a_src->data.ptr + a_src->meta.ri;
|
||||
}
|
||||
if (status.repr) {
|
||||
goto suspend;
|
||||
}
|
||||
} else {
|
||||
self->private_data.s_do_transform_io.scratch = ((uint64_t)(self->private_impl.f_mb_remaining));
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(4);
|
||||
if (self->private_data.s_do_transform_io.scratch > ((uint64_t)(io2_a_src - iop_a_src))) {
|
||||
self->private_data.s_do_transform_io.scratch -= ((uint64_t)(io2_a_src - iop_a_src));
|
||||
iop_a_src = io2_a_src;
|
||||
status = wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
goto suspend;
|
||||
}
|
||||
iop_a_src += self->private_data.s_do_transform_io.scratch;
|
||||
}
|
||||
iop_a_src += self->private_data.s_do_transform_io.scratch;
|
||||
self->private_impl.f_mb_remaining = 0u;
|
||||
} else if (self->private_impl.f_mb_remaining > 0u) {
|
||||
wuffs_brotli__decoder__calculate_ring_buffer_size(self);
|
||||
@@ -81451,7 +81524,7 @@ wuffs_brotli__decoder__do_transform_io(
|
||||
if (a_src) {
|
||||
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
|
||||
}
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(4);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(5);
|
||||
status = wuffs_brotli__decoder__copy_uncompressed(self, a_src, a_workbuf);
|
||||
if (a_src) {
|
||||
iop_a_src = a_src->data.ptr + a_src->meta.ri;
|
||||
@@ -81463,7 +81536,7 @@ wuffs_brotli__decoder__do_transform_io(
|
||||
if (a_src) {
|
||||
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
|
||||
}
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(5);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(6);
|
||||
status = wuffs_brotli__decoder__decode_compressed_metablock(self, a_src, a_workbuf, a_dictbuf);
|
||||
if (a_src) {
|
||||
iop_a_src = a_src->data.ptr + a_src->meta.ri;
|
||||
@@ -81507,6 +81580,92 @@ wuffs_brotli__decoder__do_transform_io(
|
||||
return status;
|
||||
}
|
||||
|
||||
// -------- func brotli.decoder.report_metadata
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
static wuffs_base__status
|
||||
wuffs_brotli__decoder__report_metadata(
|
||||
wuffs_brotli__decoder* self,
|
||||
wuffs_base__io_buffer* a_dst,
|
||||
wuffs_base__io_buffer* a_src) {
|
||||
wuffs_base__status status = wuffs_base__make_status(NULL);
|
||||
|
||||
uint32_t v_n = 0;
|
||||
|
||||
uint8_t* iop_a_dst = NULL;
|
||||
uint8_t* io0_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
|
||||
uint8_t* io1_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
|
||||
uint8_t* io2_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
|
||||
if (a_dst && a_dst->data.ptr) {
|
||||
io0_a_dst = a_dst->data.ptr;
|
||||
io1_a_dst = io0_a_dst + a_dst->meta.wi;
|
||||
iop_a_dst = io1_a_dst;
|
||||
io2_a_dst = io0_a_dst + a_dst->data.len;
|
||||
if (a_dst->meta.closed) {
|
||||
io2_a_dst = iop_a_dst;
|
||||
}
|
||||
}
|
||||
const uint8_t* iop_a_src = NULL;
|
||||
const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
|
||||
const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
|
||||
const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
|
||||
if (a_src && a_src->data.ptr) {
|
||||
io0_a_src = a_src->data.ptr;
|
||||
io1_a_src = io0_a_src + a_src->meta.ri;
|
||||
iop_a_src = io1_a_src;
|
||||
io2_a_src = io0_a_src + a_src->meta.wi;
|
||||
}
|
||||
|
||||
uint32_t coro_susp_point = self->private_impl.p_report_metadata;
|
||||
switch (coro_susp_point) {
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0;
|
||||
|
||||
while ( ! wuffs_brotli__decoder__is_fully_flushed(self)) {
|
||||
status = wuffs_base__make_status(wuffs_base__suspension__short_write);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(1);
|
||||
}
|
||||
self->private_impl.f_md_length = wuffs_base__u32__min(self->private_impl.f_mb_remaining, 16777216u);
|
||||
self->private_impl.f_md_note_pending = true;
|
||||
status = wuffs_base__make_status(wuffs_base__suspension__short_write);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(2);
|
||||
self->private_impl.f_md_copying = true;
|
||||
while (self->private_impl.f_mb_remaining > 0u) {
|
||||
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0u) {
|
||||
status = wuffs_base__make_status(wuffs_base__suspension__short_write);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(3);
|
||||
continue;
|
||||
} else if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) {
|
||||
status = wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(4);
|
||||
continue;
|
||||
}
|
||||
v_n = wuffs_private_impl__io_writer__limited_copy_u32_from_reader(
|
||||
&iop_a_dst, io2_a_dst,((uint32_t)(self->private_impl.f_mb_remaining)), &iop_a_src, io2_a_src);
|
||||
self->private_impl.f_mb_remaining -= v_n;
|
||||
}
|
||||
self->private_impl.f_md_copying = false;
|
||||
|
||||
ok:
|
||||
self->private_impl.p_report_metadata = 0;
|
||||
goto exit;
|
||||
}
|
||||
|
||||
goto suspend;
|
||||
suspend:
|
||||
self->private_impl.p_report_metadata = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0;
|
||||
|
||||
goto exit;
|
||||
exit:
|
||||
if (a_dst && a_dst->data.ptr) {
|
||||
a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr));
|
||||
}
|
||||
if (a_src && a_src->data.ptr) {
|
||||
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
// -------- func brotli.decoder.initialize_codes
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
|
||||
@@ -31,7 +31,9 @@
|
||||
// mostly 1 to 7 bytes), with a random large-window quirk, random
|
||||
// (raw and serialized) dictionaries and a random workbuf length (at
|
||||
// or above workbuf_len, or exactly the stream's minimum, or one byte
|
||||
// less than that, which must fail).
|
||||
// less than that, which must fail). Some decodes (with a large dst)
|
||||
// also enable QUIRK_REPORT_METADATA and libbrotlidec's metadata
|
||||
// callbacks and check that both report the same metadata blocks.
|
||||
//
|
||||
// -enc encodes random inputs with random parameters (quality, lgwin,
|
||||
// large window, lgblock, mode, size hint, npostfix, ndirect,
|
||||
@@ -1068,8 +1070,41 @@ typedef struct {
|
||||
size_t in_used;
|
||||
const char* wuffs_status;
|
||||
uint64_t calls;
|
||||
size_t md_n; // With report_md: the metadata bytes (see g_wmd and g_cmd).
|
||||
} dec_result;
|
||||
|
||||
// With dec_config.report_md, the Wuffs decoder (QUIRK_REPORT_METADATA) and
|
||||
// libbrotlidec (BrotliDecoderSetMetadataCallbacks) collect each metadata
|
||||
// block (its length as 4 little-endian bytes, then its contents) in g_wmd and
|
||||
// g_cmd. A block's contents can be up to 16 MiB long.
|
||||
#define MD_CAP ((size_t)40 << 20)
|
||||
#define MAX_OUT (((size_t)2 << 20) + 4096)
|
||||
static uint8_t* g_wmd;
|
||||
static uint8_t* g_cmd;
|
||||
static size_t g_cmd_n;
|
||||
static bool g_cmd_overflow;
|
||||
|
||||
static void //
|
||||
c_md_start(void* opaque, size_t size) {
|
||||
if (MD_CAP - g_cmd_n < 4) {
|
||||
g_cmd_overflow = true;
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < 4; i++) {
|
||||
g_cmd[g_cmd_n++] = (uint8_t)(size >> (8 * i));
|
||||
}
|
||||
}
|
||||
|
||||
static void //
|
||||
c_md_chunk(void* opaque, const uint8_t* data, size_t size) {
|
||||
if (MD_CAP - g_cmd_n < size) {
|
||||
g_cmd_overflow = true;
|
||||
return;
|
||||
}
|
||||
memcpy(g_cmd + g_cmd_n, data, size);
|
||||
g_cmd_n += size;
|
||||
}
|
||||
|
||||
static int //
|
||||
wuffs_status_class(const char* s) {
|
||||
static const struct {
|
||||
@@ -1143,6 +1178,7 @@ typedef struct {
|
||||
uint64_t wb_len; // The Wuffs workbuf length.
|
||||
bool expect_bad_wb; // wb_len is one byte too short for the stream.
|
||||
bool uninit; // WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED.
|
||||
bool report_md; // QUIRK_REPORT_METADATA and metadata callbacks.
|
||||
size_t out_cap;
|
||||
} dec_config;
|
||||
|
||||
@@ -1166,6 +1202,13 @@ wuffs_attach(wuffs_brotli__decoder* dec, const dec_config* dc, size_t* n_ok) {
|
||||
return status.repr;
|
||||
}
|
||||
}
|
||||
if (dc->report_md) {
|
||||
status = wuffs_brotli__decoder__set_quirk(
|
||||
dec, WUFFS_BROTLI__QUIRK_REPORT_METADATA, 1);
|
||||
if (status.repr) {
|
||||
return status.repr;
|
||||
}
|
||||
}
|
||||
*n_ok = 0;
|
||||
for (size_t i = 0; i < dc->dicts.n; i++) {
|
||||
status = wuffs_brotli__decoder__add_dictionary(
|
||||
@@ -1206,6 +1249,12 @@ wuffs_decode(const dec_config* dc,
|
||||
size_t ri = 0;
|
||||
size_t wi = 0;
|
||||
uint64_t max_calls = 3 * ((uint64_t)n + dc->out_cap) + 100;
|
||||
// With report_md, metadata bytes are written to dst (out) and then moved to
|
||||
// g_wmd. While md_remaining > 0, dst gets up to md_remaining bytes of slack
|
||||
// past lout (within the physical buffer's MAX_OUT bytes), which the
|
||||
// metadata bytes fill before any decoded bytes, so that the decoded bytes
|
||||
// still fit in lout (like libbrotlidec's, which uses no dst for metadata).
|
||||
size_t md_remaining = 0;
|
||||
while (true) {
|
||||
if (++res.calls > max_calls) {
|
||||
res.cls = CLS_BUG;
|
||||
@@ -1215,7 +1264,12 @@ wuffs_decode(const dec_config* dc,
|
||||
wuffs_base__io_buffer src =
|
||||
wuffs_base__ptr_u8__reader((uint8_t*)in, lin, lin == n);
|
||||
src.meta.ri = ri;
|
||||
wuffs_base__io_buffer dst = wuffs_base__ptr_u8__writer(out, lout);
|
||||
size_t slack = 0;
|
||||
if ((md_remaining > 0) && (lout < MAX_OUT)) {
|
||||
slack = (md_remaining < (MAX_OUT - lout)) ? md_remaining
|
||||
: (MAX_OUT - lout);
|
||||
}
|
||||
wuffs_base__io_buffer dst = wuffs_base__ptr_u8__writer(out, lout + slack);
|
||||
dst.meta.wi = wi;
|
||||
wuffs_base__status status =
|
||||
wuffs_brotli__decoder__transform_io(g_dec, &dst, &src, wb);
|
||||
@@ -1224,9 +1278,37 @@ wuffs_decode(const dec_config* dc,
|
||||
res.wuffs_status = "went backwards";
|
||||
break;
|
||||
}
|
||||
if (md_remaining > 0) {
|
||||
size_t k = dst.meta.wi - wi;
|
||||
k = (k < md_remaining) ? k : md_remaining;
|
||||
memcpy(g_wmd + res.md_n, out + wi, k);
|
||||
res.md_n += k;
|
||||
md_remaining -= k;
|
||||
memmove(out + wi, out + wi + k, dst.meta.wi - wi - k);
|
||||
dst.meta.wi -= k;
|
||||
}
|
||||
if (dst.meta.wi > lout) {
|
||||
res.cls = CLS_BUG;
|
||||
res.wuffs_status = "decoded bytes in the metadata slack";
|
||||
break;
|
||||
}
|
||||
ri = src.meta.ri;
|
||||
wi = dst.meta.wi;
|
||||
res.wuffs_status = status.repr;
|
||||
if (status.repr == wuffs_base__note__metadata_reported) {
|
||||
uint64_t len = wuffs_brotli__decoder__metadata_length(g_dec);
|
||||
if (!dc->report_md || (md_remaining > 0) || (len > (1 << 24)) ||
|
||||
(MD_CAP - res.md_n < 4 + len)) {
|
||||
res.cls = CLS_BUG;
|
||||
res.wuffs_status = "unexpected metadata report";
|
||||
break;
|
||||
}
|
||||
for (int i = 0; i < 4; i++) {
|
||||
g_wmd[res.md_n++] = (uint8_t)(len >> (8 * i));
|
||||
}
|
||||
md_remaining = (size_t)len;
|
||||
continue;
|
||||
}
|
||||
if (g_trace) {
|
||||
printf("wuffs call %" PRIu64 ": lin=%zu lout=%zu -> ri=%zu wi=%zu %s\n",
|
||||
res.calls, lin, lout, ri, wi, status.repr ? status.repr : "ok");
|
||||
@@ -1276,6 +1358,11 @@ c_decode(const dec_config* dc,
|
||||
if (dc->large_window) {
|
||||
BrotliDecoderSetParameter(s, BROTLI_DECODER_PARAM_LARGE_WINDOW, 1);
|
||||
}
|
||||
g_cmd_n = 0;
|
||||
g_cmd_overflow = false;
|
||||
if (dc->report_md) {
|
||||
BrotliDecoderSetMetadataCallbacks(s, c_md_start, c_md_chunk, NULL);
|
||||
}
|
||||
*c_attached = 0;
|
||||
for (size_t i = 0; i < dc->dicts.n; i++) {
|
||||
if (!BrotliDecoderAttachDictionary(
|
||||
@@ -1336,6 +1423,7 @@ c_decode(const dec_config* dc,
|
||||
BrotliDecoderDestroyInstance(s);
|
||||
res.out_n = wi;
|
||||
res.in_used = ri;
|
||||
res.md_n = g_cmd_overflow ? SIZE_MAX : g_cmd_n;
|
||||
return res;
|
||||
}
|
||||
|
||||
@@ -1429,6 +1517,10 @@ static uint64_t g_dec_gen_wuffs;
|
||||
static uint64_t g_dec_wuffs_shorter_output;
|
||||
static uint64_t g_dec_known_block_length;
|
||||
static uint64_t g_dec_known_saved;
|
||||
static uint64_t g_dec_report_md;
|
||||
static uint64_t g_dec_md_compared;
|
||||
static uint64_t g_dec_md_blocks;
|
||||
static uint64_t g_dec_md_inconclusive;
|
||||
|
||||
static int //
|
||||
cls_index(int cls) {
|
||||
@@ -1759,9 +1851,10 @@ run_dec(uint64_t iters, double secs) {
|
||||
const size_t tmp_cap = (8 << 20);
|
||||
uint8_t* tmp_in = xmalloc(tmp_cap);
|
||||
uint8_t* tmp_out = xmalloc(tmp_cap);
|
||||
const size_t max_out = (2 << 20) + 4096;
|
||||
uint8_t* wout = xmalloc(max_out);
|
||||
uint8_t* cout = xmalloc(max_out);
|
||||
uint8_t* wout = xmalloc(MAX_OUT);
|
||||
uint8_t* cout = xmalloc(MAX_OUT);
|
||||
g_wmd = xmalloc(MD_CAP);
|
||||
g_cmd = xmalloc(MD_CAP);
|
||||
const size_t mcap = (1 << 20);
|
||||
uint8_t* m = xmalloc(mcap);
|
||||
|
||||
@@ -1821,6 +1914,10 @@ run_dec(uint64_t iters, double secs) {
|
||||
g_dec_with_dicts += (dc.dicts.n > 0);
|
||||
dc.uninit = chance(500);
|
||||
dc.out_cap = chance(900) ? ((size_t)1 << 20) + rn(1 << 20) : rn(4096);
|
||||
// Only with a large dst: libbrotlidec reports metadata (via callbacks)
|
||||
// even when its dst is full, but Wuffs needs dst space for it.
|
||||
dc.report_md = (dc.out_cap >= ((size_t)1 << 20)) && chance(300);
|
||||
g_dec_report_md += dc.report_md;
|
||||
|
||||
// Attach the dictionaries to both decoders, checking that they agree on
|
||||
// which ones are accepted. libbrotlidec can be left in a modified state
|
||||
@@ -1899,6 +1996,44 @@ run_dec(uint64_t iters, double secs) {
|
||||
size_t c_attached = 0;
|
||||
dec_result w = wuffs_decode(&dc, &ch, m, n, wout);
|
||||
dec_result c = c_decode(&dc, &ch, m, n, cout, &c_attached);
|
||||
if (dc.report_md && (w.cls == CLS_NEEDS_MORE_OUTPUT)) {
|
||||
// A full dst stops Wuffs at a metadata block, but not libbrotlidec,
|
||||
// so this is inconclusive. Compare without reporting instead.
|
||||
g_dec_md_inconclusive++;
|
||||
dc.report_md = false;
|
||||
w = wuffs_decode(&dc, &ch, m, n, wout);
|
||||
c = c_decode(&dc, &ch, m, n, cout, &c_attached);
|
||||
dc.report_md = true;
|
||||
} else if (dc.report_md) {
|
||||
// Before an error, both have reported the same metadata, except that
|
||||
// one can have more of the last block's contents (or also the next
|
||||
// block's start). After success, they agree exactly.
|
||||
size_t k = (w.md_n < c.md_n) ? w.md_n : c.md_n;
|
||||
const char* mwhy = NULL;
|
||||
if (c.md_n == SIZE_MAX) {
|
||||
mwhy = "libbrotlidec metadata overflow";
|
||||
} else if (k && memcmp(g_wmd, g_cmd, k)) {
|
||||
mwhy = "different metadata";
|
||||
} else if ((w.cls == CLS_OK) && (c.cls == CLS_OK) &&
|
||||
(w.md_n != c.md_n)) {
|
||||
mwhy = "different metadata lengths";
|
||||
}
|
||||
if ((w.cls == CLS_OK) && (c.cls == CLS_OK)) {
|
||||
g_dec_md_compared++;
|
||||
for (size_t j = 0; j + 4 <= w.md_n;) {
|
||||
size_t len = (size_t)g_wmd[j] | ((size_t)g_wmd[j + 1] << 8) |
|
||||
((size_t)g_wmd[j + 2] << 16) |
|
||||
((size_t)g_wmd[j + 3] << 24);
|
||||
g_dec_md_blocks++;
|
||||
j += 4 + len;
|
||||
}
|
||||
}
|
||||
if (mwhy) {
|
||||
char desc[4096];
|
||||
describe_dec(desc, sizeof desc, mwhy, s, &dc, &ch, &w, &c, n);
|
||||
save_failure("md", desc, m, n, &dc.dicts);
|
||||
}
|
||||
}
|
||||
g_dec_decodes += 2;
|
||||
g_dec_cls_counts[pass][cls_index(c.cls)]++;
|
||||
const char* why = NULL;
|
||||
@@ -1948,6 +2083,10 @@ run_dec(uint64_t iters, double secs) {
|
||||
g_dec_wuffs_shorter_output);
|
||||
printf(" known BLOCK_LENGTH_1/2 deviations (chunked)=%" PRIu64 "\n",
|
||||
g_dec_known_block_length);
|
||||
printf(" report metadata=%" PRIu64 " (compared after success=%" PRIu64
|
||||
", blocks=%" PRIu64 ", inconclusive=%" PRIu64 ")\n",
|
||||
g_dec_report_md, g_dec_md_compared, g_dec_md_blocks,
|
||||
g_dec_md_inconclusive);
|
||||
static const char* names[23] = {
|
||||
"other", "ok", "needs-more-input", "needs-more-output",
|
||||
"-1", "-2", "-3", "-4",
|
||||
|
||||
@@ -15,12 +15,12 @@
|
||||
// particular its c/dec/decode.c file. Comments like "Mirrors Foo in decode.c"
|
||||
// name the C function whose logic (and validation checks) a Wuffs function
|
||||
// mirrors. The C code's explicit BROTLI_STATE_ETC state machine is replaced by
|
||||
// Wuffs coroutines. The C code's "safe" and "unsafe" (fast path) variants are
|
||||
// implemented once, as the "safe" (read as few bits as possible) variant.
|
||||
// Wuffs coroutines. The C code's "safe" variants (read as few bits as
|
||||
// possible, so that decoding can suspend anywhere) are coroutines and its
|
||||
// "unsafe" fast path (ProcessCommands) is process_commands_fast, which reads
|
||||
// ahead and gives back the unused bytes (see README.md).
|
||||
//
|
||||
// See README.md for how C error codes map to Wuffs statuses.
|
||||
//
|
||||
// TODO: support metadata callbacks and fast paths.
|
||||
|
||||
pub status "#bad block length"
|
||||
pub status "#bad block switch"
|
||||
@@ -245,6 +245,17 @@ pub struct decoder? implements base.io_transformer(
|
||||
quirk_large_window : base.bool,
|
||||
large_window : base.bool,
|
||||
|
||||
// quirk_report_metadata is the QUIRK_REPORT_METADATA quirk.
|
||||
// md_length is the length of the last reported metadata block.
|
||||
// md_note_pending is whether do_transform_io has suspended (with
|
||||
// "$short write") so that transform_io returns "@metadata reported".
|
||||
// md_copying is whether do_transform_io is writing a metadata block's
|
||||
// contents to args.dst.
|
||||
quirk_report_metadata : base.bool,
|
||||
md_length : base.u32[..= 0x100_0000],
|
||||
md_note_pending : base.bool,
|
||||
md_copying : base.bool,
|
||||
|
||||
// started is whether the C code's s->state would no longer be
|
||||
// BROTLI_STATE_UNINITED. Quirks and dictionaries can only be set or
|
||||
// added before then.
|
||||
@@ -355,6 +366,8 @@ pub struct decoder? implements base.io_transformer(
|
||||
pub func decoder.get_quirk(key: base.u32) base.u64 {
|
||||
if (args.key == QUIRK_LARGE_WINDOW) and this.quirk_large_window {
|
||||
return 1
|
||||
} else if (args.key == QUIRK_REPORT_METADATA) and this.quirk_report_metadata {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -368,10 +381,22 @@ pub func decoder.set_quirk!(key: base.u32, value: base.u64) base.status {
|
||||
}
|
||||
this.quirk_large_window = args.value > 0
|
||||
return ok
|
||||
} else if args.key == QUIRK_REPORT_METADATA {
|
||||
if this.started {
|
||||
return base."#bad call sequence"
|
||||
}
|
||||
this.quirk_report_metadata = args.value > 0
|
||||
return ok
|
||||
}
|
||||
return base."#unsupported option"
|
||||
}
|
||||
|
||||
// metadata_length returns the length of the metadata block that was last
|
||||
// reported (see QUIRK_REPORT_METADATA).
|
||||
pub func decoder.metadata_length() base.u64 {
|
||||
return this.md_length as base.u64
|
||||
}
|
||||
|
||||
pub func decoder.dst_history_retain_length() base.optional_u63 {
|
||||
return this.util.make_optional_u63(has_value: true, value: 0)
|
||||
}
|
||||
@@ -409,7 +434,7 @@ pub func decoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf
|
||||
if this.dict_total > args.workbuf.length() {
|
||||
return base."#bad workbuf length"
|
||||
}
|
||||
status =? this.do_transform_io?(src: args.src,
|
||||
status =? this.do_transform_io?(dst: args.dst, src: args.src,
|
||||
workbuf: args.workbuf[this.dict_total ..],
|
||||
dictbuf: args.workbuf[.. this.dict_total])
|
||||
if this.dict_total > args.workbuf.length() {
|
||||
@@ -428,7 +453,10 @@ pub func decoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf
|
||||
return ok
|
||||
|
||||
} else if status == base."$short write" {
|
||||
if this.is_fully_flushed() {
|
||||
if this.md_note_pending {
|
||||
this.md_note_pending = false
|
||||
return base."@metadata reported"
|
||||
} else if this.is_fully_flushed() and (not this.md_copying) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -471,7 +499,7 @@ pri func decoder.flush!(dst: base.io_writer, workbuf: slice base.u8) {
|
||||
|
||||
// Mirrors the BROTLI_STATE_ETC state machine in BrotliDecoderDecompressStream
|
||||
// in decode.c.
|
||||
pri func decoder.do_transform_io?(src: base.io_reader, workbuf: slice base.u8, dictbuf: roslice base.u8) {
|
||||
pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, workbuf: slice base.u8, dictbuf: roslice base.u8) {
|
||||
var w : base.u64
|
||||
var aligned : base.bool
|
||||
|
||||
@@ -508,9 +536,11 @@ pri func decoder.do_transform_io?(src: base.io_reader, workbuf: slice base.u8, d
|
||||
|
||||
if this.is_metadata {
|
||||
// BROTLI_STATE_METADATA. Mirrors SkipMetadataBlock.
|
||||
//
|
||||
// TODO: support metadata callbacks.
|
||||
args.src.skip?(n: this.mb_remaining as base.u64)
|
||||
if this.quirk_report_metadata {
|
||||
this.report_metadata?(dst: args.dst, src: args.src)
|
||||
} else {
|
||||
args.src.skip?(n: this.mb_remaining as base.u64)
|
||||
}
|
||||
this.mb_remaining = 0
|
||||
|
||||
} else if this.mb_remaining > 0 {
|
||||
@@ -538,6 +568,36 @@ pri func decoder.do_transform_io?(src: base.io_reader, workbuf: slice base.u8, d
|
||||
}
|
||||
}
|
||||
|
||||
// report_metadata mirrors the C code's metadata callbacks (the start callback
|
||||
// in BROTLI_STATE_METABLOCK_HEADER and the chunk callbacks in
|
||||
// SkipMetadataBlock). It waits until every decoded byte has been written to
|
||||
// args.dst, then has transform_io return "@metadata reported" and then copies
|
||||
// the metadata block's contents from args.src to args.dst.
|
||||
pri func decoder.report_metadata?(dst: base.io_writer, src: base.io_reader) {
|
||||
var n : base.u32
|
||||
|
||||
while not this.is_fully_flushed() {
|
||||
yield? base."$short write"
|
||||
}
|
||||
this.md_length = this.mb_remaining.min(no_more_than: 0x100_0000)
|
||||
this.md_note_pending = true
|
||||
yield? base."$short write"
|
||||
|
||||
this.md_copying = true
|
||||
while this.mb_remaining > 0 {
|
||||
if args.dst.length() <= 0 {
|
||||
yield? base."$short write"
|
||||
continue
|
||||
} else if args.src.length() <= 0 {
|
||||
yield? base."$short read"
|
||||
continue
|
||||
}
|
||||
n = args.dst.limited_copy_u32_from_reader!(up_to: this.mb_remaining as base.u32, r: args.src)
|
||||
this.mb_remaining ~mod-= n
|
||||
}
|
||||
this.md_copying = false
|
||||
}
|
||||
|
||||
// initialize_codes sets up the decoder state that the C code sets in
|
||||
// BrotliDecoderStateInit (in state.c). It also builds the fixed table for
|
||||
// decoding the code length code lengths (kCodeLengthPrefixLength and
|
||||
|
||||
@@ -30,3 +30,19 @@ pri const QUIRKS_BASE : base.u32 = 0x2AB7_9000
|
||||
// is rejected with "#unsupported large window". With this quirk, a stream that
|
||||
// does not start with that escape is decoded as standard brotli.
|
||||
pub const QUIRK_LARGE_WINDOW : base.u32 = 0x2AB7_9000 | 0x00
|
||||
|
||||
// When this quirk is set, a positive value makes the decoder report metadata
|
||||
// blocks, like the C code's BrotliDecoderSetMetadataCallbacks. Zero (the
|
||||
// default) means that metadata blocks are skipped, like the C code without
|
||||
// callbacks.
|
||||
//
|
||||
// When reporting, at each metadata block, after all of the previously decoded
|
||||
// bytes have been written to dst, transform_io returns
|
||||
// base."@metadata reported" and metadata_length returns the block's length
|
||||
// (the C code's start callback's size argument), which can be zero. The next
|
||||
// metadata_length bytes that transform_io writes to dst (in subsequent calls)
|
||||
// are the metadata block's contents (the C code's chunk callbacks' data),
|
||||
// after which decoded bytes follow as usual.
|
||||
//
|
||||
// It must be set before decoding starts. This quirk is decoder-only.
|
||||
pub const QUIRK_REPORT_METADATA : base.u32 = 0x2AB7_9000 | 0x0A
|
||||
@@ -1740,6 +1740,377 @@ test_wuffs_brotli_encode_interface() {
|
||||
return status;
|
||||
}
|
||||
|
||||
// ---------------- Decoder Metadata Tests
|
||||
|
||||
// The metadata tests encode romeo.txt with metadata blocks (whose contents
|
||||
// are also taken from romeo.txt) and then decode it with
|
||||
// QUIRK_REPORT_METADATA. The decoded output ("out") and the metadata ("md",
|
||||
// each block prefixed by its length as 4 little-endian bytes) are collected
|
||||
// separately.
|
||||
|
||||
#define BROTLI_MD_NUM_STEPS 8
|
||||
#define BROTLI_MD_MAX_BLOCKS 8
|
||||
|
||||
static const uint32_t g_brotli_md_step_ops[BROTLI_MD_NUM_STEPS] = {
|
||||
BROTLI_ENC_OP_EMIT_METADATA, BROTLI_ENC_OP_PROCESS,
|
||||
BROTLI_ENC_OP_EMIT_METADATA, BROTLI_ENC_OP_EMIT_METADATA,
|
||||
BROTLI_ENC_OP_FLUSH, BROTLI_ENC_OP_EMIT_METADATA,
|
||||
BROTLI_ENC_OP_PROCESS, BROTLI_ENC_OP_FINISH,
|
||||
};
|
||||
|
||||
static const size_t g_brotli_md_step_lens[BROTLI_MD_NUM_STEPS] = {
|
||||
3, 500, 10, 0, 0, 70000, 1000, 50000,
|
||||
};
|
||||
|
||||
// brotli_md_want fills out and md with the expected results for the
|
||||
// g_brotli_md_step_etc plan applied to src. It sets md_pos[i] to the length of
|
||||
// out before the i'th metadata block.
|
||||
static void //
|
||||
brotli_md_want(wuffs_base__io_buffer* out,
|
||||
wuffs_base__io_buffer* md,
|
||||
size_t* md_pos,
|
||||
const uint8_t* src) {
|
||||
size_t n_md = 0;
|
||||
for (size_t i = 0; i < BROTLI_MD_NUM_STEPS; i++) {
|
||||
size_t n = g_brotli_md_step_lens[i];
|
||||
if (g_brotli_md_step_ops[i] == BROTLI_ENC_OP_EMIT_METADATA) {
|
||||
md_pos[n_md++] = out->meta.wi;
|
||||
wuffs_base__poke_u32le__no_bounds_check(md->data.ptr + md->meta.wi,
|
||||
(uint32_t)n);
|
||||
md->meta.wi += 4;
|
||||
memcpy(md->data.ptr + md->meta.wi, src, n);
|
||||
md->meta.wi += n;
|
||||
} else {
|
||||
memcpy(out->data.ptr + out->meta.wi, src, n);
|
||||
out->meta.wi += n;
|
||||
}
|
||||
src += n;
|
||||
}
|
||||
}
|
||||
|
||||
// wuffs_brotli_decode_metadata decodes src_ptr[.. src_len] with
|
||||
// QUIRK_REPORT_METADATA, at most dst_chunk bytes of dst and src_chunk more
|
||||
// bytes of src per transform_io call.
|
||||
static const char* //
|
||||
wuffs_brotli_decode_metadata(wuffs_base__io_buffer* out,
|
||||
wuffs_base__io_buffer* md,
|
||||
size_t* md_pos,
|
||||
size_t* num_md,
|
||||
const uint8_t* src_ptr,
|
||||
size_t src_len,
|
||||
size_t dst_chunk,
|
||||
size_t src_chunk) {
|
||||
wuffs_brotli__decoder dec;
|
||||
CHECK_STATUS("initialize",
|
||||
wuffs_brotli__decoder__initialize(
|
||||
&dec, sizeof dec, WUFFS_VERSION,
|
||||
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
|
||||
CHECK_STATUS("set_quirk",
|
||||
wuffs_brotli__decoder__set_quirk(
|
||||
&dec, WUFFS_BROTLI__QUIRK_REPORT_METADATA, 1));
|
||||
if (wuffs_brotli__decoder__get_quirk(
|
||||
&dec, WUFFS_BROTLI__QUIRK_REPORT_METADATA) != 1) {
|
||||
RETURN_FAIL("get_quirk: have 0, want 1");
|
||||
}
|
||||
|
||||
uint8_t tmp[4096];
|
||||
if (dst_chunk > sizeof tmp) {
|
||||
RETURN_FAIL("dst_chunk is too large");
|
||||
}
|
||||
size_t md_remaining = 0;
|
||||
*num_md = 0;
|
||||
wuffs_base__io_buffer src = wuffs_base__ptr_u8__reader(
|
||||
(uint8_t*)src_ptr, src_chunk < src_len ? src_chunk : src_len,
|
||||
src_chunk >= src_len);
|
||||
while (true) {
|
||||
wuffs_base__io_buffer dst = wuffs_base__ptr_u8__writer(tmp, dst_chunk);
|
||||
wuffs_base__status status = wuffs_brotli__decoder__transform_io(
|
||||
&dec, &dst, &src, g_work_slice_u8);
|
||||
|
||||
uint8_t* p = tmp;
|
||||
size_t n = dst.meta.wi;
|
||||
if (md_remaining > 0) {
|
||||
size_t m = (n < md_remaining) ? n : md_remaining;
|
||||
if (md->data.len - md->meta.wi < m) {
|
||||
RETURN_FAIL("md is too short");
|
||||
}
|
||||
memcpy(md->data.ptr + md->meta.wi, p, m);
|
||||
md->meta.wi += m;
|
||||
md_remaining -= m;
|
||||
p += m;
|
||||
n -= m;
|
||||
}
|
||||
if (n > 0) {
|
||||
if (md_remaining > 0) {
|
||||
RETURN_FAIL("inconsistent md_remaining");
|
||||
} else if (out->data.len - out->meta.wi < n) {
|
||||
RETURN_FAIL("out is too short");
|
||||
}
|
||||
memcpy(out->data.ptr + out->meta.wi, p, n);
|
||||
out->meta.wi += n;
|
||||
}
|
||||
|
||||
if (status.repr == NULL) {
|
||||
break;
|
||||
} else if (status.repr == wuffs_base__note__metadata_reported) {
|
||||
if (md_remaining > 0) {
|
||||
RETURN_FAIL("metadata reported while copying metadata");
|
||||
} else if (*num_md >= BROTLI_MD_MAX_BLOCKS) {
|
||||
RETURN_FAIL("too many metadata blocks");
|
||||
}
|
||||
uint64_t len = wuffs_brotli__decoder__metadata_length(&dec);
|
||||
md_pos[(*num_md)++] = out->meta.wi;
|
||||
if (md->data.len - md->meta.wi < 4) {
|
||||
RETURN_FAIL("md is too short");
|
||||
}
|
||||
wuffs_base__poke_u32le__no_bounds_check(md->data.ptr + md->meta.wi,
|
||||
(uint32_t)len);
|
||||
md->meta.wi += 4;
|
||||
md_remaining = (size_t)len;
|
||||
} else if (status.repr == wuffs_base__suspension__short_read) {
|
||||
if (src.meta.closed) {
|
||||
RETURN_FAIL("short read on closed src");
|
||||
}
|
||||
src.meta.wi = (src_len - src.meta.wi > src_chunk) ? src.meta.wi + src_chunk
|
||||
: src_len;
|
||||
src.meta.closed = src.meta.wi == src_len;
|
||||
} else if (status.repr != wuffs_base__suspension__short_write) {
|
||||
RETURN_FAIL("transform_io: \"%s\"", status.repr);
|
||||
}
|
||||
}
|
||||
if (md_remaining > 0) {
|
||||
RETURN_FAIL("incomplete metadata");
|
||||
}
|
||||
|
||||
// Setting the quirk after decoding has started fails.
|
||||
wuffs_base__status status = wuffs_brotli__decoder__set_quirk(
|
||||
&dec, WUFFS_BROTLI__QUIRK_REPORT_METADATA, 0);
|
||||
if (status.repr != wuffs_base__error__bad_call_sequence) {
|
||||
RETURN_FAIL("set_quirk after start: have \"%s\", want \"%s\"", status.repr,
|
||||
wuffs_base__error__bad_call_sequence);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static const char* //
|
||||
do_test_brotli_decode_metadata(const char* (*encode)(wuffs_base__io_buffer*,
|
||||
wuffs_base__io_buffer*,
|
||||
const uint32_t*,
|
||||
const uint32_t*,
|
||||
size_t,
|
||||
const uint32_t*,
|
||||
const size_t*,
|
||||
size_t,
|
||||
size_t)) {
|
||||
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
|
||||
.data = g_mimiclib_scratch_slice_u8,
|
||||
});
|
||||
CHECK_STRING(read_file(&data, "test/data/romeo.txt"));
|
||||
// The plan needs 121513 bytes, so repeat romeo.txt (which is shorter).
|
||||
size_t total = 0;
|
||||
for (size_t i = 0; i < BROTLI_MD_NUM_STEPS; i++) {
|
||||
total += g_brotli_md_step_lens[i];
|
||||
}
|
||||
if (data.meta.wi == 0) {
|
||||
RETURN_FAIL("empty romeo.txt");
|
||||
}
|
||||
while (data.meta.wi < total) {
|
||||
size_t n = data.meta.wi;
|
||||
if (n > total - data.meta.wi) {
|
||||
n = total - data.meta.wi;
|
||||
}
|
||||
memmove(data.data.ptr + data.meta.wi, data.data.ptr, n);
|
||||
data.meta.wi += n;
|
||||
}
|
||||
|
||||
// The want and have halves of g_want_slice_u8 and g_have_slice_u8.
|
||||
size_t half_w = g_want_slice_u8.len / 2;
|
||||
size_t half_h = g_have_slice_u8.len / 2;
|
||||
wuffs_base__io_buffer want_out = ((wuffs_base__io_buffer){
|
||||
.data = wuffs_base__make_slice_u8(g_want_slice_u8.ptr, half_w),
|
||||
});
|
||||
wuffs_base__io_buffer want_md = ((wuffs_base__io_buffer){
|
||||
.data = wuffs_base__make_slice_u8(g_want_slice_u8.ptr + half_w, half_w),
|
||||
});
|
||||
size_t want_md_pos[BROTLI_MD_MAX_BLOCKS] = {0};
|
||||
brotli_md_want(&want_out, &want_md, want_md_pos, data.data.ptr);
|
||||
|
||||
static const uint32_t qualities[3] = {0, 5, 11};
|
||||
static const size_t chunks[4][2] = {{4096, 1 << 30}, {1, 1}, {7, 3}, {3, 64}};
|
||||
for (int qi = 0; qi < 3; qi++) {
|
||||
// The decoder uses g_work_slice_u8 as its workbuf, so encode into the
|
||||
// second half of g_have_slice_u8.
|
||||
wuffs_base__io_buffer enc = ((wuffs_base__io_buffer){
|
||||
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr + half_h, half_h),
|
||||
});
|
||||
wuffs_base__io_buffer src = data;
|
||||
src.meta.ri = 0;
|
||||
uint32_t keys[2] = {BROTLI_ENC_PARAM_QUALITY, BROTLI_ENC_PARAM_LGWIN};
|
||||
uint32_t values[2] = {qualities[qi], 18};
|
||||
CHECK_STRING((*encode)(&enc, &src, keys, values, 2, g_brotli_md_step_ops,
|
||||
g_brotli_md_step_lens, BROTLI_MD_NUM_STEPS, 0));
|
||||
|
||||
for (int ci = 0; ci < 4; ci++) {
|
||||
// Decoded output and metadata share the first half of g_have_slice_u8.
|
||||
wuffs_base__io_buffer have_out = ((wuffs_base__io_buffer){
|
||||
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr, half_h / 2),
|
||||
});
|
||||
wuffs_base__io_buffer have_md = ((wuffs_base__io_buffer){
|
||||
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr + half_h / 2,
|
||||
half_h / 2),
|
||||
});
|
||||
size_t have_md_pos[BROTLI_MD_MAX_BLOCKS] = {0};
|
||||
size_t num_md = 0;
|
||||
const char* z = wuffs_brotli_decode_metadata(
|
||||
&have_out, &have_md, have_md_pos, &num_md, enc.data.ptr, enc.meta.wi,
|
||||
chunks[ci][0], chunks[ci][1]);
|
||||
if (z) {
|
||||
RETURN_FAIL("q=%u, chunks=%d: %s", qualities[qi], ci, z);
|
||||
}
|
||||
char s[64];
|
||||
snprintf(s, sizeof s, "q=%u, chunks=%d: out", qualities[qi], ci);
|
||||
CHECK_STRING(check_io_buffers_equal(s, &have_out, &want_out));
|
||||
snprintf(s, sizeof s, "q=%u, chunks=%d: md", qualities[qi], ci);
|
||||
CHECK_STRING(check_io_buffers_equal(s, &have_md, &want_md));
|
||||
if (num_md != 4) {
|
||||
RETURN_FAIL("q=%u, chunks=%d: num_md: have %zu, want 4",
|
||||
qualities[qi], ci, num_md);
|
||||
}
|
||||
for (size_t i = 0; i < num_md; i++) {
|
||||
if (have_md_pos[i] != want_md_pos[i]) {
|
||||
RETURN_FAIL("q=%u, chunks=%d: md_pos[%zu]: have %zu, want %zu",
|
||||
qualities[qi], ci, i, have_md_pos[i], want_md_pos[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_brotli_decode_metadata() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_test_brotli_decode_metadata(wuffs_brotli_encode_plan);
|
||||
}
|
||||
|
||||
#ifdef WUFFS_MIMIC
|
||||
|
||||
// mimic_brotli_md_start_func and mimic_brotli_md_chunk_func are libbrotlidec
|
||||
// metadata callbacks that append to the wuffs_base__io_buffer opaque.
|
||||
static void //
|
||||
mimic_brotli_md_start_func(void* opaque, size_t size) {
|
||||
wuffs_base__io_buffer* md = (wuffs_base__io_buffer*)opaque;
|
||||
if (md->data.len - md->meta.wi >= 4) {
|
||||
wuffs_base__poke_u32le__no_bounds_check(md->data.ptr + md->meta.wi,
|
||||
(uint32_t)size);
|
||||
md->meta.wi += 4;
|
||||
} else {
|
||||
md->meta.closed = true; // Overflow.
|
||||
}
|
||||
}
|
||||
|
||||
static void //
|
||||
mimic_brotli_md_chunk_func(void* opaque, const uint8_t* data, size_t size) {
|
||||
wuffs_base__io_buffer* md = (wuffs_base__io_buffer*)opaque;
|
||||
if (md->data.len - md->meta.wi >= size) {
|
||||
memcpy(md->data.ptr + md->meta.wi, data, size);
|
||||
md->meta.wi += size;
|
||||
} else {
|
||||
md->meta.closed = true; // Overflow.
|
||||
}
|
||||
}
|
||||
|
||||
// test_mimic_brotli_decode_metadata checks that QUIRK_REPORT_METADATA reports
|
||||
// the same metadata as libbrotlidec's metadata callbacks, for streams that
|
||||
// libbrotlienc produces (and for the streams of the brotli test data, which
|
||||
// have no metadata blocks), with a variety of input chunk sizes.
|
||||
const char* //
|
||||
test_mimic_brotli_decode_metadata() {
|
||||
CHECK_FOCUS(__func__);
|
||||
CHECK_STRING(do_test_brotli_decode_metadata(mimic_brotli_encode));
|
||||
|
||||
// Also compare libbrotlidec's callbacks with Wuffs on a libbrotlienc
|
||||
// stream, fed to libbrotlidec in 5-byte pieces.
|
||||
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
|
||||
.data = g_mimiclib_scratch_slice_u8,
|
||||
});
|
||||
CHECK_STRING(read_file(&data, "test/data/romeo.txt"));
|
||||
size_t half_h = g_have_slice_u8.len / 2;
|
||||
size_t half_w = g_want_slice_u8.len / 2;
|
||||
wuffs_base__io_buffer enc = ((wuffs_base__io_buffer){
|
||||
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr + half_h, half_h),
|
||||
});
|
||||
static const uint32_t ops[5] = {
|
||||
BROTLI_ENC_OP_PROCESS, BROTLI_ENC_OP_EMIT_METADATA,
|
||||
BROTLI_ENC_OP_EMIT_METADATA, BROTLI_ENC_OP_PROCESS,
|
||||
BROTLI_ENC_OP_FINISH,
|
||||
};
|
||||
size_t lens[5] = {100, 200, 0, 300, 0};
|
||||
lens[4] = data.meta.wi - 600;
|
||||
uint32_t keys[1] = {BROTLI_ENC_PARAM_QUALITY};
|
||||
uint32_t values[1] = {9};
|
||||
CHECK_STRING(mimic_brotli_encode(&enc, &data, keys, values, 1, ops, lens, 5,
|
||||
0));
|
||||
|
||||
wuffs_base__io_buffer c_out = ((wuffs_base__io_buffer){
|
||||
.data = wuffs_base__make_slice_u8(g_want_slice_u8.ptr, half_w),
|
||||
});
|
||||
wuffs_base__io_buffer c_md = ((wuffs_base__io_buffer){
|
||||
.data = wuffs_base__make_slice_u8(g_want_slice_u8.ptr + half_w, half_w),
|
||||
});
|
||||
BrotliDecoderState* state = BrotliDecoderCreateInstance(NULL, NULL, NULL);
|
||||
if (!state) {
|
||||
RETURN_FAIL("BrotliDecoderCreateInstance failed");
|
||||
}
|
||||
BrotliDecoderSetMetadataCallbacks(state, mimic_brotli_md_start_func,
|
||||
mimic_brotli_md_chunk_func, &c_md);
|
||||
size_t pos = 0;
|
||||
BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
|
||||
while (r != BROTLI_DECODER_RESULT_SUCCESS) {
|
||||
size_t end = (enc.meta.wi - pos > 5) ? pos + 5 : enc.meta.wi;
|
||||
size_t avail_in = end - pos;
|
||||
const uint8_t* next_in = enc.data.ptr + pos;
|
||||
size_t avail_out = c_out.data.len - c_out.meta.wi;
|
||||
uint8_t* next_out = c_out.data.ptr + c_out.meta.wi;
|
||||
r = BrotliDecoderDecompressStream(state, &avail_in, &next_in, &avail_out,
|
||||
&next_out, NULL);
|
||||
pos = end - avail_in;
|
||||
c_out.meta.wi = c_out.data.len - avail_out;
|
||||
if ((r == BROTLI_DECODER_RESULT_ERROR) ||
|
||||
((r == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) &&
|
||||
(pos == enc.meta.wi))) {
|
||||
BrotliDecoderDestroyInstance(state);
|
||||
RETURN_FAIL("libbrotlidec failed");
|
||||
}
|
||||
}
|
||||
BrotliDecoderDestroyInstance(state);
|
||||
if (c_md.meta.closed) {
|
||||
RETURN_FAIL("libbrotlidec metadata overflow");
|
||||
}
|
||||
|
||||
wuffs_base__io_buffer have_out = ((wuffs_base__io_buffer){
|
||||
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr, half_h / 2),
|
||||
});
|
||||
wuffs_base__io_buffer have_md = ((wuffs_base__io_buffer){
|
||||
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr + half_h / 2,
|
||||
half_h / 2),
|
||||
});
|
||||
size_t have_md_pos[BROTLI_MD_MAX_BLOCKS] = {0};
|
||||
size_t num_md = 0;
|
||||
CHECK_STRING(wuffs_brotli_decode_metadata(&have_out, &have_md, have_md_pos,
|
||||
&num_md, enc.data.ptr, enc.meta.wi,
|
||||
5, 5));
|
||||
CHECK_STRING(check_io_buffers_equal("out", &have_out, &c_out));
|
||||
CHECK_STRING(check_io_buffers_equal("md", &have_md, &c_md));
|
||||
if ((num_md != 2) || (have_md_pos[0] != 100) || (have_md_pos[1] != 100)) {
|
||||
RETURN_FAIL("md_pos: have %zu blocks at %zu, %zu; want 2 at 100, 100",
|
||||
num_md, have_md_pos[0], have_md_pos[1]);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#endif // WUFFS_MIMIC
|
||||
|
||||
// ---------------- Encoder Mimic Tests
|
||||
|
||||
#ifdef WUFFS_MIMIC
|
||||
@@ -4618,6 +4989,7 @@ proc g_tests[] = {
|
||||
test_wuffs_brotli_decode_all,
|
||||
test_wuffs_brotli_decode_all_with_io_limits,
|
||||
test_wuffs_brotli_decode_interface,
|
||||
test_wuffs_brotli_decode_metadata,
|
||||
test_wuffs_brotli_decode_malformed,
|
||||
test_wuffs_brotli_decode_romeo,
|
||||
test_wuffs_brotli_decode_trailing_data,
|
||||
@@ -4642,6 +5014,7 @@ proc g_tests[] = {
|
||||
test_mimic_brotli_decode_all,
|
||||
test_mimic_brotli_decode_differential,
|
||||
test_mimic_brotli_decode_malformed,
|
||||
test_mimic_brotli_decode_metadata,
|
||||
test_mimic_brotli_decode_romeo,
|
||||
test_mimic_brotli_decode_split_block_length,
|
||||
|
||||
|
||||
Reference in new issue
Block a user