std/brotli: add QUIRK_REPORT_METADATA

The decoder can now report metadata blocks, like libbrotlidec's
BrotliDecoderSetMetadataCallbacks: transform_io returns
"@metadata reported", metadata_length gives the block's length and the
block's contents are written to dst. The differential fuzzer compares
the reported metadata with libbrotlidec's callbacks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UZ978Fjdsyig4v3wYAZRg5
This commit is contained in:
Claude committed 2026-10-27 17:06:08 +00:00
1 parent 5d6de9880f
commit bc1b5fcd49
5 files changed
+774 -27

No files matched your search

+171 -12
View File
@@ -9253,6 +9253,8 @@ extern const char wuffs_brotli__error__unsupported_serialized_encoder_dictionary
#define WUFFS_BROTLI__QUIRK_LARGE_WINDOW 716673024u #define WUFFS_BROTLI__QUIRK_LARGE_WINDOW 716673024u
#define WUFFS_BROTLI__QUIRK_REPORT_METADATA 716673034u
#define WUFFS_BROTLI__ENCODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE 0u #define WUFFS_BROTLI__ENCODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE 0u
#define WUFFS_BROTLI__QUIRK_ENCODER_QUALITY 716673025u #define WUFFS_BROTLI__QUIRK_ENCODER_QUALITY 716673025u
@@ -9383,6 +9385,11 @@ wuffs_brotli__decoder__set_quirk(
uint32_t a_key, uint32_t a_key,
uint64_t a_value); uint64_t a_value);
WUFFS_BASE__GENERATED_C_CODE
WUFFS_BASE__MAYBE_STATIC uint64_t
wuffs_brotli__decoder__metadata_length(
const wuffs_brotli__decoder* self);
WUFFS_BASE__GENERATED_C_CODE WUFFS_BASE__GENERATED_C_CODE
WUFFS_BASE__MAYBE_STATIC wuffs_base__optional_u63 WUFFS_BASE__MAYBE_STATIC wuffs_base__optional_u63
wuffs_brotli__decoder__dst_history_retain_length( wuffs_brotli__decoder__dst_history_retain_length(
@@ -9612,6 +9619,10 @@ struct wuffs_brotli__decoder__struct {
uint32_t f_cmd_state; uint32_t f_cmd_state;
bool f_quirk_large_window; bool f_quirk_large_window;
bool f_large_window; bool f_large_window;
bool f_quirk_report_metadata;
uint32_t f_md_length;
bool f_md_note_pending;
bool f_md_copying;
bool f_started; bool f_started;
uint64_t f_dict_total; uint64_t f_dict_total;
uint32_t f_num_chunks; uint32_t f_num_chunks;
@@ -9634,6 +9645,7 @@ struct wuffs_brotli__decoder__struct {
uint32_t p_transform_io; uint32_t p_transform_io;
uint32_t p_do_transform_io; uint32_t p_do_transform_io;
uint32_t p_report_metadata;
uint32_t p_read_bits; uint32_t p_read_bits;
uint32_t p_read_bits32; uint32_t p_read_bits32;
uint32_t p_read_symbol; uint32_t p_read_symbol;
@@ -9852,6 +9864,11 @@ struct wuffs_brotli__decoder__struct {
return wuffs_brotli__decoder__set_quirk(this, a_key, a_value); return wuffs_brotli__decoder__set_quirk(this, a_key, a_value);
} }
inline uint64_t
metadata_length() const {
return wuffs_brotli__decoder__metadata_length(this);
}
inline wuffs_base__optional_u63 inline wuffs_base__optional_u63
dst_history_retain_length() const { dst_history_retain_length() const {
return wuffs_brotli__decoder__dst_history_retain_length(this); return wuffs_brotli__decoder__dst_history_retain_length(this);
@@ -77384,10 +77401,18 @@ WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__status static wuffs_base__status
wuffs_brotli__decoder__do_transform_io( wuffs_brotli__decoder__do_transform_io(
wuffs_brotli__decoder* self, wuffs_brotli__decoder* self,
wuffs_base__io_buffer* a_dst,
wuffs_base__io_buffer* a_src, wuffs_base__io_buffer* a_src,
wuffs_base__slice_u8 a_workbuf, wuffs_base__slice_u8 a_workbuf,
wuffs_base__slice_u8 a_dictbuf); wuffs_base__slice_u8 a_dictbuf);
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__status
wuffs_brotli__decoder__report_metadata(
wuffs_brotli__decoder* self,
wuffs_base__io_buffer* a_dst,
wuffs_base__io_buffer* a_src);
WUFFS_BASE__GENERATED_C_CODE WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__empty_struct static wuffs_base__empty_struct
wuffs_brotli__decoder__initialize_codes( wuffs_brotli__decoder__initialize_codes(
@@ -81135,6 +81160,8 @@ wuffs_brotli__decoder__get_quirk(
if ((a_key == 716673024u) && self->private_impl.f_quirk_large_window) { if ((a_key == 716673024u) && self->private_impl.f_quirk_large_window) {
return 1u; return 1u;
} else if ((a_key == 716673034u) && self->private_impl.f_quirk_report_metadata) {
return 1u;
} }
return 0u; return 0u;
} }
@@ -81163,10 +81190,33 @@ wuffs_brotli__decoder__set_quirk(
} }
self->private_impl.f_quirk_large_window = (a_value > 0u); self->private_impl.f_quirk_large_window = (a_value > 0u);
return wuffs_base__make_status(NULL); return wuffs_base__make_status(NULL);
} else if (a_key == 716673034u) {
if (self->private_impl.f_started) {
return wuffs_base__make_status(wuffs_base__error__bad_call_sequence);
}
self->private_impl.f_quirk_report_metadata = (a_value > 0u);
return wuffs_base__make_status(NULL);
} }
return wuffs_base__make_status(wuffs_base__error__unsupported_option); return wuffs_base__make_status(wuffs_base__error__unsupported_option);
} }
// -------- func brotli.decoder.metadata_length
WUFFS_BASE__GENERATED_C_CODE
WUFFS_BASE__MAYBE_STATIC uint64_t
wuffs_brotli__decoder__metadata_length(
const wuffs_brotli__decoder* self) {
if (!self) {
return 0;
}
if ((self->private_impl.magic != WUFFS_BASE__MAGIC) &&
(self->private_impl.magic != WUFFS_BASE__DISABLED)) {
return 0;
}
return ((uint64_t)(self->private_impl.f_md_length));
}
// -------- func brotli.decoder.dst_history_retain_length // -------- func brotli.decoder.dst_history_retain_length
WUFFS_BASE__GENERATED_C_CODE WUFFS_BASE__GENERATED_C_CODE
@@ -81249,7 +81299,11 @@ wuffs_brotli__decoder__transform_io(
goto exit; goto exit;
} }
{ {
wuffs_base__status t_0 = wuffs_brotli__decoder__do_transform_io(self, a_src, wuffs_base__slice_u8__subslice_i(a_workbuf, self->private_impl.f_dict_total), wuffs_base__slice_u8__subslice_j(a_workbuf, self->private_impl.f_dict_total)); wuffs_base__status t_0 = wuffs_brotli__decoder__do_transform_io(self,
a_dst,
a_src,
wuffs_base__slice_u8__subslice_i(a_workbuf, self->private_impl.f_dict_total),
wuffs_base__slice_u8__subslice_j(a_workbuf, self->private_impl.f_dict_total));
v_status = t_0; v_status = t_0;
} }
if (self->private_impl.f_dict_total > ((uint64_t)(a_workbuf.len))) { if (self->private_impl.f_dict_total > ((uint64_t)(a_workbuf.len))) {
@@ -81270,7 +81324,11 @@ wuffs_brotli__decoder__transform_io(
status = wuffs_base__make_status(NULL); status = wuffs_base__make_status(NULL);
goto ok; goto ok;
} else if (v_status.repr == wuffs_base__suspension__short_write) { } else if (v_status.repr == wuffs_base__suspension__short_write) {
if (wuffs_brotli__decoder__is_fully_flushed(self)) { if (self->private_impl.f_md_note_pending) {
self->private_impl.f_md_note_pending = false;
status = wuffs_base__make_status(wuffs_base__note__metadata_reported);
goto ok;
} else if (wuffs_brotli__decoder__is_fully_flushed(self) && ! self->private_impl.f_md_copying) {
continue; continue;
} }
} else if (v_status.repr == wuffs_base__suspension__short_read) { } else if (v_status.repr == wuffs_base__suspension__short_read) {
@@ -81367,6 +81425,7 @@ WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__status static wuffs_base__status
wuffs_brotli__decoder__do_transform_io( wuffs_brotli__decoder__do_transform_io(
wuffs_brotli__decoder* self, wuffs_brotli__decoder* self,
wuffs_base__io_buffer* a_dst,
wuffs_base__io_buffer* a_src, wuffs_base__io_buffer* a_src,
wuffs_base__slice_u8 a_workbuf, wuffs_base__slice_u8 a_workbuf,
wuffs_base__slice_u8 a_dictbuf) { wuffs_base__slice_u8 a_dictbuf) {
@@ -81435,15 +81494,29 @@ wuffs_brotli__decoder__do_transform_io(
} }
} }
if (self->private_impl.f_is_metadata) { if (self->private_impl.f_is_metadata) {
self->private_data.s_do_transform_io.scratch = ((uint64_t)(self->private_impl.f_mb_remaining)); if (self->private_impl.f_quirk_report_metadata) {
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(3); if (a_src) {
if (self->private_data.s_do_transform_io.scratch > ((uint64_t)(io2_a_src - iop_a_src))) { a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
self->private_data.s_do_transform_io.scratch -= ((uint64_t)(io2_a_src - iop_a_src)); }
iop_a_src = io2_a_src; WUFFS_BASE__COROUTINE_SUSPENSION_POINT(3);
status = wuffs_base__make_status(wuffs_base__suspension__short_read); status = wuffs_brotli__decoder__report_metadata(self, a_dst, a_src);
goto suspend; if (a_src) {
iop_a_src = a_src->data.ptr + a_src->meta.ri;
}
if (status.repr) {
goto suspend;
}
} else {
self->private_data.s_do_transform_io.scratch = ((uint64_t)(self->private_impl.f_mb_remaining));
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(4);
if (self->private_data.s_do_transform_io.scratch > ((uint64_t)(io2_a_src - iop_a_src))) {
self->private_data.s_do_transform_io.scratch -= ((uint64_t)(io2_a_src - iop_a_src));
iop_a_src = io2_a_src;
status = wuffs_base__make_status(wuffs_base__suspension__short_read);
goto suspend;
}
iop_a_src += self->private_data.s_do_transform_io.scratch;
} }
iop_a_src += self->private_data.s_do_transform_io.scratch;
self->private_impl.f_mb_remaining = 0u; self->private_impl.f_mb_remaining = 0u;
} else if (self->private_impl.f_mb_remaining > 0u) { } else if (self->private_impl.f_mb_remaining > 0u) {
wuffs_brotli__decoder__calculate_ring_buffer_size(self); wuffs_brotli__decoder__calculate_ring_buffer_size(self);
@@ -81451,7 +81524,7 @@ wuffs_brotli__decoder__do_transform_io(
if (a_src) { if (a_src) {
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
} }
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(4); WUFFS_BASE__COROUTINE_SUSPENSION_POINT(5);
status = wuffs_brotli__decoder__copy_uncompressed(self, a_src, a_workbuf); status = wuffs_brotli__decoder__copy_uncompressed(self, a_src, a_workbuf);
if (a_src) { if (a_src) {
iop_a_src = a_src->data.ptr + a_src->meta.ri; iop_a_src = a_src->data.ptr + a_src->meta.ri;
@@ -81463,7 +81536,7 @@ wuffs_brotli__decoder__do_transform_io(
if (a_src) { if (a_src) {
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
} }
WUFFS_BASE__COROUTINE_SUSPENSION_POINT(5); WUFFS_BASE__COROUTINE_SUSPENSION_POINT(6);
status = wuffs_brotli__decoder__decode_compressed_metablock(self, a_src, a_workbuf, a_dictbuf); status = wuffs_brotli__decoder__decode_compressed_metablock(self, a_src, a_workbuf, a_dictbuf);
if (a_src) { if (a_src) {
iop_a_src = a_src->data.ptr + a_src->meta.ri; iop_a_src = a_src->data.ptr + a_src->meta.ri;
@@ -81507,6 +81580,92 @@ wuffs_brotli__decoder__do_transform_io(
return status; return status;
} }
// -------- func brotli.decoder.report_metadata
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__status
wuffs_brotli__decoder__report_metadata(
wuffs_brotli__decoder* self,
wuffs_base__io_buffer* a_dst,
wuffs_base__io_buffer* a_src) {
wuffs_base__status status = wuffs_base__make_status(NULL);
uint32_t v_n = 0;
uint8_t* iop_a_dst = NULL;
uint8_t* io0_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
uint8_t* io1_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
uint8_t* io2_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
if (a_dst && a_dst->data.ptr) {
io0_a_dst = a_dst->data.ptr;
io1_a_dst = io0_a_dst + a_dst->meta.wi;
iop_a_dst = io1_a_dst;
io2_a_dst = io0_a_dst + a_dst->data.len;
if (a_dst->meta.closed) {
io2_a_dst = iop_a_dst;
}
}
const uint8_t* iop_a_src = NULL;
const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
if (a_src && a_src->data.ptr) {
io0_a_src = a_src->data.ptr;
io1_a_src = io0_a_src + a_src->meta.ri;
iop_a_src = io1_a_src;
io2_a_src = io0_a_src + a_src->meta.wi;
}
uint32_t coro_susp_point = self->private_impl.p_report_metadata;
switch (coro_susp_point) {
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0;
while ( ! wuffs_brotli__decoder__is_fully_flushed(self)) {
status = wuffs_base__make_status(wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(1);
}
self->private_impl.f_md_length = wuffs_base__u32__min(self->private_impl.f_mb_remaining, 16777216u);
self->private_impl.f_md_note_pending = true;
status = wuffs_base__make_status(wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(2);
self->private_impl.f_md_copying = true;
while (self->private_impl.f_mb_remaining > 0u) {
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0u) {
status = wuffs_base__make_status(wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(3);
continue;
} else if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) {
status = wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(4);
continue;
}
v_n = wuffs_private_impl__io_writer__limited_copy_u32_from_reader(
&iop_a_dst, io2_a_dst,((uint32_t)(self->private_impl.f_mb_remaining)), &iop_a_src, io2_a_src);
self->private_impl.f_mb_remaining -= v_n;
}
self->private_impl.f_md_copying = false;
ok:
self->private_impl.p_report_metadata = 0;
goto exit;
}
goto suspend;
suspend:
self->private_impl.p_report_metadata = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0;
goto exit;
exit:
if (a_dst && a_dst->data.ptr) {
a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr));
}
if (a_src && a_src->data.ptr) {
a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr));
}
return status;
}
// -------- func brotli.decoder.initialize_codes // -------- func brotli.decoder.initialize_codes
WUFFS_BASE__GENERATED_C_CODE WUFFS_BASE__GENERATED_C_CODE
+144 -5
View File
@@ -31,7 +31,9 @@
// mostly 1 to 7 bytes), with a random large-window quirk, random // mostly 1 to 7 bytes), with a random large-window quirk, random
// (raw and serialized) dictionaries and a random workbuf length (at // (raw and serialized) dictionaries and a random workbuf length (at
// or above workbuf_len, or exactly the stream's minimum, or one byte // or above workbuf_len, or exactly the stream's minimum, or one byte
// less than that, which must fail). // less than that, which must fail). Some decodes (with a large dst)
// also enable QUIRK_REPORT_METADATA and libbrotlidec's metadata
// callbacks and check that both report the same metadata blocks.
// //
// -enc encodes random inputs with random parameters (quality, lgwin, // -enc encodes random inputs with random parameters (quality, lgwin,
// large window, lgblock, mode, size hint, npostfix, ndirect, // large window, lgblock, mode, size hint, npostfix, ndirect,
@@ -1068,8 +1070,41 @@ typedef struct {
size_t in_used; size_t in_used;
const char* wuffs_status; const char* wuffs_status;
uint64_t calls; uint64_t calls;
size_t md_n; // With report_md: the metadata bytes (see g_wmd and g_cmd).
} dec_result; } dec_result;
// With dec_config.report_md, the Wuffs decoder (QUIRK_REPORT_METADATA) and
// libbrotlidec (BrotliDecoderSetMetadataCallbacks) collect each metadata
// block (its length as 4 little-endian bytes, then its contents) in g_wmd and
// g_cmd. A block's contents can be up to 16 MiB long.
#define MD_CAP ((size_t)40 << 20)
#define MAX_OUT (((size_t)2 << 20) + 4096)
static uint8_t* g_wmd;
static uint8_t* g_cmd;
static size_t g_cmd_n;
static bool g_cmd_overflow;
static void //
c_md_start(void* opaque, size_t size) {
if (MD_CAP - g_cmd_n < 4) {
g_cmd_overflow = true;
return;
}
for (int i = 0; i < 4; i++) {
g_cmd[g_cmd_n++] = (uint8_t)(size >> (8 * i));
}
}
static void //
c_md_chunk(void* opaque, const uint8_t* data, size_t size) {
if (MD_CAP - g_cmd_n < size) {
g_cmd_overflow = true;
return;
}
memcpy(g_cmd + g_cmd_n, data, size);
g_cmd_n += size;
}
static int // static int //
wuffs_status_class(const char* s) { wuffs_status_class(const char* s) {
static const struct { static const struct {
@@ -1143,6 +1178,7 @@ typedef struct {
uint64_t wb_len; // The Wuffs workbuf length. uint64_t wb_len; // The Wuffs workbuf length.
bool expect_bad_wb; // wb_len is one byte too short for the stream. bool expect_bad_wb; // wb_len is one byte too short for the stream.
bool uninit; // WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED. bool uninit; // WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED.
bool report_md; // QUIRK_REPORT_METADATA and metadata callbacks.
size_t out_cap; size_t out_cap;
} dec_config; } dec_config;
@@ -1166,6 +1202,13 @@ wuffs_attach(wuffs_brotli__decoder* dec, const dec_config* dc, size_t* n_ok) {
return status.repr; return status.repr;
} }
} }
if (dc->report_md) {
status = wuffs_brotli__decoder__set_quirk(
dec, WUFFS_BROTLI__QUIRK_REPORT_METADATA, 1);
if (status.repr) {
return status.repr;
}
}
*n_ok = 0; *n_ok = 0;
for (size_t i = 0; i < dc->dicts.n; i++) { for (size_t i = 0; i < dc->dicts.n; i++) {
status = wuffs_brotli__decoder__add_dictionary( status = wuffs_brotli__decoder__add_dictionary(
@@ -1206,6 +1249,12 @@ wuffs_decode(const dec_config* dc,
size_t ri = 0; size_t ri = 0;
size_t wi = 0; size_t wi = 0;
uint64_t max_calls = 3 * ((uint64_t)n + dc->out_cap) + 100; uint64_t max_calls = 3 * ((uint64_t)n + dc->out_cap) + 100;
// With report_md, metadata bytes are written to dst (out) and then moved to
// g_wmd. While md_remaining > 0, dst gets up to md_remaining bytes of slack
// past lout (within the physical buffer's MAX_OUT bytes), which the
// metadata bytes fill before any decoded bytes, so that the decoded bytes
// still fit in lout (like libbrotlidec's, which uses no dst for metadata).
size_t md_remaining = 0;
while (true) { while (true) {
if (++res.calls > max_calls) { if (++res.calls > max_calls) {
res.cls = CLS_BUG; res.cls = CLS_BUG;
@@ -1215,7 +1264,12 @@ wuffs_decode(const dec_config* dc,
wuffs_base__io_buffer src = wuffs_base__io_buffer src =
wuffs_base__ptr_u8__reader((uint8_t*)in, lin, lin == n); wuffs_base__ptr_u8__reader((uint8_t*)in, lin, lin == n);
src.meta.ri = ri; src.meta.ri = ri;
wuffs_base__io_buffer dst = wuffs_base__ptr_u8__writer(out, lout); size_t slack = 0;
if ((md_remaining > 0) && (lout < MAX_OUT)) {
slack = (md_remaining < (MAX_OUT - lout)) ? md_remaining
: (MAX_OUT - lout);
}
wuffs_base__io_buffer dst = wuffs_base__ptr_u8__writer(out, lout + slack);
dst.meta.wi = wi; dst.meta.wi = wi;
wuffs_base__status status = wuffs_base__status status =
wuffs_brotli__decoder__transform_io(g_dec, &dst, &src, wb); wuffs_brotli__decoder__transform_io(g_dec, &dst, &src, wb);
@@ -1224,9 +1278,37 @@ wuffs_decode(const dec_config* dc,
res.wuffs_status = "went backwards"; res.wuffs_status = "went backwards";
break; break;
} }
if (md_remaining > 0) {
size_t k = dst.meta.wi - wi;
k = (k < md_remaining) ? k : md_remaining;
memcpy(g_wmd + res.md_n, out + wi, k);
res.md_n += k;
md_remaining -= k;
memmove(out + wi, out + wi + k, dst.meta.wi - wi - k);
dst.meta.wi -= k;
}
if (dst.meta.wi > lout) {
res.cls = CLS_BUG;
res.wuffs_status = "decoded bytes in the metadata slack";
break;
}
ri = src.meta.ri; ri = src.meta.ri;
wi = dst.meta.wi; wi = dst.meta.wi;
res.wuffs_status = status.repr; res.wuffs_status = status.repr;
if (status.repr == wuffs_base__note__metadata_reported) {
uint64_t len = wuffs_brotli__decoder__metadata_length(g_dec);
if (!dc->report_md || (md_remaining > 0) || (len > (1 << 24)) ||
(MD_CAP - res.md_n < 4 + len)) {
res.cls = CLS_BUG;
res.wuffs_status = "unexpected metadata report";
break;
}
for (int i = 0; i < 4; i++) {
g_wmd[res.md_n++] = (uint8_t)(len >> (8 * i));
}
md_remaining = (size_t)len;
continue;
}
if (g_trace) { if (g_trace) {
printf("wuffs call %" PRIu64 ": lin=%zu lout=%zu -> ri=%zu wi=%zu %s\n", printf("wuffs call %" PRIu64 ": lin=%zu lout=%zu -> ri=%zu wi=%zu %s\n",
res.calls, lin, lout, ri, wi, status.repr ? status.repr : "ok"); res.calls, lin, lout, ri, wi, status.repr ? status.repr : "ok");
@@ -1276,6 +1358,11 @@ c_decode(const dec_config* dc,
if (dc->large_window) { if (dc->large_window) {
BrotliDecoderSetParameter(s, BROTLI_DECODER_PARAM_LARGE_WINDOW, 1); BrotliDecoderSetParameter(s, BROTLI_DECODER_PARAM_LARGE_WINDOW, 1);
} }
g_cmd_n = 0;
g_cmd_overflow = false;
if (dc->report_md) {
BrotliDecoderSetMetadataCallbacks(s, c_md_start, c_md_chunk, NULL);
}
*c_attached = 0; *c_attached = 0;
for (size_t i = 0; i < dc->dicts.n; i++) { for (size_t i = 0; i < dc->dicts.n; i++) {
if (!BrotliDecoderAttachDictionary( if (!BrotliDecoderAttachDictionary(
@@ -1336,6 +1423,7 @@ c_decode(const dec_config* dc,
BrotliDecoderDestroyInstance(s); BrotliDecoderDestroyInstance(s);
res.out_n = wi; res.out_n = wi;
res.in_used = ri; res.in_used = ri;
res.md_n = g_cmd_overflow ? SIZE_MAX : g_cmd_n;
return res; return res;
} }
@@ -1429,6 +1517,10 @@ static uint64_t g_dec_gen_wuffs;
static uint64_t g_dec_wuffs_shorter_output; static uint64_t g_dec_wuffs_shorter_output;
static uint64_t g_dec_known_block_length; static uint64_t g_dec_known_block_length;
static uint64_t g_dec_known_saved; static uint64_t g_dec_known_saved;
static uint64_t g_dec_report_md;
static uint64_t g_dec_md_compared;
static uint64_t g_dec_md_blocks;
static uint64_t g_dec_md_inconclusive;
static int // static int //
cls_index(int cls) { cls_index(int cls) {
@@ -1759,9 +1851,10 @@ run_dec(uint64_t iters, double secs) {
const size_t tmp_cap = (8 << 20); const size_t tmp_cap = (8 << 20);
uint8_t* tmp_in = xmalloc(tmp_cap); uint8_t* tmp_in = xmalloc(tmp_cap);
uint8_t* tmp_out = xmalloc(tmp_cap); uint8_t* tmp_out = xmalloc(tmp_cap);
const size_t max_out = (2 << 20) + 4096; uint8_t* wout = xmalloc(MAX_OUT);
uint8_t* wout = xmalloc(max_out); uint8_t* cout = xmalloc(MAX_OUT);
uint8_t* cout = xmalloc(max_out); g_wmd = xmalloc(MD_CAP);
g_cmd = xmalloc(MD_CAP);
const size_t mcap = (1 << 20); const size_t mcap = (1 << 20);
uint8_t* m = xmalloc(mcap); uint8_t* m = xmalloc(mcap);
@@ -1821,6 +1914,10 @@ run_dec(uint64_t iters, double secs) {
g_dec_with_dicts += (dc.dicts.n > 0); g_dec_with_dicts += (dc.dicts.n > 0);
dc.uninit = chance(500); dc.uninit = chance(500);
dc.out_cap = chance(900) ? ((size_t)1 << 20) + rn(1 << 20) : rn(4096); dc.out_cap = chance(900) ? ((size_t)1 << 20) + rn(1 << 20) : rn(4096);
// Only with a large dst: libbrotlidec reports metadata (via callbacks)
// even when its dst is full, but Wuffs needs dst space for it.
dc.report_md = (dc.out_cap >= ((size_t)1 << 20)) && chance(300);
g_dec_report_md += dc.report_md;
// Attach the dictionaries to both decoders, checking that they agree on // Attach the dictionaries to both decoders, checking that they agree on
// which ones are accepted. libbrotlidec can be left in a modified state // which ones are accepted. libbrotlidec can be left in a modified state
@@ -1899,6 +1996,44 @@ run_dec(uint64_t iters, double secs) {
size_t c_attached = 0; size_t c_attached = 0;
dec_result w = wuffs_decode(&dc, &ch, m, n, wout); dec_result w = wuffs_decode(&dc, &ch, m, n, wout);
dec_result c = c_decode(&dc, &ch, m, n, cout, &c_attached); dec_result c = c_decode(&dc, &ch, m, n, cout, &c_attached);
if (dc.report_md && (w.cls == CLS_NEEDS_MORE_OUTPUT)) {
// A full dst stops Wuffs at a metadata block, but not libbrotlidec,
// so this is inconclusive. Compare without reporting instead.
g_dec_md_inconclusive++;
dc.report_md = false;
w = wuffs_decode(&dc, &ch, m, n, wout);
c = c_decode(&dc, &ch, m, n, cout, &c_attached);
dc.report_md = true;
} else if (dc.report_md) {
// Before an error, both have reported the same metadata, except that
// one can have more of the last block's contents (or also the next
// block's start). After success, they agree exactly.
size_t k = (w.md_n < c.md_n) ? w.md_n : c.md_n;
const char* mwhy = NULL;
if (c.md_n == SIZE_MAX) {
mwhy = "libbrotlidec metadata overflow";
} else if (k && memcmp(g_wmd, g_cmd, k)) {
mwhy = "different metadata";
} else if ((w.cls == CLS_OK) && (c.cls == CLS_OK) &&
(w.md_n != c.md_n)) {
mwhy = "different metadata lengths";
}
if ((w.cls == CLS_OK) && (c.cls == CLS_OK)) {
g_dec_md_compared++;
for (size_t j = 0; j + 4 <= w.md_n;) {
size_t len = (size_t)g_wmd[j] | ((size_t)g_wmd[j + 1] << 8) |
((size_t)g_wmd[j + 2] << 16) |
((size_t)g_wmd[j + 3] << 24);
g_dec_md_blocks++;
j += 4 + len;
}
}
if (mwhy) {
char desc[4096];
describe_dec(desc, sizeof desc, mwhy, s, &dc, &ch, &w, &c, n);
save_failure("md", desc, m, n, &dc.dicts);
}
}
g_dec_decodes += 2; g_dec_decodes += 2;
g_dec_cls_counts[pass][cls_index(c.cls)]++; g_dec_cls_counts[pass][cls_index(c.cls)]++;
const char* why = NULL; const char* why = NULL;
@@ -1948,6 +2083,10 @@ run_dec(uint64_t iters, double secs) {
g_dec_wuffs_shorter_output); g_dec_wuffs_shorter_output);
printf(" known BLOCK_LENGTH_1/2 deviations (chunked)=%" PRIu64 "\n", printf(" known BLOCK_LENGTH_1/2 deviations (chunked)=%" PRIu64 "\n",
g_dec_known_block_length); g_dec_known_block_length);
printf(" report metadata=%" PRIu64 " (compared after success=%" PRIu64
", blocks=%" PRIu64 ", inconclusive=%" PRIu64 ")\n",
g_dec_report_md, g_dec_md_compared, g_dec_md_blocks,
g_dec_md_inconclusive);
static const char* names[23] = { static const char* names[23] = {
"other", "ok", "needs-more-input", "needs-more-output", "other", "ok", "needs-more-input", "needs-more-output",
"-1", "-2", "-3", "-4", "-1", "-2", "-3", "-4",
+70 -10
View File
@@ -15,12 +15,12 @@
// particular its c/dec/decode.c file. Comments like "Mirrors Foo in decode.c" // particular its c/dec/decode.c file. Comments like "Mirrors Foo in decode.c"
// name the C function whose logic (and validation checks) a Wuffs function // name the C function whose logic (and validation checks) a Wuffs function
// mirrors. The C code's explicit BROTLI_STATE_ETC state machine is replaced by // mirrors. The C code's explicit BROTLI_STATE_ETC state machine is replaced by
// Wuffs coroutines. The C code's "safe" and "unsafe" (fast path) variants are // Wuffs coroutines. The C code's "safe" variants (read as few bits as
// implemented once, as the "safe" (read as few bits as possible) variant. // possible, so that decoding can suspend anywhere) are coroutines and its
// "unsafe" fast path (ProcessCommands) is process_commands_fast, which reads
// ahead and gives back the unused bytes (see README.md).
// //
// See README.md for how C error codes map to Wuffs statuses. // See README.md for how C error codes map to Wuffs statuses.
//
// TODO: support metadata callbacks and fast paths.
pub status "#bad block length" pub status "#bad block length"
pub status "#bad block switch" pub status "#bad block switch"
@@ -245,6 +245,17 @@ pub struct decoder? implements base.io_transformer(
quirk_large_window : base.bool, quirk_large_window : base.bool,
large_window : base.bool, large_window : base.bool,
// quirk_report_metadata is the QUIRK_REPORT_METADATA quirk.
// md_length is the length of the last reported metadata block.
// md_note_pending is whether do_transform_io has suspended (with
// "$short write") so that transform_io returns "@metadata reported".
// md_copying is whether do_transform_io is writing a metadata block's
// contents to args.dst.
quirk_report_metadata : base.bool,
md_length : base.u32[..= 0x100_0000],
md_note_pending : base.bool,
md_copying : base.bool,
// started is whether the C code's s->state would no longer be // started is whether the C code's s->state would no longer be
// BROTLI_STATE_UNINITED. Quirks and dictionaries can only be set or // BROTLI_STATE_UNINITED. Quirks and dictionaries can only be set or
// added before then. // added before then.
@@ -355,6 +366,8 @@ pub struct decoder? implements base.io_transformer(
pub func decoder.get_quirk(key: base.u32) base.u64 { pub func decoder.get_quirk(key: base.u32) base.u64 {
if (args.key == QUIRK_LARGE_WINDOW) and this.quirk_large_window { if (args.key == QUIRK_LARGE_WINDOW) and this.quirk_large_window {
return 1 return 1
} else if (args.key == QUIRK_REPORT_METADATA) and this.quirk_report_metadata {
return 1
} }
return 0 return 0
} }
@@ -368,10 +381,22 @@ pub func decoder.set_quirk!(key: base.u32, value: base.u64) base.status {
} }
this.quirk_large_window = args.value > 0 this.quirk_large_window = args.value > 0
return ok return ok
} else if args.key == QUIRK_REPORT_METADATA {
if this.started {
return base."#bad call sequence"
}
this.quirk_report_metadata = args.value > 0
return ok
} }
return base."#unsupported option" return base."#unsupported option"
} }
// metadata_length returns the length of the metadata block that was last
// reported (see QUIRK_REPORT_METADATA).
pub func decoder.metadata_length() base.u64 {
return this.md_length as base.u64
}
pub func decoder.dst_history_retain_length() base.optional_u63 { pub func decoder.dst_history_retain_length() base.optional_u63 {
return this.util.make_optional_u63(has_value: true, value: 0) return this.util.make_optional_u63(has_value: true, value: 0)
} }
@@ -409,7 +434,7 @@ pub func decoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf
if this.dict_total > args.workbuf.length() { if this.dict_total > args.workbuf.length() {
return base."#bad workbuf length" return base."#bad workbuf length"
} }
status =? this.do_transform_io?(src: args.src, status =? this.do_transform_io?(dst: args.dst, src: args.src,
workbuf: args.workbuf[this.dict_total ..], workbuf: args.workbuf[this.dict_total ..],
dictbuf: args.workbuf[.. this.dict_total]) dictbuf: args.workbuf[.. this.dict_total])
if this.dict_total > args.workbuf.length() { if this.dict_total > args.workbuf.length() {
@@ -428,7 +453,10 @@ pub func decoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf
return ok return ok
} else if status == base."$short write" { } else if status == base."$short write" {
if this.is_fully_flushed() { if this.md_note_pending {
this.md_note_pending = false
return base."@metadata reported"
} else if this.is_fully_flushed() and (not this.md_copying) {
continue continue
} }
@@ -471,7 +499,7 @@ pri func decoder.flush!(dst: base.io_writer, workbuf: slice base.u8) {
// Mirrors the BROTLI_STATE_ETC state machine in BrotliDecoderDecompressStream // Mirrors the BROTLI_STATE_ETC state machine in BrotliDecoderDecompressStream
// in decode.c. // in decode.c.
pri func decoder.do_transform_io?(src: base.io_reader, workbuf: slice base.u8, dictbuf: roslice base.u8) { pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, workbuf: slice base.u8, dictbuf: roslice base.u8) {
var w : base.u64 var w : base.u64
var aligned : base.bool var aligned : base.bool
@@ -508,9 +536,11 @@ pri func decoder.do_transform_io?(src: base.io_reader, workbuf: slice base.u8, d
if this.is_metadata { if this.is_metadata {
// BROTLI_STATE_METADATA. Mirrors SkipMetadataBlock. // BROTLI_STATE_METADATA. Mirrors SkipMetadataBlock.
// if this.quirk_report_metadata {
// TODO: support metadata callbacks. this.report_metadata?(dst: args.dst, src: args.src)
args.src.skip?(n: this.mb_remaining as base.u64) } else {
args.src.skip?(n: this.mb_remaining as base.u64)
}
this.mb_remaining = 0 this.mb_remaining = 0
} else if this.mb_remaining > 0 { } else if this.mb_remaining > 0 {
@@ -538,6 +568,36 @@ pri func decoder.do_transform_io?(src: base.io_reader, workbuf: slice base.u8, d
} }
} }
// report_metadata mirrors the C code's metadata callbacks (the start callback
// in BROTLI_STATE_METABLOCK_HEADER and the chunk callbacks in
// SkipMetadataBlock). It waits until every decoded byte has been written to
// args.dst, then has transform_io return "@metadata reported" and then copies
// the metadata block's contents from args.src to args.dst.
pri func decoder.report_metadata?(dst: base.io_writer, src: base.io_reader) {
var n : base.u32
while not this.is_fully_flushed() {
yield? base."$short write"
}
this.md_length = this.mb_remaining.min(no_more_than: 0x100_0000)
this.md_note_pending = true
yield? base."$short write"
this.md_copying = true
while this.mb_remaining > 0 {
if args.dst.length() <= 0 {
yield? base."$short write"
continue
} else if args.src.length() <= 0 {
yield? base."$short read"
continue
}
n = args.dst.limited_copy_u32_from_reader!(up_to: this.mb_remaining as base.u32, r: args.src)
this.mb_remaining ~mod-= n
}
this.md_copying = false
}
// initialize_codes sets up the decoder state that the C code sets in // initialize_codes sets up the decoder state that the C code sets in
// BrotliDecoderStateInit (in state.c). It also builds the fixed table for // BrotliDecoderStateInit (in state.c). It also builds the fixed table for
// decoding the code length code lengths (kCodeLengthPrefixLength and // decoding the code length code lengths (kCodeLengthPrefixLength and
+16
View File
@@ -30,3 +30,19 @@ pri const QUIRKS_BASE : base.u32 = 0x2AB7_9000
// is rejected with "#unsupported large window". With this quirk, a stream that // is rejected with "#unsupported large window". With this quirk, a stream that
// does not start with that escape is decoded as standard brotli. // does not start with that escape is decoded as standard brotli.
pub const QUIRK_LARGE_WINDOW : base.u32 = 0x2AB7_9000 | 0x00 pub const QUIRK_LARGE_WINDOW : base.u32 = 0x2AB7_9000 | 0x00
// When this quirk is set, a positive value makes the decoder report metadata
// blocks, like the C code's BrotliDecoderSetMetadataCallbacks. Zero (the
// default) means that metadata blocks are skipped, like the C code without
// callbacks.
//
// When reporting, at each metadata block, after all of the previously decoded
// bytes have been written to dst, transform_io returns
// base."@metadata reported" and metadata_length returns the block's length
// (the C code's start callback's size argument), which can be zero. The next
// metadata_length bytes that transform_io writes to dst (in subsequent calls)
// are the metadata block's contents (the C code's chunk callbacks' data),
// after which decoded bytes follow as usual.
//
// It must be set before decoding starts. This quirk is decoder-only.
pub const QUIRK_REPORT_METADATA : base.u32 = 0x2AB7_9000 | 0x0A
+373
View File
@@ -1740,6 +1740,377 @@ test_wuffs_brotli_encode_interface() {
return status; return status;
} }
// ---------------- Decoder Metadata Tests
// The metadata tests encode romeo.txt with metadata blocks (whose contents
// are also taken from romeo.txt) and then decode it with
// QUIRK_REPORT_METADATA. The decoded output ("out") and the metadata ("md",
// each block prefixed by its length as 4 little-endian bytes) are collected
// separately.
#define BROTLI_MD_NUM_STEPS 8
#define BROTLI_MD_MAX_BLOCKS 8
static const uint32_t g_brotli_md_step_ops[BROTLI_MD_NUM_STEPS] = {
BROTLI_ENC_OP_EMIT_METADATA, BROTLI_ENC_OP_PROCESS,
BROTLI_ENC_OP_EMIT_METADATA, BROTLI_ENC_OP_EMIT_METADATA,
BROTLI_ENC_OP_FLUSH, BROTLI_ENC_OP_EMIT_METADATA,
BROTLI_ENC_OP_PROCESS, BROTLI_ENC_OP_FINISH,
};
static const size_t g_brotli_md_step_lens[BROTLI_MD_NUM_STEPS] = {
3, 500, 10, 0, 0, 70000, 1000, 50000,
};
// brotli_md_want fills out and md with the expected results for the
// g_brotli_md_step_etc plan applied to src. It sets md_pos[i] to the length of
// out before the i'th metadata block.
static void //
brotli_md_want(wuffs_base__io_buffer* out,
wuffs_base__io_buffer* md,
size_t* md_pos,
const uint8_t* src) {
size_t n_md = 0;
for (size_t i = 0; i < BROTLI_MD_NUM_STEPS; i++) {
size_t n = g_brotli_md_step_lens[i];
if (g_brotli_md_step_ops[i] == BROTLI_ENC_OP_EMIT_METADATA) {
md_pos[n_md++] = out->meta.wi;
wuffs_base__poke_u32le__no_bounds_check(md->data.ptr + md->meta.wi,
(uint32_t)n);
md->meta.wi += 4;
memcpy(md->data.ptr + md->meta.wi, src, n);
md->meta.wi += n;
} else {
memcpy(out->data.ptr + out->meta.wi, src, n);
out->meta.wi += n;
}
src += n;
}
}
// wuffs_brotli_decode_metadata decodes src_ptr[.. src_len] with
// QUIRK_REPORT_METADATA, at most dst_chunk bytes of dst and src_chunk more
// bytes of src per transform_io call.
static const char* //
wuffs_brotli_decode_metadata(wuffs_base__io_buffer* out,
wuffs_base__io_buffer* md,
size_t* md_pos,
size_t* num_md,
const uint8_t* src_ptr,
size_t src_len,
size_t dst_chunk,
size_t src_chunk) {
wuffs_brotli__decoder dec;
CHECK_STATUS("initialize",
wuffs_brotli__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
CHECK_STATUS("set_quirk",
wuffs_brotli__decoder__set_quirk(
&dec, WUFFS_BROTLI__QUIRK_REPORT_METADATA, 1));
if (wuffs_brotli__decoder__get_quirk(
&dec, WUFFS_BROTLI__QUIRK_REPORT_METADATA) != 1) {
RETURN_FAIL("get_quirk: have 0, want 1");
}
uint8_t tmp[4096];
if (dst_chunk > sizeof tmp) {
RETURN_FAIL("dst_chunk is too large");
}
size_t md_remaining = 0;
*num_md = 0;
wuffs_base__io_buffer src = wuffs_base__ptr_u8__reader(
(uint8_t*)src_ptr, src_chunk < src_len ? src_chunk : src_len,
src_chunk >= src_len);
while (true) {
wuffs_base__io_buffer dst = wuffs_base__ptr_u8__writer(tmp, dst_chunk);
wuffs_base__status status = wuffs_brotli__decoder__transform_io(
&dec, &dst, &src, g_work_slice_u8);
uint8_t* p = tmp;
size_t n = dst.meta.wi;
if (md_remaining > 0) {
size_t m = (n < md_remaining) ? n : md_remaining;
if (md->data.len - md->meta.wi < m) {
RETURN_FAIL("md is too short");
}
memcpy(md->data.ptr + md->meta.wi, p, m);
md->meta.wi += m;
md_remaining -= m;
p += m;
n -= m;
}
if (n > 0) {
if (md_remaining > 0) {
RETURN_FAIL("inconsistent md_remaining");
} else if (out->data.len - out->meta.wi < n) {
RETURN_FAIL("out is too short");
}
memcpy(out->data.ptr + out->meta.wi, p, n);
out->meta.wi += n;
}
if (status.repr == NULL) {
break;
} else if (status.repr == wuffs_base__note__metadata_reported) {
if (md_remaining > 0) {
RETURN_FAIL("metadata reported while copying metadata");
} else if (*num_md >= BROTLI_MD_MAX_BLOCKS) {
RETURN_FAIL("too many metadata blocks");
}
uint64_t len = wuffs_brotli__decoder__metadata_length(&dec);
md_pos[(*num_md)++] = out->meta.wi;
if (md->data.len - md->meta.wi < 4) {
RETURN_FAIL("md is too short");
}
wuffs_base__poke_u32le__no_bounds_check(md->data.ptr + md->meta.wi,
(uint32_t)len);
md->meta.wi += 4;
md_remaining = (size_t)len;
} else if (status.repr == wuffs_base__suspension__short_read) {
if (src.meta.closed) {
RETURN_FAIL("short read on closed src");
}
src.meta.wi = (src_len - src.meta.wi > src_chunk) ? src.meta.wi + src_chunk
: src_len;
src.meta.closed = src.meta.wi == src_len;
} else if (status.repr != wuffs_base__suspension__short_write) {
RETURN_FAIL("transform_io: \"%s\"", status.repr);
}
}
if (md_remaining > 0) {
RETURN_FAIL("incomplete metadata");
}
// Setting the quirk after decoding has started fails.
wuffs_base__status status = wuffs_brotli__decoder__set_quirk(
&dec, WUFFS_BROTLI__QUIRK_REPORT_METADATA, 0);
if (status.repr != wuffs_base__error__bad_call_sequence) {
RETURN_FAIL("set_quirk after start: have \"%s\", want \"%s\"", status.repr,
wuffs_base__error__bad_call_sequence);
}
return NULL;
}
static const char* //
do_test_brotli_decode_metadata(const char* (*encode)(wuffs_base__io_buffer*,
wuffs_base__io_buffer*,
const uint32_t*,
const uint32_t*,
size_t,
const uint32_t*,
const size_t*,
size_t,
size_t)) {
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_mimiclib_scratch_slice_u8,
});
CHECK_STRING(read_file(&data, "test/data/romeo.txt"));
// The plan needs 121513 bytes, so repeat romeo.txt (which is shorter).
size_t total = 0;
for (size_t i = 0; i < BROTLI_MD_NUM_STEPS; i++) {
total += g_brotli_md_step_lens[i];
}
if (data.meta.wi == 0) {
RETURN_FAIL("empty romeo.txt");
}
while (data.meta.wi < total) {
size_t n = data.meta.wi;
if (n > total - data.meta.wi) {
n = total - data.meta.wi;
}
memmove(data.data.ptr + data.meta.wi, data.data.ptr, n);
data.meta.wi += n;
}
// The want and have halves of g_want_slice_u8 and g_have_slice_u8.
size_t half_w = g_want_slice_u8.len / 2;
size_t half_h = g_have_slice_u8.len / 2;
wuffs_base__io_buffer want_out = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_want_slice_u8.ptr, half_w),
});
wuffs_base__io_buffer want_md = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_want_slice_u8.ptr + half_w, half_w),
});
size_t want_md_pos[BROTLI_MD_MAX_BLOCKS] = {0};
brotli_md_want(&want_out, &want_md, want_md_pos, data.data.ptr);
static const uint32_t qualities[3] = {0, 5, 11};
static const size_t chunks[4][2] = {{4096, 1 << 30}, {1, 1}, {7, 3}, {3, 64}};
for (int qi = 0; qi < 3; qi++) {
// The decoder uses g_work_slice_u8 as its workbuf, so encode into the
// second half of g_have_slice_u8.
wuffs_base__io_buffer enc = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr + half_h, half_h),
});
wuffs_base__io_buffer src = data;
src.meta.ri = 0;
uint32_t keys[2] = {BROTLI_ENC_PARAM_QUALITY, BROTLI_ENC_PARAM_LGWIN};
uint32_t values[2] = {qualities[qi], 18};
CHECK_STRING((*encode)(&enc, &src, keys, values, 2, g_brotli_md_step_ops,
g_brotli_md_step_lens, BROTLI_MD_NUM_STEPS, 0));
for (int ci = 0; ci < 4; ci++) {
// Decoded output and metadata share the first half of g_have_slice_u8.
wuffs_base__io_buffer have_out = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr, half_h / 2),
});
wuffs_base__io_buffer have_md = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr + half_h / 2,
half_h / 2),
});
size_t have_md_pos[BROTLI_MD_MAX_BLOCKS] = {0};
size_t num_md = 0;
const char* z = wuffs_brotli_decode_metadata(
&have_out, &have_md, have_md_pos, &num_md, enc.data.ptr, enc.meta.wi,
chunks[ci][0], chunks[ci][1]);
if (z) {
RETURN_FAIL("q=%u, chunks=%d: %s", qualities[qi], ci, z);
}
char s[64];
snprintf(s, sizeof s, "q=%u, chunks=%d: out", qualities[qi], ci);
CHECK_STRING(check_io_buffers_equal(s, &have_out, &want_out));
snprintf(s, sizeof s, "q=%u, chunks=%d: md", qualities[qi], ci);
CHECK_STRING(check_io_buffers_equal(s, &have_md, &want_md));
if (num_md != 4) {
RETURN_FAIL("q=%u, chunks=%d: num_md: have %zu, want 4",
qualities[qi], ci, num_md);
}
for (size_t i = 0; i < num_md; i++) {
if (have_md_pos[i] != want_md_pos[i]) {
RETURN_FAIL("q=%u, chunks=%d: md_pos[%zu]: have %zu, want %zu",
qualities[qi], ci, i, have_md_pos[i], want_md_pos[i]);
}
}
}
}
return NULL;
}
const char* //
test_wuffs_brotli_decode_metadata() {
CHECK_FOCUS(__func__);
return do_test_brotli_decode_metadata(wuffs_brotli_encode_plan);
}
#ifdef WUFFS_MIMIC
// mimic_brotli_md_start_func and mimic_brotli_md_chunk_func are libbrotlidec
// metadata callbacks that append to the wuffs_base__io_buffer opaque.
static void //
mimic_brotli_md_start_func(void* opaque, size_t size) {
wuffs_base__io_buffer* md = (wuffs_base__io_buffer*)opaque;
if (md->data.len - md->meta.wi >= 4) {
wuffs_base__poke_u32le__no_bounds_check(md->data.ptr + md->meta.wi,
(uint32_t)size);
md->meta.wi += 4;
} else {
md->meta.closed = true; // Overflow.
}
}
static void //
mimic_brotli_md_chunk_func(void* opaque, const uint8_t* data, size_t size) {
wuffs_base__io_buffer* md = (wuffs_base__io_buffer*)opaque;
if (md->data.len - md->meta.wi >= size) {
memcpy(md->data.ptr + md->meta.wi, data, size);
md->meta.wi += size;
} else {
md->meta.closed = true; // Overflow.
}
}
// test_mimic_brotli_decode_metadata checks that QUIRK_REPORT_METADATA reports
// the same metadata as libbrotlidec's metadata callbacks, for streams that
// libbrotlienc produces (and for the streams of the brotli test data, which
// have no metadata blocks), with a variety of input chunk sizes.
const char* //
test_mimic_brotli_decode_metadata() {
CHECK_FOCUS(__func__);
CHECK_STRING(do_test_brotli_decode_metadata(mimic_brotli_encode));
// Also compare libbrotlidec's callbacks with Wuffs on a libbrotlienc
// stream, fed to libbrotlidec in 5-byte pieces.
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_mimiclib_scratch_slice_u8,
});
CHECK_STRING(read_file(&data, "test/data/romeo.txt"));
size_t half_h = g_have_slice_u8.len / 2;
size_t half_w = g_want_slice_u8.len / 2;
wuffs_base__io_buffer enc = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr + half_h, half_h),
});
static const uint32_t ops[5] = {
BROTLI_ENC_OP_PROCESS, BROTLI_ENC_OP_EMIT_METADATA,
BROTLI_ENC_OP_EMIT_METADATA, BROTLI_ENC_OP_PROCESS,
BROTLI_ENC_OP_FINISH,
};
size_t lens[5] = {100, 200, 0, 300, 0};
lens[4] = data.meta.wi - 600;
uint32_t keys[1] = {BROTLI_ENC_PARAM_QUALITY};
uint32_t values[1] = {9};
CHECK_STRING(mimic_brotli_encode(&enc, &data, keys, values, 1, ops, lens, 5,
0));
wuffs_base__io_buffer c_out = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_want_slice_u8.ptr, half_w),
});
wuffs_base__io_buffer c_md = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_want_slice_u8.ptr + half_w, half_w),
});
BrotliDecoderState* state = BrotliDecoderCreateInstance(NULL, NULL, NULL);
if (!state) {
RETURN_FAIL("BrotliDecoderCreateInstance failed");
}
BrotliDecoderSetMetadataCallbacks(state, mimic_brotli_md_start_func,
mimic_brotli_md_chunk_func, &c_md);
size_t pos = 0;
BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
while (r != BROTLI_DECODER_RESULT_SUCCESS) {
size_t end = (enc.meta.wi - pos > 5) ? pos + 5 : enc.meta.wi;
size_t avail_in = end - pos;
const uint8_t* next_in = enc.data.ptr + pos;
size_t avail_out = c_out.data.len - c_out.meta.wi;
uint8_t* next_out = c_out.data.ptr + c_out.meta.wi;
r = BrotliDecoderDecompressStream(state, &avail_in, &next_in, &avail_out,
&next_out, NULL);
pos = end - avail_in;
c_out.meta.wi = c_out.data.len - avail_out;
if ((r == BROTLI_DECODER_RESULT_ERROR) ||
((r == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) &&
(pos == enc.meta.wi))) {
BrotliDecoderDestroyInstance(state);
RETURN_FAIL("libbrotlidec failed");
}
}
BrotliDecoderDestroyInstance(state);
if (c_md.meta.closed) {
RETURN_FAIL("libbrotlidec metadata overflow");
}
wuffs_base__io_buffer have_out = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr, half_h / 2),
});
wuffs_base__io_buffer have_md = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr + half_h / 2,
half_h / 2),
});
size_t have_md_pos[BROTLI_MD_MAX_BLOCKS] = {0};
size_t num_md = 0;
CHECK_STRING(wuffs_brotli_decode_metadata(&have_out, &have_md, have_md_pos,
&num_md, enc.data.ptr, enc.meta.wi,
5, 5));
CHECK_STRING(check_io_buffers_equal("out", &have_out, &c_out));
CHECK_STRING(check_io_buffers_equal("md", &have_md, &c_md));
if ((num_md != 2) || (have_md_pos[0] != 100) || (have_md_pos[1] != 100)) {
RETURN_FAIL("md_pos: have %zu blocks at %zu, %zu; want 2 at 100, 100",
num_md, have_md_pos[0], have_md_pos[1]);
}
return NULL;
}
#endif // WUFFS_MIMIC
// ---------------- Encoder Mimic Tests // ---------------- Encoder Mimic Tests
#ifdef WUFFS_MIMIC #ifdef WUFFS_MIMIC
@@ -4618,6 +4989,7 @@ proc g_tests[] = {
test_wuffs_brotli_decode_all, test_wuffs_brotli_decode_all,
test_wuffs_brotli_decode_all_with_io_limits, test_wuffs_brotli_decode_all_with_io_limits,
test_wuffs_brotli_decode_interface, test_wuffs_brotli_decode_interface,
test_wuffs_brotli_decode_metadata,
test_wuffs_brotli_decode_malformed, test_wuffs_brotli_decode_malformed,
test_wuffs_brotli_decode_romeo, test_wuffs_brotli_decode_romeo,
test_wuffs_brotli_decode_trailing_data, test_wuffs_brotli_decode_trailing_data,
@@ -4642,6 +5014,7 @@ proc g_tests[] = {
test_mimic_brotli_decode_all, test_mimic_brotli_decode_all,
test_mimic_brotli_decode_differential, test_mimic_brotli_decode_differential,
test_mimic_brotli_decode_malformed, test_mimic_brotli_decode_malformed,
test_mimic_brotli_decode_metadata,
test_mimic_brotli_decode_romeo, test_mimic_brotli_decode_romeo,
test_mimic_brotli_decode_split_block_length, test_mimic_brotli_decode_split_block_length,