From 7875e0bd41a671e05f3d676da7425a7f1f7a386a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 03:07:53 +0000 Subject: [PATCH] std/brotli: size the q10/q11 workbuf with a practical metablock bound workbuf_len's minimum now assumes at most (n / 16) + 1 blocks per category and (n / 64) + 64 first-phase clusters (measured maxima are about 1/83 and 1/258 of n), and 16 BackwardMatches per byte at quality 11 (measured maximum about 7.4). The maximum stays the C worst case. Add q10/q11 mimic tests and workbuf_len checks, and use the minimum in the tests' encode plan driver. --- release/c/wuffs-unsupported-snapshot.c | 110 +++++++++---- std/brotli/encode_metablock.wuffs | 75 +++++++-- std/brotli/encode_store_hq.wuffs | 16 +- test/c/std/brotli.c | 210 ++++++++++++++++++++++++- 4 files changed, 346 insertions(+), 65 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 3d92c4a0..b380afc5 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -79288,7 +79288,9 @@ wuffs_brotli__metablock_builder__sbv_scratch_len( uint64_t a_length, uint64_t a_hs, uint64_t a_symbols_per_histogram, - uint64_t a_max_histograms); + uint64_t a_max_histograms, + uint64_t a_max_blocks, + uint64_t a_max_clusters); WUFFS_BASE__GENERATED_C_CODE static uint64_t @@ -79304,6 +79306,21 @@ wuffs_brotli__metablock_builder__workbuf_len_worst_case( uint64_t a_num_literals, uint64_t a_num_commands); +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__metablock_builder__workbuf_len_practical( + const wuffs_brotli__metablock_builder* self, + uint64_t a_num_literals, + uint64_t a_num_commands); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__metablock_builder__workbuf_len_bounded( + const wuffs_brotli__metablock_builder* self, + uint64_t a_num_literals, + uint64_t a_num_commands, + bool a_practical); + WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__encoder__gbs_histo_off( @@ -79464,13 +79481,6 @@ wuffs_brotli__encoder__hq_zopfli_len( uint32_t a_lgblock, uint64_t a_matches_per_byte); -WUFFS_BASE__GENERATED_C_CODE -static uint64_t -wuffs_brotli__encoder__hq_metablock_budget( - const wuffs_brotli__encoder* self, - uint32_t a_lgwin, - uint32_t a_lgblock); - WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__encoder__hq_var_len( @@ -96060,11 +96070,15 @@ wuffs_brotli__metablock_builder__sbv_scratch_len( uint64_t a_length, uint64_t a_hs, uint64_t a_symbols_per_histogram, - uint64_t a_max_histograms) { + uint64_t a_max_histograms, + uint64_t a_max_blocks, + uint64_t a_max_clusters) { uint64_t v_nh = 0; uint64_t v_a = 0; uint64_t v_b = 0; uint64_t v_nb = 0; + uint64_t v_nc = 0; + uint64_t v_np = 0; if (a_length < 128u) { return 0u; @@ -96072,13 +96086,14 @@ wuffs_brotli__metablock_builder__sbv_scratch_len( v_nh = wuffs_base__u64__sat_add((a_length / wuffs_base__u64__max(a_symbols_per_histogram, 1u)), 1u); v_nh = wuffs_base__u64__min(v_nh, a_max_histograms); v_a = wuffs_base__u64__sat_add(((uint64_t)(wuffs_base__u64__sat_add(v_nh, 1u) * a_hs)), ((uint64_t)(a_length * (wuffs_base__u64__sat_add(v_nh, 7u) >> 3u)))); - v_nb = a_length; + v_nb = wuffs_base__u64__min(a_max_blocks, a_length); + v_nc = wuffs_base__u64__min(a_max_clusters, v_nb); v_b = wuffs_base__u64__sat_add(((uint64_t)(v_nb * wuffs_base__u64__sat_add(a_hs, 24u))), 1024u); wuffs_private_impl__u64__sat_add_indirect(&v_b, ((uint64_t)(wuffs_base__u64__min(v_nb, 64u) * a_hs))); wuffs_private_impl__u64__sat_add_indirect(&v_b, wuffs_base__u64__sat_add(49176u, ((uint64_t)(2u * a_hs)))); - v_nb = ((uint64_t)(64u * a_length)); - v_nb = wuffs_base__u64__min(v_nb, ((uint64_t)((a_length / 2u) * a_length))); - wuffs_private_impl__u64__sat_add_indirect(&v_b, ((uint64_t)(wuffs_base__u64__sat_add(v_nb, 1u) * 24u))); + v_np = ((uint64_t)(64u * v_nc)); + v_np = wuffs_base__u64__min(v_np, ((uint64_t)((v_nc / 2u) * v_nc))); + wuffs_private_impl__u64__sat_add_indirect(&v_b, ((uint64_t)(wuffs_base__u64__sat_add(v_np, 1u) * 24u))); return wuffs_base__u64__sat_add(a_length, wuffs_base__u64__max(v_a, v_b)); } @@ -96108,8 +96123,35 @@ wuffs_brotli__metablock_builder__workbuf_len_worst_case( const wuffs_brotli__metablock_builder* self, uint64_t a_num_literals, uint64_t a_num_commands) { + return wuffs_brotli__metablock_builder__workbuf_len_bounded(self, a_num_literals, a_num_commands, false); +} + +// -------- func brotli.metablock_builder.workbuf_len_practical + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__metablock_builder__workbuf_len_practical( + const wuffs_brotli__metablock_builder* self, + uint64_t a_num_literals, + uint64_t a_num_commands) { + return wuffs_brotli__metablock_builder__workbuf_len_bounded(self, a_num_literals, a_num_commands, true); +} + +// -------- func brotli.metablock_builder.workbuf_len_bounded + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__metablock_builder__workbuf_len_bounded( + const wuffs_brotli__metablock_builder* self, + uint64_t a_num_literals, + uint64_t a_num_commands, + bool a_practical) { uint64_t v_l = 0; uint64_t v_c = 0; + uint64_t v_bl = 0; + uint64_t v_bc = 0; + uint64_t v_cl = 0; + uint64_t v_cc = 0; uint64_t v_tl = 0; uint64_t v_tc = 0; uint64_t v_n = 0; @@ -96117,25 +96159,41 @@ wuffs_brotli__metablock_builder__workbuf_len_worst_case( v_l = wuffs_base__u64__min(a_num_literals, 4294967295u); v_c = wuffs_base__u64__min(a_num_commands, 4294967295u); + v_bl = v_l; + v_bc = v_c; + v_cl = v_l; + v_cc = v_c; + if (a_practical) { + v_bl = ((v_l / 16u) + 1u); + v_bc = ((v_c / 16u) + 1u); + v_cl = ((v_l / 64u) + 64u); + v_cc = ((v_c / 64u) + 64u); + } v_n = wuffs_base__u64__sat_add(((uint64_t)(5u * wuffs_base__u64__max(v_l, 1u))), ((uint64_t)(10u * wuffs_base__u64__max(v_c, 1u)))); v_split = wuffs_base__u64__sat_add(v_l, wuffs_brotli__metablock_builder__sbv_scratch_len(self, v_l, 1040u, 544u, - 100u)); + 100u, + v_bl, + v_cl)); v_split = wuffs_base__u64__max(v_split, wuffs_base__u64__sat_add(((uint64_t)(2u * v_c)), wuffs_brotli__metablock_builder__sbv_scratch_len(self, v_c, 2832u, 530u, - 50u))); + 50u, + v_bc, + v_cc))); v_split = wuffs_base__u64__max(v_split, wuffs_base__u64__sat_add(((uint64_t)(2u * v_c)), wuffs_brotli__metablock_builder__sbv_scratch_len(self, v_c, 2192u, 544u, - 50u))); + 50u, + v_bc, + v_cc))); wuffs_private_impl__u64__sat_add_indirect(&v_n, wuffs_base__u64__max(v_split, 2192u)); - v_tl = wuffs_base__u64__min(wuffs_base__u64__max(v_l, 1u), 256u); - v_tc = wuffs_base__u64__min(wuffs_base__u64__max(v_c, 1u), 256u); + v_tl = wuffs_base__u64__min(wuffs_base__u64__max(wuffs_base__u64__min(v_cl, v_l), 1u), 256u); + v_tc = wuffs_base__u64__min(wuffs_base__u64__max(wuffs_base__u64__min(v_cc, v_c), 1u), 256u); wuffs_private_impl__u64__sat_add_indirect(&v_n, ((v_tc * 2832u) + (v_tl * 64u * 4u) + (v_tl * 64u * 1040u) + @@ -97390,20 +97448,6 @@ wuffs_brotli__encoder__hq_zopfli_len( return wuffs_base__u64__sat_add((((16u * (v_n + 1u)) + (4u * (v_n + 2u))) + (4u * v_n)), ((uint64_t)(8u * wuffs_base__u64__sat_add(((uint64_t)(a_matches_per_byte * v_n)), 128u)))); } -// -------- func brotli.encoder.hq_metablock_budget - -WUFFS_BASE__GENERATED_C_CODE -static uint64_t -wuffs_brotli__encoder__hq_metablock_budget( - const wuffs_brotli__encoder* self, - uint32_t a_lgwin, - uint32_t a_lgblock) { - uint64_t v_l = 0; - - v_l = wuffs_brotli__encoder__hq_max_literals(self, a_lgwin, a_lgblock); - return wuffs_base__u64__sat_add(((uint64_t)(24u * v_l)), 67108864u); -} - // -------- func brotli.encoder.hq_var_len WUFFS_BASE__GENERATED_C_CODE @@ -97422,7 +97466,7 @@ wuffs_brotli__encoder__hq_var_len( v_m = wuffs_brotli__metablock_builder__workbuf_len_worst_case(&self->private_data.f_mb, wuffs_brotli__encoder__hq_max_literals(self, a_lgwin, a_lgblock), wuffs_brotli__encoder__commands_capacity(self, a_quality, a_lgwin, a_lgblock)); } else { v_z = wuffs_brotli__encoder__hq_zopfli_len(self, a_quality, a_lgblock, 16u); - v_m = wuffs_brotli__encoder__hq_metablock_budget(self, a_lgwin, a_lgblock); + v_m = wuffs_brotli__metablock_builder__workbuf_len_practical(&self->private_data.f_mb, wuffs_brotli__encoder__hq_max_literals(self, a_lgwin, a_lgblock), wuffs_brotli__encoder__commands_capacity(self, a_quality, a_lgwin, a_lgblock)); } return wuffs_base__u64__max(v_z, v_m); } diff --git a/std/brotli/encode_metablock.wuffs b/std/brotli/encode_metablock.wuffs index 6cc0743f..fbbd15b4 100644 --- a/std/brotli/encode_metablock.wuffs +++ b/std/brotli/encode_metablock.wuffs @@ -974,12 +974,17 @@ pri func metablock_builder.optimize_huffman_counts_for_rle!(wb: slice base.u8, c // sbv_scratch_len is an upper bound on how much workbuf (beyond the split // arrays) split_byte_vector uses, for length symbols with the given -// histogram size, symbols_per_histogram and max_histograms. -pri func metablock_builder.sbv_scratch_len(length: base.u64, hs: base.u64, symbols_per_histogram: base.u64, max_histograms: base.u64) base.u64 { +// histogram size, symbols_per_histogram and max_histograms, when FindBlocks +// makes at most max_blocks blocks and ClusterBlocks' first phase makes at +// most max_clusters clusters (both at most length). +pri func metablock_builder.sbv_scratch_len(length: base.u64, hs: base.u64, symbols_per_histogram: base.u64, max_histograms: base.u64, + max_blocks: base.u64, max_clusters: base.u64) base.u64 { var nh : base.u64 var a : base.u64 var b : base.u64 var nb : base.u64 + var nc : base.u64 + var np : base.u64 if args.length < 128 { return 0 @@ -988,14 +993,15 @@ pri func metablock_builder.sbv_scratch_len(length: base.u64, hs: base.u64, symbo nh = nh.min(no_more_than: args.max_histograms) // The histograms and the switch signal bitmaps. a = ((nh ~sat+ 1) ~mod* args.hs) ~sat+ (args.length ~mod* ((nh ~sat+ 7) >> 3)) - // cluster_blocks, with num_blocks (and num_clusters) at most length. - nb = args.length + // cluster_blocks, with num_blocks at most nb and num_clusters at most nc. + nb = args.max_blocks.min(no_more_than: args.length) + nc = args.max_clusters.min(no_more_than: nb) b = (nb ~mod* (args.hs ~sat+ 24)) ~sat+ 1024 b ~sat+= (nb.min(no_more_than: 64)) ~mod* args.hs b ~sat+= (2049 * HISTOGRAM_PAIR_SIZE) ~sat+ (2 ~mod* args.hs) - nb = 64 ~mod* args.length - nb = nb.min(no_more_than: (args.length / 2) ~mod* args.length) - b ~sat+= (nb ~sat+ 1) ~mod* HISTOGRAM_PAIR_SIZE + np = 64 ~mod* nc + np = np.min(no_more_than: (nc / 2) ~mod* nc) + b ~sat+= (np ~sat+ 1) ~mod* HISTOGRAM_PAIR_SIZE return args.length ~sat+ a.max(no_less_than: b) } @@ -1019,8 +1025,31 @@ pri func metablock_builder.cluster_scratch_len(in_size: base.u64, hs: base.u64) // time (e.g. one FindBlocks block per symbol), so it is much larger than what // typical inputs need. The wb_high field records the actual high water mark. pri func metablock_builder.workbuf_len_worst_case(num_literals: base.u64, num_commands: base.u64) base.u64 { + return this.workbuf_len_bounded(num_literals: args.num_literals, num_commands: args.num_commands, practical: false) +} + +// workbuf_len_practical is like workbuf_len_worst_case but it assumes these +// caps (see README.md, "Qualities 10 and 11 workbuf length"): +// - FindBlocks makes at most (n / 16) + 1 literal blocks and at most +// (n / 16) + 1 command (or distance) blocks, for n symbols of that +// category, and +// - ClusterBlocks' first phase makes at most (n / 64) + 64 clusters, +// which hold, with a wide margin, for every input that the C code was +// measured on. An input that breaks a cap may still fit (the caps are +// applied independently), but it may also need more than this. +pri func metablock_builder.workbuf_len_practical(num_literals: base.u64, num_commands: base.u64) base.u64 { + return this.workbuf_len_bounded(num_literals: args.num_literals, num_commands: args.num_commands, practical: true) +} + +// workbuf_len_bounded implements workbuf_len_worst_case (if not practical) +// and workbuf_len_practical (if practical). +pri func metablock_builder.workbuf_len_bounded(num_literals: base.u64, num_commands: base.u64, practical: base.bool) base.u64 { var l : base.u64 var c : base.u64 + var bl : base.u64 + var bc : base.u64 + var cl : base.u64 + var cc : base.u64 var tl : base.u64 var tc : base.u64 var n : base.u64 @@ -1030,21 +1059,37 @@ pri func metablock_builder.workbuf_len_worst_case(num_literals: base.u64, num_co // keep the arithmetic below from overflowing) change nothing. l = args.num_literals.min(no_more_than: 0xFFFF_FFFF) c = args.num_commands.min(no_more_than: 0xFFFF_FFFF) + // The number of blocks (bl and bc) and of first phase clusters (cl and + // cc) for the literals and for the commands (there are at most as many + // distances as commands). + bl = l + bc = c + cl = l + cc = c + if args.practical { + bl = (l / 16) + 1 + bc = (c / 16) + 1 + cl = (l / 64) + 64 + cc = (c / 64) + 64 + } // The split arrays (types and lengths) for the three categories. n = (5 ~mod* l.max(no_less_than: 1)) ~sat+ (10 ~mod* c.max(no_less_than: 1)) // split_block's scratch. - split = l ~sat+ this.sbv_scratch_len(length: l, hs: HISTOGRAM_LITERAL_SIZE, symbols_per_histogram: 544, max_histograms: 100) + split = l ~sat+ this.sbv_scratch_len(length: l, hs: HISTOGRAM_LITERAL_SIZE, symbols_per_histogram: 544, max_histograms: 100, + max_blocks: bl, max_clusters: cl) split = split.max(no_less_than: (2 ~mod* c) ~sat+ - this.sbv_scratch_len(length: c, hs: HISTOGRAM_COMMAND_SIZE, symbols_per_histogram: 530, max_histograms: 50)) + this.sbv_scratch_len(length: c, hs: HISTOGRAM_COMMAND_SIZE, symbols_per_histogram: 530, max_histograms: 50, + max_blocks: bc, max_clusters: cc)) split = split.max(no_less_than: (2 ~mod* c) ~sat+ - this.sbv_scratch_len(length: c, hs: HISTOGRAM_DISTANCE_SIZE, symbols_per_histogram: 544, max_histograms: 50)) + this.sbv_scratch_len(length: c, hs: HISTOGRAM_DISTANCE_SIZE, symbols_per_histogram: 544, max_histograms: 50, + max_blocks: bc, max_clusters: cc)) n ~sat+= split.max(no_less_than: HISTOGRAM_DISTANCE_SIZE) // The number of block types is at most 256 (MAX_NUMBER_OF_HISTOGRAMS) - // and at most the number of symbols (but at least 1). tl bounds the - // literal block types and tc bounds both the command and the distance - // block types (there are at most as many distances as commands). - tl = l.max(no_less_than: 1).min(no_more_than: 256) - tc = c.max(no_less_than: 1).min(no_more_than: 256) + // and at most the number of first phase clusters (but at least 1). tl + // bounds the literal block types and tc bounds both the command and the + // distance block types. + tl = cl.min(no_more_than: l).max(no_less_than: 1).min(no_more_than: 256) + tc = cc.min(no_more_than: c).max(no_less_than: 1).min(no_more_than: 256) // The MetaBlockSplit's histograms and context maps, plus the input // histograms for clustering. n ~sat+= (tc * HISTOGRAM_COMMAND_SIZE) + (tl * 64 * 4) + (tl * 64 * HISTOGRAM_LITERAL_SIZE) + diff --git a/std/brotli/encode_store_hq.wuffs b/std/brotli/encode_store_hq.wuffs index 83805810..ab2a6f18 100644 --- a/std/brotli/encode_store_hq.wuffs +++ b/std/brotli/encode_store_hq.wuffs @@ -62,18 +62,6 @@ pri func encoder.hq_zopfli_len(quality: base.u32[..= 11], lgblock: base.u32[..= (8 ~mod* ((args.matches_per_byte ~mod* n) ~sat+ MAX_NUM_MATCHES_H10)) } -// hq_metablock_budget returns the variable region length that the minimum -// workbuf length reserves for the metablock_builder: room for its block -// split arrays and BrotliSplitBlock's per-symbol scratch arrays at the -// largest metablock size, plus a fixed allowance for the histograms and -// clustering state. It is not a worst case (see README.md). -pri func encoder.hq_metablock_budget(lgwin: base.u32[..= 30], lgblock: base.u32[..= 30]) base.u64 { - var l : base.u64 - - l = this.hq_max_literals(lgwin: args.lgwin, lgblock: args.lgblock) - return (24 ~mod* l) ~sat+ 0x400_0000 -} - // hq_var_len returns the minimum (if not worst_case) or the worst case (if // worst_case) length of the variable region, at qualities 10 and 11. pri func encoder.hq_var_len(quality: base.u32[..= 11], lgwin: base.u32[..= 30], lgblock: base.u32[..= 30], worst_case: base.bool) base.u64 { @@ -87,7 +75,9 @@ pri func encoder.hq_var_len(quality: base.u32[..= 11], lgwin: base.u32[..= 30], num_commands: this.commands_capacity(quality: args.quality, lgwin: args.lgwin, lgblock: args.lgblock)) } else { z = this.hq_zopfli_len(quality: args.quality, lgblock: args.lgblock, matches_per_byte: HQ_MATCHES_PER_BYTE_BUDGET) - m = this.hq_metablock_budget(lgwin: args.lgwin, lgblock: args.lgblock) + m = this.mb.workbuf_len_practical( + num_literals: this.hq_max_literals(lgwin: args.lgwin, lgblock: args.lgblock), + num_commands: this.commands_capacity(quality: args.quality, lgwin: args.lgwin, lgblock: args.lgblock)) } return z.max(no_less_than: m) } diff --git a/test/c/std/brotli.c b/test/c/std/brotli.c index 869e5a5e..945e5eaf 100644 --- a/test/c/std/brotli.c +++ b/test/c/std/brotli.c @@ -896,7 +896,7 @@ wuffs_brotli_encode_plan(wuffs_base__io_buffer* dst, wuffs_base__slice_u8 workbuf = g_work_slice_u8; // Use the minimum, so that the qualities 10 and 11 practical bound (see // the README) is exercised. At other qualities, min_incl == max_incl. - if (workbuf_len.min_incl > (1ull << 31)) { + if (workbuf_len.min_incl > (1ull << 32)) { ret = "workbuf_len is too large"; goto cleanup; } else if (workbuf_len.min_incl > g_work_slice_u8.len) { @@ -1126,7 +1126,7 @@ const char* // do_test_brotli_encode_round_trip_file(const char* name, const uint8_t* data, size_t len) { - for (uint32_t q = 0; q <= 9; q++) { + for (uint32_t q = 0; q <= 11; q++) { for (int k = (q <= 1) ? 0 : 1; k < 3; k++) { brotli_enc_config c = make_brotli_enc_config(q, (k == 0) ? 10 : 22); if (k == 2) { @@ -1191,7 +1191,7 @@ test_wuffs_brotli_encode_metadata_round_trip() { }); CHECK_STRING(read_file(&data, "test/data/romeo.txt")); static const size_t lens[] = {0, 1, 100, 70000}; - for (uint32_t q = 0; q <= 9; q++) { + for (uint32_t q = 0; q <= 11; q++) { for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(lens); i++) { for (uint32_t pos = 1; pos <= 3; pos++) { brotli_enc_config c = make_brotli_enc_config(q, 18); @@ -1338,6 +1338,48 @@ test_wuffs_brotli_encode_parameters() { src = make_io_buffer_from_string("", 0); dst.meta.wi = 0; + // At qualities 10 and 11, workbuf_len is a range (see the README's + // "Qualities 10 and 11 workbuf length" section). + for (uint32_t q = 10; q <= 12; q++) { + static const struct { + uint32_t lgwin; + uint64_t min_incl; + uint64_t max_incl; + } want[] = { + {16, 96686024ull, 302507168ull}, + {22, 422539208ull, 5364573344ull}, + {24, 813133768ull, 10148177056ull}, + }; + for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(want); i++) { + free(enc); + enc = wuffs_brotli__encoder__alloc(); + if (!enc) { + RETURN_FAIL("alloc failed"); + } + wuffs_brotli__encoder__set_quirk( + enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, q); + wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LGWIN, + want[i].lgwin); + wuffs_base__range_ii_u64 r = wuffs_brotli__encoder__workbuf_len(enc); + if ((r.min_incl != want[i].min_incl) || + (r.max_incl != want[i].max_incl)) { + RETURN_FAIL("workbuf_len at q=%u lgwin=%u: have %" PRIu64 + "..%" PRIu64 ", want %" PRIu64 "..%" PRIu64, + q, want[i].lgwin, r.min_incl, r.max_incl, + want[i].min_incl, want[i].max_incl); + } + } + // A workbuf shorter than the minimum is rejected up front. + src = make_io_buffer_from_string("abc", 3); + dst.meta.wi = 0; + status = wuffs_brotli__encoder__transform_io( + enc, &dst, &src, wuffs_base__make_slice_u8(g_work_slice_u8.ptr, 1000)); + BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_workbuf_length, + "short workbuf at quality 10..11"); + } + src = make_io_buffer_from_string("", 0); + dst.meta.wi = 0; + // Large-window brotli at qualities 3 to 9 is unsupported. for (uint32_t q = 3; q <= 9; q++) { bool large_window = true; @@ -1765,11 +1807,168 @@ test_mimic_brotli_encode_parity_q2_params() { return NULL; } +// test_mimic_brotli_encode_parity_q10_grid is like +// test_mimic_brotli_encode_parity_q2_grid but for qualities 10 and 11, with +// shorter inputs (to keep it fast). +const char* // +test_mimic_brotli_encode_parity_q10_grid() { + CHECK_FOCUS(__func__); + static const uint32_t lgwins[] = {10, 24}; + static const uint32_t chunkings[] = { + BROTLI_ENC_CHUNK_SINGLE, + BROTLI_ENC_CHUNK_7, + BROTLI_ENC_CHUNK_RANDOM, + }; + wuffs_base__io_buffer data = ((wuffs_base__io_buffer){ + .data = g_mimiclib_scratch_slice_u8, + }); + for (size_t f = 0; f < WUFFS_TESTLIB_ARRAY_SIZE(brotli_enc_parity_filenames); + f++) { + data.meta.ri = 0; + data.meta.wi = 0; + data.meta.closed = false; + CHECK_STRING(read_file(&data, brotli_enc_parity_filenames[f])); + for (uint32_t q = 10; q <= 11; q++) { + for (size_t w = 0; w < WUFFS_TESTLIB_ARRAY_SIZE(lgwins); w++) { + for (uint32_t mode = 0; mode <= 2; mode++) { + for (size_t k = 0; k < WUFFS_TESTLIB_ARRAY_SIZE(chunkings); k++) { + uint32_t ch = chunkings[k]; + for (uint32_t fl = 0; fl < BROTLI_ENC_NUM_FLUSHES; fl++) { + brotli_enc_config c = make_brotli_enc_config(q, lgwins[w]); + brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_MODE, mode); + c.chunking = ch; + c.flush = fl; + c.seed = (f << 16) | (w << 8) | (mode << 4) | fl | (q << 20); + // Use the first 4 KiB (7-byte chunks) or 24 KiB (otherwise) + // of the input. + size_t n = data.meta.wi; + if ((ch == BROTLI_ENC_CHUNK_7) && (n > 4096)) { + n = 4096; + } else if (n > 24576) { + n = 24576; + } + char s[256]; + snprintf(s, sizeof s, + "%s: q=%u lgwin=%u mode=%u chunking=%u flush=%u", + brotli_enc_parity_filenames[f], q, lgwins[w], mode, ch, + fl); + CHECK_STRING( + do_test_mimic_brotli_encode(s, &c, data.data.ptr, n)); + } + } + } + } + } + } + return NULL; +} + +const char* // +test_mimic_brotli_encode_parity_q10_metadata() { + CHECK_FOCUS(__func__); + static const size_t lens[] = {0, 1, 100, 70000}; + wuffs_base__io_buffer data = ((wuffs_base__io_buffer){ + .data = g_mimiclib_scratch_slice_u8, + }); + CHECK_STRING(read_file(&data, "test/data/romeo.txt")); + for (uint32_t q = 10; q <= 11; q++) { + for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(lens); i++) { + for (uint32_t pos = 1; pos <= 3; pos++) { + for (uint32_t fl = 0; fl < BROTLI_ENC_NUM_FLUSHES; fl++) { + for (size_t dc = 0; dc < 2; dc++) { + brotli_enc_config c = make_brotli_enc_config(q, 16); + c.chunking = dc ? BROTLI_ENC_CHUNK_RANDOM : BROTLI_ENC_CHUNK_7; + c.flush = fl; + c.metadata_position = pos; + c.metadata_len = lens[i]; + c.dst_chunk = dc ? 5 : 0; + c.seed = (i << 8) | pos | (q << 12); + char s[256]; + snprintf(s, sizeof s, + "q=%u metadata_len=%zu pos=%u flush=%u dst_chunk=%zu", q, + lens[i], pos, fl, c.dst_chunk); + CHECK_STRING(do_test_mimic_brotli_encode(s, &c, data.data.ptr, + data.meta.wi)); + } + } + } + } + } + return NULL; +} + +const char* // +test_mimic_brotli_encode_parity_q10_params() { + CHECK_FOCUS(__func__); + wuffs_base__io_buffer data = ((wuffs_base__io_buffer){ + .data = g_mimiclib_scratch_slice_u8, + }); + CHECK_STRING(read_file(&data, "test/data/brotli/alice29.txt")); + // Use the first 40 KiB, to keep it fast. + uint32_t n = (uint32_t)data.meta.wi; + if (n > 40960) { + n = 40960; + } + // Each row is up to three (BrotliEncoderParameter, value) pairs. A zero key + // (other than in the first pair) ends the row. + static const uint32_t extras[][6] = { + {BROTLI_ENC_PARAM_SIZE_HINT, 0xFFFFFFFF}, // Replaced by the exact size. + {BROTLI_ENC_PARAM_SIZE_HINT, 1}, + {BROTLI_ENC_PARAM_SIZE_HINT, 1u << 30}, + {BROTLI_ENC_PARAM_LGBLOCK, 16}, + {BROTLI_ENC_PARAM_LGBLOCK, 17}, + {BROTLI_ENC_PARAM_LGBLOCK, 24}, + {BROTLI_ENC_PARAM_DISABLE_LITERAL_CONTEXT_MODELING, 1}, + {BROTLI_ENC_PARAM_DISABLE_LITERAL_CONTEXT_MODELING, 1, // + BROTLI_ENC_PARAM_MODE, 1}, + {BROTLI_ENC_PARAM_MODE, 1}, + {BROTLI_ENC_PARAM_MODE, 2}, + {BROTLI_ENC_PARAM_NPOSTFIX, 1}, + {BROTLI_ENC_PARAM_NPOSTFIX, 2, BROTLI_ENC_PARAM_NDIRECT, 12}, + {BROTLI_ENC_PARAM_NPOSTFIX, 3, BROTLI_ENC_PARAM_NDIRECT, 120}, + {BROTLI_ENC_PARAM_NPOSTFIX, 0, BROTLI_ENC_PARAM_NDIRECT, 15}, + {BROTLI_ENC_PARAM_MODE, 2, BROTLI_ENC_PARAM_NPOSTFIX, 3, + BROTLI_ENC_PARAM_NDIRECT, 30}, + {BROTLI_ENC_PARAM_STREAM_OFFSET, 1}, + {BROTLI_ENC_PARAM_STREAM_OFFSET, 1000}, + {BROTLI_ENC_PARAM_STREAM_OFFSET, 1u << 30}, + {BROTLI_ENC_PARAM_LGWIN, 0xFFFFFFFF}, // (int)-1. + {BROTLI_ENC_PARAM_SIZE_HINT, 1u << 21, // + BROTLI_ENC_PARAM_LGBLOCK, 18}, + }; + for (uint32_t q = 10; q <= 11; q++) { + for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(extras); i++) { + for (uint32_t ch = 0; ch < BROTLI_ENC_NUM_CHUNKINGS; ch += 4) { + brotli_enc_config c = make_brotli_enc_config(q, 22); + char s[256]; + int k = snprintf(s, sizeof s, "q=%u chunking=%u", q, ch); + for (size_t j = 0; (j < 6) && ((j == 0) || extras[i][j]); j += 2) { + uint32_t v = extras[i][j + 1]; + if ((extras[i][j] == BROTLI_ENC_PARAM_SIZE_HINT) && + (v == 0xFFFFFFFF)) { + v = n; + } + brotli_enc_config__add_param(&c, extras[i][j], v); + if ((k > 0) && ((size_t)k < sizeof s)) { + k += snprintf(s + k, sizeof s - (size_t)k, " param[%u]=%u", + extras[i][j], v); + } + } + c.chunking = ch; + c.flush = BROTLI_ENC_FLUSH_MIDDLE; + c.seed = i | (q << 8); + CHECK_STRING(do_test_mimic_brotli_encode(s, &c, data.data.ptr, n)); + } + } + } + return NULL; +} + const char* // do_test_mimic_brotli_encode_file(const char* name, const uint8_t* data, size_t len) { - for (uint32_t q = 0; q <= 9; q++) { + for (uint32_t q = 0; q <= 11; q++) { for (uint32_t w = 10; w <= 24; w += 14) { brotli_enc_config c = make_brotli_enc_config(q, w); c.chunking = (w == 10) ? BROTLI_ENC_CHUNK_SINGLE : BROTLI_ENC_CHUNK_RANDOM; @@ -3220,6 +3419,9 @@ proc g_tests[] = { test_mimic_brotli_encode_parity_q2_grid, test_mimic_brotli_encode_parity_q2_metadata, test_mimic_brotli_encode_parity_q2_params, + test_mimic_brotli_encode_parity_q10_grid, + test_mimic_brotli_encode_parity_q10_metadata, + test_mimic_brotli_encode_parity_q10_params, #endif // WUFFS_MIMIC