Merge claude/project-thread-0izcgq (q10/q11 dictionaries)

This commit is contained in:
Claude committed 2026-10-27 12:38:06 +00:00
commit c5c1c00ba6
11 files changed
+898 -135

No files matched your search

+102 -58
View File
@@ -1604,30 +1604,54 @@ test_wuffs_brotli_encode_parameters() {
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_workbuf_length,
"short workbuf at quality 10..11");
}
// Compound dictionaries are not supported (yet) at qualities 10 and 11.
{
// Compound dictionaries at qualities 10 and 11. The want bytes are
// libbrotlienc's output (with the dictionary prepared and attached, lgwin
// 16 and BROTLI_OPERATION_FINISH), which is the same at both qualities.
for (uint32_t q = 10; q <= 11; q++) {
free(enc);
enc = wuffs_brotli__encoder__alloc();
if (!enc) {
RETURN_FAIL("alloc failed");
}
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY,
q);
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LGWIN, 16);
static const char dict_bytes[] = "a raw dictionary, for quality 11";
static const char input[] = "for quality 10 or 11, a raw dictionary";
static const uint8_t want[] = {
0xA2, 0x04, 0x00, 0xBF, 0x34, 0x08, 0x0B, 0x4C, 0x8C, 0x8C,
0x20, 0x18, 0x71, 0xEA, 0x81, 0x22, 0x4A, 0xEC, 0x4B,
};
wuffs_base__slice_u8 d = wuffs_base__make_slice_u8(
(uint8_t*)dict_bytes, sizeof dict_bytes - 1);
// The workbuf (whose prefix holds the prepared dictionary) is longer
// than g_work_slice_u8, so it is malloc'ed and then realloc'ed.
uint64_t n = wuffs_brotli__encoder__add_dictionary_workbuf_len(enc, d.len);
BROTLI_ENC_CHECK(n <= g_work_slice_u8.len, "add_dictionary_workbuf_len");
uint8_t* p = malloc((size_t)n);
BROTLI_ENC_CHECK(p, "malloc failed");
status = wuffs_brotli__encoder__add_dictionary(
enc, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, d,
wuffs_base__make_slice_u8(g_work_slice_u8.ptr, (size_t)n));
BROTLI_ENC_CHECK(!status.repr, "add_dictionary at quality 11");
src = make_io_buffer_from_string("abc", 3);
wuffs_base__make_slice_u8(p, (size_t)n));
if (status.repr) {
free(p);
}
BROTLI_ENC_CHECK(!status.repr, "add_dictionary at quality 10..11");
n = wuffs_brotli__encoder__workbuf_len(enc).min_incl;
uint8_t* p2 = realloc(p, (size_t)n);
if (!p2) {
free(p);
}
BROTLI_ENC_CHECK(p2, "realloc failed");
src = make_io_buffer_from_string(input, sizeof input - 1);
src.meta.closed = true;
dst.meta.wi = 0;
status = wuffs_brotli__encoder__transform_io(enc, &dst, &src,
g_work_slice_u8);
BROTLI_ENC_CHECK(
status.repr == wuffs_brotli__error__unsupported_encoder_quality,
"dictionary at quality 11");
status = wuffs_brotli__encoder__transform_io(
enc, &dst, &src, wuffs_base__make_slice_u8(p2, (size_t)n));
free(p2);
BROTLI_ENC_CHECK(!status.repr, "dictionary at quality 10..11");
BROTLI_ENC_CHECK((dst.meta.wi == sizeof want) &&
!memcmp(dst.data.ptr, want, sizeof want),
"dictionary at quality 10..11: output bytes");
}
src = make_io_buffer_from_string("", 0);
dst.meta.wi = 0;
@@ -4186,11 +4210,16 @@ do_test_brotli_encode_dictionary_grid(bool mimic) {
r_len);
const uint8_t* data = (set == 4) ? r : (a + (a_len / 2));
size_t data_len = (set == 4) ? r_len : (a_len - (a_len / 2));
for (uint32_t q = 0; (q <= 9) && !ret; q++) {
for (uint32_t q = 0; (q <= 11) && !ret; q++) {
for (size_t i = 0; (i < WUFFS_TESTLIB_ARRAY_SIZE(configs)) && !ret;
i++) {
if (!mimic && (i >= 2) && (q & 1)) {
continue;
} else if ((q >= 10) &&
((i == 2) || (i == 7) || (!mimic && (i != 0)) ||
((q == 11) && (i & 1)))) {
// Qualities 10 and 11 are slow, so they skip some configs.
continue;
}
brotli_enc_config c = make_brotli_enc_config(q, configs[i][0]);
if (configs[i][4]) {
@@ -4263,10 +4292,17 @@ do_test_brotli_encode_dictionary_big(bool mimic) {
wuffs_base__make_slice_u8(dict, dlen),
wuffs_base__make_slice_u8(data + (len - 1000), 1000),
};
static const uint32_t qs[] = {2, 4, 5, 9};
static const uint32_t qs[] = {2, 4, 5, 9, 10, 11};
for (size_t i = 0; (i < WUFFS_TESTLIB_ARRAY_SIZE(qs)) && !ret; i++) {
// Without mimic, and for the largest dictionary, a smaller grid.
if ((!mimic && ((k & 1) == 0)) || ((!mimic || (k == 3)) && (i != 2))) {
// Without mimic, and for the largest dictionary, a smaller grid. The
// slow qualities 10 and 11 only run with mimic, for one dictionary
// each.
if (qs[i] >= 10) {
if (!mimic || (k != ((qs[i] == 10) ? 3 : 1))) {
continue;
}
} else if ((!mimic && ((k & 1) == 0)) ||
((!mimic || (k == 3)) && (i != 2))) {
continue;
}
brotli_enc_config c = make_brotli_enc_config(qs[i], 24);
@@ -4279,13 +4315,17 @@ do_test_brotli_encode_dictionary_big(bool mimic) {
char s[256];
snprintf(s, sizeof s, "dictionary of %zu bytes: q=%u", dlen, qs[i]);
size_t have_len = 0;
ret = do_test_brotli_encode_dictionary(s, &c, data, len, 2, dicts,
mimic, &have_len);
// At qualities 10 and 11, only the first 128 KiB of the input (which
// has the first piece) is encoded.
ret = do_test_brotli_encode_dictionary(
s, &c, data, (qs[i] >= 10) ? (128 << 10) : len, 2, dicts, mimic,
&have_len);
// The three 64 KiB pieces (of 3.7 bits of entropy per byte) should be
// found at quality 5 and above, even when they are more than 16 MiB
// (lgwin 24's window) into the dictionary.
// (lgwin 24's window) into the dictionary. (At the slow qualities 10
// and 11, this is left to the parity check.)
size_t want_max = 0;
if (!ret && (qs[i] >= 5)) {
if (!ret && (qs[i] >= 5) && (qs[i] <= 9)) {
c = make_brotli_enc_config(qs[i], 24);
ret = do_test_brotli_encode_dictionary(s, &c, data, len, 0, NULL,
false, &want_max);
@@ -4359,47 +4399,51 @@ test_mimic_brotli_encode_dictionary_rejections() {
{1, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 0, NULL, NULL},
{3, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 0, NULL, NULL},
};
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(cases); i++) {
uint32_t dict_types[16];
wuffs_base__slice_u8 dicts[16];
for (size_t j = 0; j < cases[i].num_dicts; j++) {
dict_types[j] = WUFFS_BROTLI__DICTIONARY_TYPE_RAW;
dicts[j] = wuffs_base__make_slice_u8((uint8_t*)text + j, 20);
}
size_t last = cases[i].num_dicts - 1;
dict_types[last] = cases[i].type;
dicts[last] = wuffs_base__make_slice_u8(
(uint8_t*)((cases[i].type == WUFFS_BROTLI__DICTIONARY_TYPE_RAW)
? text
: serialized),
cases[i].len);
// Quality 5 and the Zopfli qualities 10 and 11.
static const uint32_t qs[] = {5, 10, 11};
for (size_t iq = 0; iq < WUFFS_TESTLIB_ARRAY_SIZE(qs); iq++) {
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(cases); i++) {
uint32_t dict_types[16];
wuffs_base__slice_u8 dicts[16];
for (size_t j = 0; j < cases[i].num_dicts; j++) {
dict_types[j] = WUFFS_BROTLI__DICTIONARY_TYPE_RAW;
dicts[j] = wuffs_base__make_slice_u8((uint8_t*)text + j, 20);
}
size_t last = cases[i].num_dicts - 1;
dict_types[last] = cases[i].type;
dicts[last] = wuffs_base__make_slice_u8(
(uint8_t*)((cases[i].type == WUFFS_BROTLI__DICTIONARY_TYPE_RAW)
? text
: serialized),
cases[i].len);
brotli_enc_config c = make_brotli_enc_config(5, 22);
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
CHECK_STRING(make_brotli_enc_plan(&src, &c, text, sizeof text - 1));
wuffs_base__io_buffer dsts[2] = {
((wuffs_base__io_buffer){.data = g_have_slice_u8}),
((wuffs_base__io_buffer){.data = g_want_slice_u8}),
};
for (int k = 0; k < 2; k++) {
wuffs_base__io_buffer src1 = src;
const char* have =
(k ? mimic_brotli_encode_with_dictionaries
: wuffs_brotli_encode_plan_with_dictionaries)(
&dsts[k], &src1, c.param_keys, c.param_values, c.num_params,
cases[i].num_dicts, dict_types, dicts, g_brotli_enc_step_ops,
g_brotli_enc_step_lens, g_brotli_enc_num_steps, c.dst_chunk);
const char* want = k ? cases[i].want_mimic : cases[i].want_wuffs;
if (have != want) {
RETURN_FAIL("cases[%zu] (%s): have \"%s\", want \"%s\"", i,
k ? "mimic" : "wuffs", have ? have : "",
want ? want : "");
brotli_enc_config c = make_brotli_enc_config(qs[iq], 22);
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
CHECK_STRING(make_brotli_enc_plan(&src, &c, text, sizeof text - 1));
wuffs_base__io_buffer dsts[2] = {
((wuffs_base__io_buffer){.data = g_have_slice_u8}),
((wuffs_base__io_buffer){.data = g_want_slice_u8}),
};
for (int k = 0; k < 2; k++) {
wuffs_base__io_buffer src1 = src;
const char* have =
(k ? mimic_brotli_encode_with_dictionaries
: wuffs_brotli_encode_plan_with_dictionaries)(
&dsts[k], &src1, c.param_keys, c.param_values, c.num_params,
cases[i].num_dicts, dict_types, dicts, g_brotli_enc_step_ops,
g_brotli_enc_step_lens, g_brotli_enc_num_steps, c.dst_chunk);
const char* want = k ? cases[i].want_mimic : cases[i].want_wuffs;
if (have != want) {
RETURN_FAIL("q=%u cases[%zu] (%s): have \"%s\", want \"%s\"",
qs[iq], i, k ? "mimic" : "wuffs", have ? have : "",
want ? want : "");
}
}
if (!cases[i].want_wuffs) {
CHECK_STRING(check_io_buffers_equal("", &dsts[0], &dsts[1]));
}
}
if (!cases[i].want_wuffs) {
CHECK_STRING(check_io_buffers_equal("", &dsts[0], &dsts[1]));
}
}
return NULL;