diff --git a/doc/std/compression-encoders.md b/doc/std/compression-encoders.md index d1a1fa66..e181adbf 100644 --- a/doc/std/compression-encoders.md +++ b/doc/std/compression-encoders.md @@ -26,8 +26,10 @@ In Wuffs syntax, the `base.io_transformer` methods are: - `workbuf_len() range_ii_u64` Encoders may have further, format-specific methods. For example, the brotli -encoder has `request_flush`, `request_emit_metadata` and -`max_compressed_size`. +encoder has `request_flush`, `request_emit_metadata`, `max_compressed_size` +and, for raw (LZ77 prefix) dictionaries, `add_dictionary` and +`add_dictionary_workbuf_len`. Like quirks, dictionaries are added before the +first `transform_io` call. ## Implementations @@ -36,4 +38,8 @@ encoder has `request_flush`, `request_emit_metadata` and including large-window brotli, with windows of up to 1 GiB, at qualities 3 to 9). Its `workbuf_len` is the worst case for the parameters (more than 2 GiB for large-window brotli at lgwin 30), but a shorter workbuf works for - short inputs: see the README's "Encoder Workbuf Layout" section. + short inputs: see the README's "Encoder Workbuf Layout" section. Up to 15 + raw dictionaries can be attached (like libbrotlienc's + `BrotliEncoderAttachPreparedDictionary`), see the README's "Encoder + Dictionaries" section. They live at the start of the workbuf, with their + prepared hash tables, and `workbuf_len` grows accordingly. diff --git a/script/manual-test-brotli-encoder-parity.c b/script/manual-test-brotli-encoder-parity.c index 7c07a315..6967ea8c 100644 --- a/script/manual-test-brotli-encoder-parity.c +++ b/script/manual-test-brotli-encoder-parity.c @@ -1257,7 +1257,7 @@ run_dict_configs(const char* name, const uint8_t* data, size_t len, uint32_t q, // check_smaller checks that, with the current dictionaries (which should // hold much of data), libbrotlienc's output (which Wuffs' output matches) is -// smaller than without them. +// smaller than without them (unless that is already tiny). static void // check_smaller(const char* name, const uint8_t* data, size_t len, uint32_t q) { config c = make_config(q, 22); @@ -1272,7 +1272,9 @@ check_smaller(const char* name, const uint8_t* data, size_t len, uint32_t q) { } g_num_dicts = saved; g_num_smaller_checks++; - if (err || (with_len >= without_len)) { + // Tiny outputs (e.g. for long runs of zeroes) cannot shrink. + if (err || (with_len > without_len) || + ((without_len >= 64) && (with_len >= without_len))) { g_num_mismatches++; printf("NOT SMALLER %s: q=%u dicts=%s: %zu bytes with, %zu without: %s\n", name, q, g_dict_desc, with_len, without_len, err ? err : ""); diff --git a/std/brotli/README.md b/std/brotli/README.md index 58f411a1..b22841e1 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -154,8 +154,7 @@ invariants for memory safety but Wuffs requires explicit bounds checks. - Metadata callbacks (metadata blocks are currently skipped). See the "Metadata Blocks" section below. - Fast paths. -- Encoder quality levels 10 and 11 and encoder dictionaries (see the - "Encoder" section below). +- Encoder quality levels 10 and 11 (see the "Encoder" section below). ## Encoder @@ -182,7 +181,8 @@ literal context modeling heuristics, `BrotliBuildMetaBlockGreedy`, `"#unsupported encoder quality"` (and `workbuf_len` is 0). Large-window brotli (see "Encoder Large Window" below) works at qualities 3 to 9 with lgwin 10 to 30. Like the C code, qualities 0 to 2 ignore the large-window -quirk. +quirk. Raw (LZ77 prefix) dictionaries (see "Encoder Dictionaries" below) work +at every quality, and, like the C code, qualities 0 and 1 ignore them. The source files are: @@ -217,6 +217,13 @@ The source files are: `hash_composite_inc.h` (H35, H55 and H65, which combine H3, H54 or H6 with a rolling hasher) and `hash_rolling_inc.h` (`HROLLING_FAST` and `HROLLING`). +- `encode_dictionary.wuffs`: `compound_dictionary.c` + (`CreatePreparedDictionary`, `AttachPreparedDictionary`), the raw + dictionary cases of `BrotliEncoderPrepareDictionary` and + `BrotliEncoderAttachPreparedDictionary` (as `add_dictionary`), the + compound dictionary search of `hash.h` (`FindCompoundDictionaryMatch`, + `LookupCompoundDictionaryMatch`) and the compound dictionary branch of + `ExtendLastCommand`. - `encode_backward_references.wuffs`: `backward_references.c` and `backward_references_inc.h` (`BrotliCreateBackwardReferences`), `command.h` and `ComputeDistanceCode`. @@ -266,6 +273,10 @@ In addition to the `base.io_transformer` methods, the encoder has: another request is pending. - `max_compressed_size(input_size: u64) u64`: mirrors `BrotliEncoderMaxCompressedSize` (0 means overflow). +- `add_dictionary!(dict_type: u32, dict: roslice u8, workbuf: slice u8) + status` and `add_dictionary_workbuf_len(dict_len: u64) u64`: attach a raw + dictionary, like `BrotliEncoderPrepareDictionary` plus + `BrotliEncoderAttachPreparedDictionary`. See "Encoder Dictionaries" below. `dst_history_retain_length` is 0: the encoder never re-reads `dst`. @@ -277,7 +288,8 @@ Each quirk mirrors one `BrotliEncoderParameter`, with the value semantics of otherwise `set_quirk` returns `"#bad argument"`). Like the C code, they are sanitized (`SanitizeParams`, `ComputeLgBlock`, `ChooseDistanceParams`) at the first `transform_io` call, after which `set_quirk` returns -`"#bad call sequence"`. +`"#bad call sequence"`. Quirks and dictionaries (`add_dictionary`) can be set +in any order before that first call. | Quirk | C parameter | | ---------------------------------------- | ---------------------------------------------- | @@ -309,6 +321,12 @@ size hint is replaced by the C code's `UpdateSizeHint` estimate. ### Encoder Workbuf Layout +Any attached dictionaries come first (see "Encoder Dictionaries" below): the +workbuf's first `D` bytes hold them (with their prepared hash tables) and +everything described in this section starts at offset `D` (so all of the +offsets and lengths below are relative to `workbuf[D ..]`, and `workbuf_len` +is `D` more than the lengths given here). `D` is 0 without dictionaries. + At qualities 0 and 1, the workbuf is the C code's metablock storage (its `GetBrotliStorage` buffer): `workbuf_len` is `(2 << lgwin) + 503 + 16` bytes (the storage for a `1 << lgwin` byte block plus 16 bytes of slack for 8-byte @@ -511,6 +529,135 @@ truncated for windows up to `1 << 30`. The rolling hashers' table holds 32-bit positions, like the C code's. +### Encoder Dictionaries + +`encoder.add_dictionary!(dict_type: base.u32, dict: roslice base.u8, workbuf: +slice base.u8) base.status` mirrors the C API's +`BrotliEncoderPrepareDictionary(BROTLI_SHARED_DICTIONARY_RAW, ...)` +followed by `BrotliEncoderAttachPreparedDictionary`: it attaches a raw (LZ77 +prefix) dictionary, as the next chunk of the C code's compound dictionary. +The decoder must be given the same dictionaries, in the same order (with +this package's `decoder.add_dictionary` or libbrotlidec's +`BrotliDecoderAttachDictionary`, as `DICTIONARY_TYPE_RAW` / +`BROTLI_SHARED_DICTIONARY_RAW`). + +Ordering: like quirks, dictionaries must be added before the first +`transform_io` call (the C API's "before the first +`BrotliEncoderCompressStream`"). Quirks and dictionaries can be set in any +order (preparing a raw dictionary does not depend on any parameter: the +`quality` argument of `BrotliEncoderPrepareDictionary` is unused for raw +dictionaries). Call `workbuf_len` after the last `add_dictionary` and +`set_quirk` call. + +Like the decoder, the encoder copies each dictionary into the workbuf, so +every `add_dictionary` and `transform_io` call must be passed the same +workbuf (or at least one whose first bytes hold the same contents). Each +dictionary is followed by its prepared hash tables (the C code's +`PreparedDictionary`), which `add_dictionary` builds exactly like +`CreatePreparedDictionary` and `CreatePreparedDictionaryWithParams` in +`compound_dictionary.c`, so that the compound dictionary search finds the +same candidates in the same order: + +- The hash table size depends only on the dictionary length `n`: 17 bucket + bits and 7 slot bits for `n` up to 2 MiB, one more of each for each + doubling of `n` (18 and 8 up to 4 MiB, etc), up to 22 and 12 (for more than + 32 MiB). Each position `i` with `i + 7 < n` is hashed (the low 40 bits of + its 8 little-endian bytes, times `kPreparedDictionaryHashMul64Long`), with + at most 32 positions per bucket (the most recent ones), and the bucket + limits are lowered per slot so that each slot has fewer than 65535 items. +- Its layout, after the `n` dictionary bytes, is `slot_offsets` (`1 << + slot_bits` u32le values), `heads` (`1 << bucket_bits` u16le values, 0xFFFF + for an empty bucket) and `items` (`num_items` u32le values, each a + dictionary position, with the high bit marking the end of a bucket's + chain), like the C struct's dynamic members. + +So the workbuf layout is: + + [0 .. D) the dictionaries, each one being its n bytes, then + (4 << slot_bits) + (2 << bucket_bits) + (4 * num_items) + bytes of hash tables. num_items is at most max(0, n - 7). + [D .. ) the encoder's other regions (see "Encoder Workbuf Layout"). + +Preparing a dictionary needs some scratch space (the C code's temporary +"bloated" hasher: `(6 << bucket_bits) + (4 * n)` bytes), which `add_dictionary` +puts after the largest possible hash tables. So `add_dictionary`'s workbuf +must be at least `add_dictionary_workbuf_len(n)` bytes long: `D` (for the +dictionaries added so far) plus `n + (4 << slot_bits) + (2 << bucket_bits) + +(4 * max(0, n - 7))` plus that scratch space. For example, for a 1 MiB +dictionary that is `D` plus about 9.8 MiB, and for a 40 MiB dictionary `D` +plus about 384 MiB. After the call, `D` grows by the dictionary's bytes and +actual hash tables (e.g. by at most 5.3 MiB for 1 MiB of dictionary). + +The compound dictionary then affects compression like in the C code: + +- Qualities 0 and 1 (`BrotliEncoderCompressStreamFast`) ignore it. +- At qualities 2 to 9, its total size (the C code's `gap`) is added to the + static dictionary's base distance (`dictionary_start + gap` in + `CreateBackwardReferences`), and distances up to `dictionary_start + gap` + (which include compound dictionary references) update the distance cache. + `ExtendLastCommand` can extend a copy from the compound dictionary. +- Only the hashers that `BrotliCreateBackwardReferences` instantiates with + `ENABLE_COMPOUND_DICTIONARY` search the dictionaries (with + `LookupCompoundDictionaryMatch`, after the hasher's own search, for each + chunk in order): H5, H6, H58, H68, H40, H41, H42, H55 and H65. So + qualities 5 to 9 search them, quality 4 only with the large-window H55 + (lgwin above 24 and a big size hint), and qualities 2 and 3 (H2, H3, H35) + and quality 4 with H4 or H54 never do (the C code falls back to its + non-dictionary variant for those hashers). The search only finds matches + that are within `params.dist.max_distance` (`BROTLI_MAX_DISTANCE`, or + `BROTLI_MAX_ALLOWED_DISTANCE` for large-window brotli) and that do not + cross a chunk's end. +- With `QUIRK_STREAM_OFFSET`, `dictionary_start` is `min(position + + stream_offset, max_backward_limit)`, as in the C code (whose output then + continues another stream and can only be decoded after it). + +`add_dictionary` statuses (the C code's functions return `NULL` or +`BROTLI_FALSE`): + +| Condition | Wuffs status | C code | +| ------------------------------------------------------------ | ---------------------------------------------- | ------ | +| `transform_io` has been called | `base."#bad call sequence"` | accepted (see below) | +| `DICTIONARY_TYPE_SERIALIZED` | `"#unsupported serialized encoder dictionary"` | `BrotliEncoderPrepareDictionary` returns `NULL` | +| unknown `dict_type` | `base."#bad argument"` | `BrotliEncoderPrepareDictionary` returns `NULL` | +| a 16th dictionary | `"#too many compound dictionaries"` | `BrotliEncoderAttachPreparedDictionary` fails | +| more than `0x7FFF_FFFF` bytes of dictionaries in total | `"#too much dictionary data"` | accepted | +| `workbuf` shorter than `add_dictionary_workbuf_len(n)` | `base."#bad workbuf length"` | n/a | + +An empty dictionary is accepted (and counts towards the 15), like in the C +code. On failure, the encoder's state is unchanged. A `transform_io` call +with a workbuf shorter than `D` returns `base."#bad workbuf length"`. + +Differences from the C code: + +- Serialized (shared) dictionaries are rejected, exactly like the default + (non-`BROTLI_EXPERIMENTAL`) libbrotlienc build, whose + `BrotliEncoderPrepareDictionary` returns `NULL` for them. (The decoder, in + contrast, accepts them, like an experimental libbrotlidec build.) With an + experimental build, the C encoder would also use their custom static + dictionaries; that is not implemented. +- The C code's `BrotliEncoderAttachPreparedDictionary` accepts a raw + dictionary even after compression has started (it only checks + `is_initialized_` for serialized dictionaries). The compound dictionary's + size then changes mid-stream, so the static dictionary references before + and after that point use different distance bases, and the output is + generally not decodable. Here, `add_dictionary` returns `"#bad call + sequence"` once `transform_io` has been called. +- The C code holds each dictionary's length in a `uint32_t` (and item + offsets in 31 bits). This package rejects more than `0x7FFF_FFFF` bytes + of dictionaries in total (like its decoder). +- When `ExtendLastCommand` extends a copy past the end of a compound + dictionary chunk into an empty chunk, the C code reads (one or more + bytes) past the end of that empty chunk's (zero-length) data. Here, the + extension stops there. This needs an empty dictionary attached after a + non-empty one and a copy extension that reaches the non-empty one's end + (with empty dictionaries only before the non-empty ones, the behaviors are + the same). +- Like the C code's `BrotliEncoderPrepareDictionary` with the default + allocator, there is no separate "prepared dictionary" object that could be + shared between encoders: each encoder prepares its own copy (the C code's + "lean" prepared dictionaries reference the caller's bytes instead). + + ### Encoder Testing `wuffs test -mimic std/brotli` checks byte-exact parity with libbrotlienc @@ -529,7 +676,19 @@ processes, `-lw` runs only its large-window grid (lgwin 10 to 30 at qualities 2 to 9, with various chunkings, flushes, size hints and other parameters, each also with the shortest workbuf that should work) and `-big` replaces the inputs with synthetic ones of 3 to 70 MiB with repeats at -distances of up to 40 MiB. With `-q=0-9` (the default) it runs 814,706 +distances of up to 40 MiB. `-dict` replaces the usual grid with a +dictionary grid (see "Encoder Dictionaries" above): for each input, seven +sets of raw dictionaries (a prefix of the input, the whole input, +`test/data/enwik5`, noise plus the input's middle third, 15 pieces of the +input, tiny and empty dictionaries, and the input's swapped halves plus some +text), with qualities 0 to 9, lgwins, chunkings, flushes, metadata blocks, +size hints, stream offsets, large window and other parameters, plus +dictionaries of 2 MiB to 40 MiB (every hash table size) with a 1 MiB input. +Each dictionary configuration's output is also decoded by libbrotlidec and +by this package's decoder (with the same dictionaries), except with a +stream offset, and the whole-input and big dictionaries check that the +output is smaller than without them (at qualities 5 and above). With +`-q=0-9` (the default) and without `-dict` it runs 814,706 configurations (96,688 of them in the large-window grid), all byte-exact against libbrotlienc v1.2.1 on x86-64. With `-lw -big` it runs 720 more (on 3, 6 and 12 MiB inputs with the large-window grid and on 24, 40 and @@ -706,10 +865,11 @@ metablock) for as long as it needs the results. - Qualities 10 and 11 (Zopfli, the H10 binary tree hasher and `BrotliBuildMetaBlock`, which is already ported in `encode_metablock.wuffs`). -- Large-window encoding at qualities 3 and above (it works at quality 2). - `workbuf_len` is a worst case. A future option could take the input length (like the C code's size hint) to give a tighter bound. -- Custom and shared dictionaries (`BrotliEncoderAttachPreparedDictionary`). +- Compound dictionary matches at qualities 10 and 11 (they need Zopfli). +- Serialized (shared) dictionaries, which only an experimental + (`BROTLI_EXPERIMENTAL`) libbrotlienc build supports. - Changing parameters mid-stream. diff --git a/test/c/mimiclib/brotli.c b/test/c/mimiclib/brotli.c index 7e3cb65d..2b2d8cd6 100644 --- a/test/c/mimiclib/brotli.c +++ b/test/c/mimiclib/brotli.c @@ -21,6 +21,7 @@ #include "brotli/decode.h" #include "brotli/encode.h" +#include "brotli/shared_dictionary.h" // mimic_brotli_decode_error_code is like mimic_brotli_decode but also stores // the libbrotlidec error code (a BrotliDecoderErrorCode value) in @@ -91,6 +92,20 @@ mimic_brotli_decode(wuffs_base__io_buffer* dst, // // Each BrotliEncoderCompressStream call gets at most dst_chunk bytes of dst // (or all of dst's writable bytes, if dst_chunk is zero). +const char* // +mimic_brotli_encode_with_dictionaries(wuffs_base__io_buffer* dst, + wuffs_base__io_buffer* src, + const uint32_t* param_keys, + const uint32_t* param_values, + size_t num_params, + size_t num_dicts, + const uint32_t* dict_types, + const wuffs_base__slice_u8* dicts, + const uint32_t* step_ops, + const size_t* step_lens, + size_t num_steps, + size_t dst_chunk); + const char* // mimic_brotli_encode(wuffs_base__io_buffer* dst, wuffs_base__io_buffer* src, @@ -101,15 +116,62 @@ mimic_brotli_encode(wuffs_base__io_buffer* dst, const size_t* step_lens, size_t num_steps, size_t dst_chunk) { + return mimic_brotli_encode_with_dictionaries( + dst, src, param_keys, param_values, num_params, 0, NULL, NULL, step_ops, + step_lens, num_steps, dst_chunk); +} + +// mimic_brotli_encode_with_dictionaries is like mimic_brotli_encode but it +// also prepares (with BrotliEncoderPrepareDictionary) and attaches (with +// BrotliEncoderAttachPreparedDictionary), after setting the parameters and +// before the first step, num_dicts dictionaries. Each dict_types[i] is a +// BrotliSharedDictionaryType value: 0 for BROTLI_SHARED_DICTIONARY_RAW and 1 +// for BROTLI_SHARED_DICTIONARY_SERIALIZED. The quality passed to +// BrotliEncoderPrepareDictionary is the last BROTLI_PARAM_QUALITY param (or +// the default, 11), although the C code ignores it for raw dictionaries. +// +// libbrotlienc only supports serialized dictionaries when built with +// BROTLI_EXPERIMENTAL (which is not the default). When preparing or +// attaching a dictionary fails, this function returns +// mimic_brotli_prepare_dictionary_failed_message or +// mimic_brotli_attach_prepared_dictionary_failed_message. +const char mimic_brotli_prepare_dictionary_failed_message[] = + "libbrotlienc: BrotliEncoderPrepareDictionary failed"; +const char mimic_brotli_attach_prepared_dictionary_failed_message[] = + "libbrotlienc: BrotliEncoderAttachPreparedDictionary failed"; + +#define MIMIC_BROTLI_ENCODE_MAX_DICTS 64 + +const char* // +mimic_brotli_encode_with_dictionaries(wuffs_base__io_buffer* dst, + wuffs_base__io_buffer* src, + const uint32_t* param_keys, + const uint32_t* param_values, + size_t num_params, + size_t num_dicts, + const uint32_t* dict_types, + const wuffs_base__slice_u8* dicts, + const uint32_t* step_ops, + const size_t* step_lens, + size_t num_steps, + size_t dst_chunk) { if (BrotliEncoderVersion() != 0x1002001) { return "libbrotlienc: unexpected version (want v1.2.1)"; + } else if (num_dicts > MIMIC_BROTLI_ENCODE_MAX_DICTS) { + return "mimic_brotli_encode: too many dictionaries"; } + BrotliEncoderPreparedDictionary* prepared[MIMIC_BROTLI_ENCODE_MAX_DICTS]; + size_t num_prepared = 0; BrotliEncoderState* state = BrotliEncoderCreateInstance(NULL, NULL, NULL); if (!state) { return "libbrotlienc: BrotliEncoderCreateInstance failed"; } const char* ret = NULL; + int quality = 11; for (size_t j = 0; j < num_params; j++) { + if (param_keys[j] == BROTLI_PARAM_QUALITY) { + quality = (int)(param_values[j]); + } if (!BrotliEncoderSetParameter(state, (BrotliEncoderParameter)(param_keys[j]), param_values[j])) { @@ -117,6 +179,20 @@ mimic_brotli_encode(wuffs_base__io_buffer* dst, goto cleanup; } } + for (size_t j = 0; j < num_dicts; j++) { + prepared[j] = BrotliEncoderPrepareDictionary( + (BrotliSharedDictionaryType)(dict_types[j]), dicts[j].len, + dicts[j].ptr, quality, NULL, NULL, NULL); + if (!prepared[j]) { + ret = mimic_brotli_prepare_dictionary_failed_message; + goto cleanup; + } + num_prepared++; + if (!BrotliEncoderAttachPreparedDictionary(state, prepared[j])) { + ret = mimic_brotli_attach_prepared_dictionary_failed_message; + goto cleanup; + } + } for (size_t i = 0; i < num_steps; i++) { size_t avail_in = step_lens[i]; @@ -156,13 +232,14 @@ mimic_brotli_encode(wuffs_base__io_buffer* dst, cleanup: BrotliEncoderDestroyInstance(state); + for (size_t j = 0; j < num_prepared; j++) { + BrotliEncoderDestroyPreparedDictionary(prepared[j]); + } return ret; } // ---------------- Large Window and Dictionaries -#include "brotli/shared_dictionary.h" - // mimic_brotli_decode_with_options is like mimic_brotli_decode_error_code but // it also sets BROTLI_DECODER_PARAM_LARGE_WINDOW (if large_window is true) and // attaches dictionaries (with BrotliDecoderAttachDictionary). Each diff --git a/test/c/std/brotli.c b/test/c/std/brotli.c index 8a1d2f4e..c8c2adba 100644 --- a/test/c/std/brotli.c +++ b/test/c/std/brotli.c @@ -854,6 +854,20 @@ brotli_enc_quirk_key(uint32_t param) { return 0; } +const char* // +wuffs_brotli_encode_plan_with_dictionaries(wuffs_base__io_buffer* dst, + wuffs_base__io_buffer* src, + const uint32_t* param_keys, + const uint32_t* param_values, + size_t num_params, + size_t num_dicts, + const uint32_t* dict_types, + const wuffs_base__slice_u8* dicts, + const uint32_t* step_ops, + const size_t* step_lens, + size_t num_steps, + size_t dst_chunk); + // wuffs_brotli_encode_plan is the Wuffs equivalent of mimic_brotli_encode // (in test/c/mimiclib/brotli.c). const char* // @@ -866,6 +880,29 @@ wuffs_brotli_encode_plan(wuffs_base__io_buffer* dst, const size_t* step_lens, size_t num_steps, size_t dst_chunk) { + return wuffs_brotli_encode_plan_with_dictionaries( + dst, src, param_keys, param_values, num_params, 0, NULL, NULL, step_ops, + step_lens, num_steps, dst_chunk); +} + +// wuffs_brotli_encode_plan_with_dictionaries is the Wuffs equivalent of +// mimic_brotli_encode_with_dictionaries: it adds (with add_dictionary) the +// num_dicts dictionaries after setting the quirks and before the first step. +// Each dict_types[i] is WUFFS_BROTLI__DICTIONARY_TYPE_RAW or +// WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED. +const char* // +wuffs_brotli_encode_plan_with_dictionaries(wuffs_base__io_buffer* dst, + wuffs_base__io_buffer* src, + const uint32_t* param_keys, + const uint32_t* param_values, + size_t num_params, + size_t num_dicts, + const uint32_t* dict_types, + const wuffs_base__slice_u8* dicts, + const uint32_t* step_ops, + const size_t* step_lens, + size_t num_steps, + size_t dst_chunk) { // The encoder struct is large (more than 1 MiB), mostly arrays that do not // need zero-initializing, so don't use wuffs_brotli__encoder__alloc. wuffs_brotli__encoder* enc = malloc(sizeof__wuffs_brotli__encoder()); @@ -891,23 +928,45 @@ wuffs_brotli_encode_plan(wuffs_base__io_buffer* dst, goto cleanup; } } - wuffs_base__range_ii_u64 workbuf_len = - wuffs_brotli__encoder__workbuf_len(enc); wuffs_base__slice_u8 workbuf = g_work_slice_u8; - // Large-window brotli at lgwin 30 needs more than 2 GiB. - if ((workbuf_len.max_incl > (3ull << 30)) || - (workbuf_len.max_incl > SIZE_MAX)) { - ret = "workbuf_len is too large"; - goto cleanup; - } else if (workbuf_len.max_incl > g_work_slice_u8.len) { - big_workbuf = malloc((size_t)workbuf_len.max_incl); - if (!big_workbuf) { - ret = "malloc failed"; + workbuf.len = 0; + size_t big_workbuf_len = 0; + for (size_t j = 0; j <= num_dicts; j++) { + // Each add_dictionary call needs add_dictionary_workbuf_len bytes and + // transform_io needs workbuf_len bytes, with the same dictionary bytes + // at the start. + uint64_t n = (j < num_dicts) + ? wuffs_brotli__encoder__add_dictionary_workbuf_len( + enc, dicts[j].len) + : wuffs_brotli__encoder__workbuf_len(enc).max_incl; + // Large-window brotli at lgwin 30 needs more than 2 GiB. + if ((n > (3ull << 30)) || (n > SIZE_MAX)) { + ret = "workbuf_len is too large"; goto cleanup; + } else if ((n > g_work_slice_u8.len) && (n > big_workbuf_len)) { + uint8_t* p = realloc(big_workbuf, (size_t)n); + if (!p) { + ret = "malloc failed"; + goto cleanup; + } + if (!big_workbuf && workbuf.len) { + // Keep the dictionaries added so far. + memcpy(p, g_work_slice_u8.ptr, workbuf.len); + } + big_workbuf = p; + big_workbuf_len = (size_t)n; + workbuf.ptr = big_workbuf; + } + workbuf.len = (size_t)n; + if (j < num_dicts) { + wuffs_base__status status = wuffs_brotli__encoder__add_dictionary( + enc, dict_types[j], dicts[j], workbuf); + if (status.repr) { + ret = status.repr; + goto cleanup; + } } - workbuf.ptr = big_workbuf; } - workbuf.len = (size_t)workbuf_len.max_incl; for (size_t i = 0; i < num_steps; i++) { size_t end = src->meta.ri + step_lens[i]; @@ -3456,6 +3515,610 @@ test_mimic_brotli_encode_large_window_far() { #endif // WUFFS_MIMIC +// ---------------- Brotli Encoder Dictionary Tests + +// brotli_enc_load_file returns a malloc'ed copy of the named file (of up to +// 1 MiB) and sets *len to its length. +uint8_t* // +brotli_enc_load_file(const char* filename, size_t* len) { + uint8_t* buf = malloc(1 << 20); + if (!buf) { + return NULL; + } + wuffs_base__io_buffer b = ((wuffs_base__io_buffer){ + .data = wuffs_base__make_slice_u8(buf, 1 << 20), + }); + if (read_file(&b, filename)) { + free(buf); + return NULL; + } + *len = b.meta.wi; + return buf; +} + +// brotli_enc_config_has_param returns whether c sets the key param to a +// non-zero value. +bool // +brotli_enc_config_has_param(const brotli_enc_config* c, uint32_t key) { + bool ret = false; + for (size_t j = 0; j < c->num_params; j++) { + if (c->param_keys[j] == key) { + ret = c->param_values[j] != 0; + } + } + return ret; +} + +// wuffs_brotli_decode_with_raw_dictionaries decodes src to dst with Wuffs' +// std/brotli decoder, with the given raw dictionaries attached. +const char* // +wuffs_brotli_decode_with_raw_dictionaries(wuffs_base__io_buffer* dst, + wuffs_base__io_buffer* src, + bool large_window, + size_t num_dicts, + const wuffs_base__slice_u8* dicts) { + wuffs_brotli__decoder dec; + CHECK_STATUS("initialize", + wuffs_brotli__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED)); + CHECK_STATUS("set_quirk", + wuffs_brotli__decoder__set_quirk( + &dec, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, large_window)); + size_t total = 0; + for (size_t j = 0; j < num_dicts; j++) { + total += dicts[j].len; + } + size_t n = total + (large_window ? (1 << 30) : (1 << 24)) + 542; + uint8_t* p = malloc(n); + if (!p) { + return "malloc failed"; + } + wuffs_base__slice_u8 workbuf = wuffs_base__make_slice_u8(p, n); + const char* ret = NULL; + for (size_t j = 0; j < num_dicts; j++) { + ret = wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, dicts[j], workbuf) + .repr; + if (ret) { + goto cleanup; + } + } + if (wuffs_brotli__decoder__workbuf_len(&dec).max_incl > n) { + ret = "decoder workbuf_len is too large"; + goto cleanup; + } + ret = wuffs_brotli__decoder__transform_io(&dec, dst, src, workbuf).repr; +cleanup: + free(p); + return ret; +} + +// do_test_brotli_encode_dictionary encodes data[:data_len] with the plan's +// config c and the given raw dictionaries. Unless the config sets a stream +// offset (whose output is the continuation of another stream), it checks +// that Wuffs' decoder (and, if mimic is true, libbrotlidec), with the same +// dictionaries, decodes the result back to data. If mimic is true, it also +// checks that libbrotlienc's output is the same. It sets *compressed_len. +const char* // +do_test_brotli_encode_dictionary(const char* name, + const brotli_enc_config* c, + const uint8_t* data, + size_t data_len, + size_t num_dicts, + const wuffs_base__slice_u8* dicts, + bool mimic, + size_t* compressed_len) { + uint32_t dict_types[16] = {0}; + for (size_t j = 0; j < 16; j++) { + dict_types[j] = WUFFS_BROTLI__DICTIONARY_TYPE_RAW; + } + if (num_dicts > 16) { + return "too many dictionaries"; + } + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = g_src_slice_u8, + }); + CHECK_STRING(make_brotli_enc_plan(&src, c, data, data_len)); + wuffs_base__io_buffer have = ((wuffs_base__io_buffer){ + .data = g_have_slice_u8, + }); + wuffs_base__io_buffer src0 = src; + const char* status = wuffs_brotli_encode_plan_with_dictionaries( + &have, &src0, c->param_keys, c->param_values, c->num_params, num_dicts, + dict_types, dicts, g_brotli_enc_step_ops, g_brotli_enc_step_lens, + g_brotli_enc_num_steps, c->dst_chunk); + if (status) { + RETURN_FAIL("%s: wuffs encode: %s", name, status); + } + have.meta.closed = true; + *compressed_len = have.meta.wi; + bool lw = brotli_enc_config_has_param(c, BROTLI_ENC_PARAM_LARGE_WINDOW); + bool so = brotli_enc_config_has_param(c, BROTLI_ENC_PARAM_STREAM_OFFSET); + + for (int k = 0; k < (mimic ? 3 : 1); k++) { + wuffs_base__io_buffer want = ((wuffs_base__io_buffer){ + .data = g_want_slice_u8, + }); + wuffs_base__io_buffer have1 = have; + if (k == 0) { + if (so) { + continue; + } + status = wuffs_brotli_decode_with_raw_dictionaries(&want, &have1, lw, + num_dicts, dicts); +#ifdef WUFFS_MIMIC + } else if (k == 1) { + if (so) { + continue; + } + int error_code = 0; + status = mimic_brotli_decode_with_options( + &want, &have1, lw, num_dicts, dict_types, dicts, &error_code); + } else { + wuffs_base__io_buffer src1 = src; + status = mimic_brotli_encode_with_dictionaries( + &want, &src1, c->param_keys, c->param_values, c->num_params, + num_dicts, dict_types, dicts, g_brotli_enc_step_ops, + g_brotli_enc_step_lens, g_brotli_enc_num_steps, c->dst_chunk); + if (status) { + RETURN_FAIL("%s: mimic encode: %s", name, status); + } + status = check_io_buffers_equal("", &have, &want); + if (status) { + RETURN_FAIL("%s: %s", name, status); + } + continue; +#endif // WUFFS_MIMIC + } + const char* which = (k == 0) ? "wuffs" : "mimic"; + if (status) { + RETURN_FAIL("%s: %s decode: %s", name, which, status); + } else if (have1.meta.ri != have1.meta.wi) { + RETURN_FAIL("%s: %s decode: trailing data", name, which); + } else if (want.meta.wi != data_len) { + RETURN_FAIL("%s: %s decoded length: have %zu, want %zu", name, which, + want.meta.wi, data_len); + } else if (memcmp(want.data.ptr, data, data_len)) { + RETURN_FAIL("%s: %s decoded bytes differ", name, which); + } + } + return NULL; +} + +const char* // +test_wuffs_brotli_encode_add_dictionary() { + CHECK_FOCUS(__func__); + static const uint8_t text[] = "the quick brown fox jumps over the lazy dog"; + wuffs_base__slice_u8 dict = + wuffs_base__make_slice_u8((uint8_t*)text, sizeof text - 1); + wuffs_base__slice_u8 workbuf = g_work_slice_u8; + + wuffs_brotli__encoder enc; + CHECK_STATUS("initialize", + wuffs_brotli__encoder__initialize( + &enc, sizeof enc, WUFFS_VERSION, + WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED)); + CHECK_STATUS("set_quirk", wuffs_brotli__encoder__set_quirk( + &enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, 5)); + uint64_t wlen0 = wuffs_brotli__encoder__workbuf_len(&enc).max_incl; + + // Serialized dictionaries are rejected (like libbrotlienc's default, + // non-experimental build) and so are unknown dictionary types. + const char* have = wuffs_brotli__encoder__add_dictionary( + &enc, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, dict, + workbuf) + .repr; + if (have != wuffs_brotli__error__unsupported_serialized_encoder_dictionary) { + RETURN_FAIL("serialized: have \"%s\"", have); + } + have = wuffs_brotli__encoder__add_dictionary(&enc, 2, dict, workbuf).repr; + if (have != wuffs_base__error__bad_argument) { + RETURN_FAIL("unknown type: have \"%s\"", have); + } + + // The add_dictionary_workbuf_len formula: for a dictionary of n bytes (and + // 17 bucket bits and 7 slot bits, for n up to 2 MiB), n bytes, (4 << 7) + // slot offsets, (2 << 17) heads and up to 4 * (n - 7) items, then (6 << + // 17) + (4 * n) bytes of scratch space. + uint64_t want_wlen = + 43 + (4 << 7) + (2 << 17) + (4 * 36) + (6 << 17) + (4 * 43); + uint64_t have_wlen = + wuffs_brotli__encoder__add_dictionary_workbuf_len(&enc, dict.len); + if (have_wlen != want_wlen) { + RETURN_FAIL("add_dictionary_workbuf_len: have %" PRIu64 ", want %" PRIu64, + have_wlen, want_wlen); + } + workbuf.len = (size_t)(have_wlen - 1); + have = wuffs_brotli__encoder__add_dictionary( + &enc, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, dict, workbuf) + .repr; + if (have != wuffs_base__error__bad_workbuf_length) { + RETURN_FAIL("short workbuf: have \"%s\"", have); + } + if (wuffs_brotli__encoder__workbuf_len(&enc).max_incl != wlen0) { + RETURN_FAIL("workbuf_len changed after failed add_dictionary calls"); + } + + // 15 dictionaries (the C code's SHARED_BROTLI_MAX_COMPOUND_DICTS), one of + // them empty, can be added, but not 16. + uint64_t dict_end = 0; + for (int i = 0; i < 16; i++) { + wuffs_base__slice_u8 d = dict; + if (i == 3) { + d.len = 0; + } + workbuf.len = g_work_slice_u8.len; + uint64_t n = wuffs_brotli__encoder__add_dictionary_workbuf_len(&enc, d.len); + if (n != (dict_end + ((i == 3) ? ((4 << 7) + (8 << 17)) : want_wlen))) { + RETURN_FAIL("i=%d: add_dictionary_workbuf_len: have %" PRIu64, i, n); + } + have = wuffs_brotli__encoder__add_dictionary( + &enc, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, d, workbuf) + .repr; + if (i == 15) { + if (have != wuffs_brotli__error__too_many_compound_dictionaries) { + RETURN_FAIL("i=%d: have \"%s\"", i, have); + } + break; + } else if (have) { + RETURN_FAIL("i=%d: have \"%s\"", i, have); + } + // The dictionary bytes, (4 << 7) slot offsets, (2 << 17) heads and 4 + // bytes per item. Each of the (43 - 7) positions of "the quick brown fox + // etc" has a distinct hash, so there are 36 items. + dict_end += (i == 3) ? ((4 << 7) + (2 << 17)) + : (43 + (4 << 7) + (2 << 17) + (4 * 36)); + uint64_t wlen = wuffs_brotli__encoder__workbuf_len(&enc).max_incl; + if (wlen != (wlen0 + dict_end)) { + RETURN_FAIL("i=%d: workbuf_len: have %" PRIu64 ", want %" PRIu64, i, + wlen, wlen0 + dict_end); + } + } + + // Quirks can still be set after add_dictionary, until transform_io. + CHECK_STATUS("set_quirk", wuffs_brotli__encoder__set_quirk( + &enc, WUFFS_BROTLI__QUIRK_LGWIN, 16)); + uint64_t wlen = wuffs_brotli__encoder__workbuf_len(&enc).max_incl; + if (wlen <= dict_end) { + RETURN_FAIL("workbuf_len: have %" PRIu64, wlen); + } + + // A workbuf shorter than the dictionaries fails. + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = wuffs_base__make_slice_u8((uint8_t*)text, sizeof text - 1), + .meta = wuffs_base__make_io_buffer_meta(sizeof text - 1, 0, 0, true), + }); + wuffs_base__io_buffer dst = ((wuffs_base__io_buffer){ + .data = g_have_slice_u8, + }); + // (An error disables the encoder, so try this on a copy.) + wuffs_brotli__encoder* copy = malloc(sizeof enc); + if (!copy) { + return "malloc failed"; + } + memcpy(copy, &enc, sizeof enc); + workbuf.len = (size_t)(dict_end - 1); + have = wuffs_brotli__encoder__transform_io(copy, &dst, &src, workbuf).repr; + free(copy); + if (have != wuffs_base__error__bad_workbuf_length) { + RETURN_FAIL("transform_io (short workbuf): have \"%s\"", have); + } + workbuf.len = (size_t)wlen; + CHECK_STATUS("transform_io", + wuffs_brotli__encoder__transform_io(&enc, &dst, &src, workbuf)); + if (dst.meta.wi >= (sizeof text - 1)) { + RETURN_FAIL("compressed length: have %zu", dst.meta.wi); + } + + // After transform_io, add_dictionary (and set_quirk) fail. (The C code's + // BrotliEncoderAttachPreparedDictionary accepts a raw dictionary at any + // time, but attaching one after compression has started changes the + // compound dictionary's size mid-stream, which is not supported here.) + have = wuffs_brotli__encoder__add_dictionary( + &enc, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, dict, workbuf) + .repr; + if (have != wuffs_base__error__bad_call_sequence) { + RETURN_FAIL("after transform_io: have \"%s\"", have); + } + return NULL; +} + +// brotli_enc_dict_set fills dicts with one of the dictionary test sets (for +// the alice29.txt, asyoulik.txt and romeo.txt files a, b and r, each of +// length a_len, b_len and r_len) and returns how many dictionaries it has. +size_t // +brotli_enc_dict_set(int which, + wuffs_base__slice_u8* dicts, + uint8_t* a, + size_t a_len, + uint8_t* b, + size_t b_len, + uint8_t* r, + size_t r_len) { + switch (which) { + case 0: // One small dictionary. + dicts[0] = wuffs_base__make_slice_u8(r, r_len); + return 1; + case 1: // The first half of alice29.txt. + dicts[0] = wuffs_base__make_slice_u8(a, a_len / 2); + return 1; + case 2: // Several dictionaries, including tiny and empty ones. + dicts[0] = wuffs_base__make_slice_u8(r, 0); + dicts[1] = wuffs_base__make_slice_u8(b + 1000, 5); + dicts[2] = wuffs_base__make_slice_u8(a + (a_len / 2), a_len / 4); + dicts[3] = wuffs_base__make_slice_u8(b, 4000); + dicts[4] = wuffs_base__make_slice_u8(r + 100, 300); + return 5; + case 3: // 15 (the maximum number of) pieces of alice29.txt. + for (size_t j = 0; j < 15; j++) { + dicts[j] = wuffs_base__make_slice_u8(a + (j * 10000), 9000); + } + return 15; + } + dicts[0] = wuffs_base__make_slice_u8(b, b_len); + return 1; +} + +const char* // +do_test_brotli_encode_dictionary_grid(bool mimic) { + size_t a_len = 0; + size_t b_len = 0; + size_t r_len = 0; + uint8_t* a = brotli_enc_load_file("test/data/brotli/alice29.txt", &a_len); + uint8_t* b = brotli_enc_load_file("test/data/brotli/asyoulik.txt", &b_len); + uint8_t* r = brotli_enc_load_file("test/data/romeo.txt", &r_len); + const char* ret = NULL; + if (!a || !b || !r || (a_len < 150000)) { + ret = "could not load the test files"; + goto cleanup; + } + + // Each config is (lgwin, chunking, flush, metadata, extra param, value). + static const uint32_t configs[][6] = { + {22, BROTLI_ENC_CHUNK_SINGLE, BROTLI_ENC_FLUSH_NONE, 0, 0, 0}, + {16, BROTLI_ENC_CHUNK_RANDOM, BROTLI_ENC_FLUSH_MIDDLE, 0, 0, 0}, + {10, BROTLI_ENC_CHUNK_RANDOM, BROTLI_ENC_FLUSH_EVERY_CHUNK, 0, 0, 0}, + {18, BROTLI_ENC_CHUNK_RANDOM, BROTLI_ENC_FLUSH_MIDDLE, + BROTLI_ENC_METADATA_MIDDLE, 0, 0}, + {22, BROTLI_ENC_CHUNK_RANDOM, BROTLI_ENC_FLUSH_MIDDLE, 0, + BROTLI_ENC_PARAM_STREAM_OFFSET, 1000}, + {24, BROTLI_ENC_CHUNK_SINGLE, BROTLI_ENC_FLUSH_NONE, 0, + BROTLI_ENC_PARAM_STREAM_OFFSET, 1 << 30}, + {26, BROTLI_ENC_CHUNK_RANDOM, BROTLI_ENC_FLUSH_MIDDLE, 0, + BROTLI_ENC_PARAM_LARGE_WINDOW, 1}, + {22, BROTLI_ENC_CHUNK_SINGLE, BROTLI_ENC_FLUSH_NONE, 0, + BROTLI_ENC_PARAM_SIZE_HINT, 1 << 20}, + }; + // The input is the second half of alice29.txt, except for dictionary set + // 4 (all of asyoulik.txt), whose input is romeo.txt. + for (int set = 0; (set < 5) && !ret; set++) { + wuffs_base__slice_u8 dicts[16]; + size_t num_dicts = brotli_enc_dict_set(set, dicts, a, a_len, b, b_len, r, + r_len); + const uint8_t* data = (set == 4) ? r : (a + (a_len / 2)); + size_t data_len = (set == 4) ? r_len : (a_len - (a_len / 2)); + for (uint32_t q = 0; (q <= 9) && !ret; q++) { + for (size_t i = 0; (i < WUFFS_TESTLIB_ARRAY_SIZE(configs)) && !ret; + i++) { + if (!mimic && (i >= 2) && (q & 1)) { + continue; + } + brotli_enc_config c = make_brotli_enc_config(q, configs[i][0]); + if (configs[i][4]) { + brotli_enc_config__add_param(&c, configs[i][4], configs[i][5]); + } + c.chunking = configs[i][1]; + c.flush = configs[i][2]; + c.metadata_position = configs[i][3]; + c.metadata_len = configs[i][3] ? 100 : 0; + c.seed = (set << 16) | (q << 8) | i; + char s[256]; + snprintf(s, sizeof s, "set=%d q=%u config=%zu", set, q, i); + size_t have_len = 0; + ret = do_test_brotli_encode_dictionary(s, &c, data, data_len, + num_dicts, dicts, mimic, + &have_len); + if (ret || (set != 1) || (q < 5) || (i != 0)) { + continue; + } + // Encoding the second half of alice29.txt with the first half as a + // dictionary should be at least 5% better than without it. + size_t without_len = 0; + ret = do_test_brotli_encode_dictionary(s, &c, data, data_len, 0, NULL, + false, &without_len); + if (!ret && ((have_len * 100) >= (without_len * 95))) { + snprintf(g_fail_msg, sizeof g_fail_msg, + "%s: compressed length: have %zu, without dictionary %zu", + s, have_len, without_len); + ret = g_fail_msg; + } + } + } + } + +cleanup: + free(a); + free(b); + free(r); + return ret; +} + +const char* // +do_test_brotli_encode_dictionary_big(bool mimic) { + // Dictionaries of exactly 2 MiB (the largest with 17 bucket bits), one + // byte more (18 bucket bits) and 5 and 33 MiB (19 and 22 bucket bits). + // Each holds, at various offsets, pieces of the 1 MiB input. + static const size_t sizes[] = {2 << 20, (2 << 20) + 1, (5 << 20) + 3, + (33 << 20) + 5}; + const size_t len = 1 << 20; + uint8_t* data = malloc(len); + uint8_t* dict = malloc(sizes[3]); + const char* ret = NULL; + if (!data || !dict) { + ret = "malloc failed"; + goto cleanup; + } + brotli_enc_fill_random(data, len, 0xDA7A); + for (size_t i = 0; i < len; i++) { + // Make it compressible: mostly lowercase letters. + data[i] = 'a' + (data[i] % 13); + } + for (size_t k = 0; (k < WUFFS_TESTLIB_ARRAY_SIZE(sizes)) && !ret; k++) { + size_t dlen = sizes[k]; + brotli_enc_fill_random(dict, dlen, 0xD1C7 + k); + for (size_t j = 0; j < 3; j++) { + size_t at = (j == 0) ? 999 : (j == 1) ? (dlen / 2) : (dlen - (65 << 10)); + memcpy(dict + at, data + (j * (300 << 10)), 64 << 10); + } + wuffs_base__slice_u8 dicts[2] = { + wuffs_base__make_slice_u8(dict, dlen), + wuffs_base__make_slice_u8(data + (len - 1000), 1000), + }; + static const uint32_t qs[] = {2, 4, 5, 9}; + for (size_t i = 0; (i < WUFFS_TESTLIB_ARRAY_SIZE(qs)) && !ret; i++) { + // Without mimic, and for the largest dictionary, a smaller grid. + if ((!mimic && ((k & 1) == 0)) || ((!mimic || (k == 3)) && (i != 2))) { + continue; + } + brotli_enc_config c = make_brotli_enc_config(qs[i], 24); + if (i & 1) { + brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_SIZE_HINT, 1 << 20); + c.chunking = BROTLI_ENC_CHUNK_RANDOM; + c.flush = BROTLI_ENC_FLUSH_MIDDLE; + } + c.seed = (k << 8) | i; + char s[256]; + snprintf(s, sizeof s, "dictionary of %zu bytes: q=%u", dlen, qs[i]); + size_t have_len = 0; + ret = do_test_brotli_encode_dictionary(s, &c, data, len, 2, dicts, + mimic, &have_len); + // The three 64 KiB pieces (of 3.7 bits of entropy per byte) should be + // found at quality 5 and above, even when they are more than 16 MiB + // (lgwin 24's window) into the dictionary. + size_t want_max = 0; + if (!ret && (qs[i] >= 5)) { + c = make_brotli_enc_config(qs[i], 24); + ret = do_test_brotli_encode_dictionary(s, &c, data, len, 0, NULL, + false, &want_max); + want_max -= 3 * (24 << 10); + if (!ret && (have_len > want_max)) { + snprintf(g_fail_msg, sizeof g_fail_msg, + "%s: compressed length: have %zu, want at most %zu", s, + have_len, want_max); + ret = g_fail_msg; + } + } + } + } +cleanup: + free(data); + free(dict); + return ret; +} + +const char* // +test_wuffs_brotli_encode_dictionary_big() { + CHECK_FOCUS(__func__); + return do_test_brotli_encode_dictionary_big(false); +} + +const char* // +test_wuffs_brotli_encode_dictionary_grid() { + CHECK_FOCUS(__func__); + return do_test_brotli_encode_dictionary_grid(false); +} + +#ifdef WUFFS_MIMIC + +const char* // +test_mimic_brotli_encode_dictionary_big() { + CHECK_FOCUS(__func__); + return do_test_brotli_encode_dictionary_big(true); +} + +const char* // +test_mimic_brotli_encode_dictionary_grid() { + CHECK_FOCUS(__func__); + return do_test_brotli_encode_dictionary_grid(true); +} + +// test_mimic_brotli_encode_dictionary_rejections checks that the dictionaries +// that libbrotlienc (a default, non-experimental build) rejects are also +// rejected by Wuffs, and that an empty dictionary is accepted by both. +const char* // +test_mimic_brotli_encode_dictionary_rejections() { + CHECK_FOCUS(__func__); + static const uint8_t text[] = "the quick brown fox jumps over the lazy dog"; + // A serialized dictionary (an LZ77 prefix only), which an experimental + // libbrotlienc build would accept. + static const uint8_t serialized[] = {0x91, 0x00, 0x03, 'f', 'o', 'x', 0x00, + 0x00}; + static const struct { + size_t num_dicts; + uint32_t type; // The last dictionary's type. + size_t len; // The last dictionary's length. + const char* want_wuffs; + const char* want_mimic; + } cases[] = { + {1, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, sizeof serialized, + wuffs_brotli__error__unsupported_serialized_encoder_dictionary, + mimic_brotli_prepare_dictionary_failed_message}, + {16, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 9, + wuffs_brotli__error__too_many_compound_dictionaries, + mimic_brotli_attach_prepared_dictionary_failed_message}, + {15, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 9, NULL, NULL}, + {1, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 0, NULL, NULL}, + {3, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 0, NULL, NULL}, + }; + for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(cases); i++) { + uint32_t dict_types[16]; + wuffs_base__slice_u8 dicts[16]; + for (size_t j = 0; j < cases[i].num_dicts; j++) { + dict_types[j] = WUFFS_BROTLI__DICTIONARY_TYPE_RAW; + dicts[j] = wuffs_base__make_slice_u8((uint8_t*)text + j, 20); + } + size_t last = cases[i].num_dicts - 1; + dict_types[last] = cases[i].type; + dicts[last] = wuffs_base__make_slice_u8( + (uint8_t*)((cases[i].type == WUFFS_BROTLI__DICTIONARY_TYPE_RAW) + ? text + : serialized), + cases[i].len); + + brotli_enc_config c = make_brotli_enc_config(5, 22); + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = g_src_slice_u8, + }); + CHECK_STRING(make_brotli_enc_plan(&src, &c, text, sizeof text - 1)); + wuffs_base__io_buffer dsts[2] = { + ((wuffs_base__io_buffer){.data = g_have_slice_u8}), + ((wuffs_base__io_buffer){.data = g_want_slice_u8}), + }; + for (int k = 0; k < 2; k++) { + wuffs_base__io_buffer src1 = src; + const char* have = + (k ? mimic_brotli_encode_with_dictionaries + : wuffs_brotli_encode_plan_with_dictionaries)( + &dsts[k], &src1, c.param_keys, c.param_values, c.num_params, + cases[i].num_dicts, dict_types, dicts, g_brotli_enc_step_ops, + g_brotli_enc_step_lens, g_brotli_enc_num_steps, c.dst_chunk); + const char* want = k ? cases[i].want_mimic : cases[i].want_wuffs; + if (have != want) { + RETURN_FAIL("cases[%zu] (%s): have \"%s\", want \"%s\"", i, + k ? "mimic" : "wuffs", have ? have : "", + want ? want : ""); + } + } + if (!cases[i].want_wuffs) { + CHECK_STRING(check_io_buffers_equal("", &dsts[0], &dsts[1])); + } + } + return NULL; +} + +#endif // WUFFS_MIMIC + // ---------------- Brotli Benches const char* // @@ -3526,6 +4189,9 @@ proc g_tests[] = { // Encoder tests. + test_wuffs_brotli_encode_add_dictionary, + test_wuffs_brotli_encode_dictionary_big, + test_wuffs_brotli_encode_dictionary_grid, test_wuffs_brotli_encode_interface, test_wuffs_brotli_encode_large_window_far, test_wuffs_brotli_encode_large_window_files, @@ -3537,6 +4203,9 @@ proc g_tests[] = { #ifdef WUFFS_MIMIC + test_mimic_brotli_encode_dictionary_big, + test_mimic_brotli_encode_dictionary_grid, + test_mimic_brotli_encode_dictionary_rejections, test_mimic_brotli_encode_large_window_far, test_mimic_brotli_encode_large_window_files, test_mimic_brotli_encode_parity_corpus,