mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-17 16:32:11 +01:00
std/brotli: test encoder dictionaries and document them
Add mimic_brotli_encode_with_dictionaries to the mimic library and encoder dictionary tests: the add_dictionary API, its workbuf accounting and rejection cases (matching libbrotlienc's default build), a parity grid over qualities 0 to 9 with several dictionary sets, stream offsets, large window, flushes and metadata, and dictionaries of 2 to 33 MiB, each round tripped through both decoders. Document the encoder's dictionary API, workbuf layout and differences from the C code.
This commit is contained in:
5 files changed
+941
-27
No files matched your search
@@ -26,8 +26,10 @@ In Wuffs syntax, the `base.io_transformer` methods are:
|
||||
- `workbuf_len() range_ii_u64`
|
||||
|
||||
Encoders may have further, format-specific methods. For example, the brotli
|
||||
encoder has `request_flush`, `request_emit_metadata` and
|
||||
`max_compressed_size`.
|
||||
encoder has `request_flush`, `request_emit_metadata`, `max_compressed_size`
|
||||
and, for raw (LZ77 prefix) dictionaries, `add_dictionary` and
|
||||
`add_dictionary_workbuf_len`. Like quirks, dictionaries are added before the
|
||||
first `transform_io` call.
|
||||
|
||||
|
||||
## Implementations
|
||||
@@ -36,4 +38,8 @@ encoder has `request_flush`, `request_emit_metadata` and
|
||||
including large-window brotli, with windows of up to 1 GiB, at qualities 3
|
||||
to 9). Its `workbuf_len` is the worst case for the parameters (more than
|
||||
2 GiB for large-window brotli at lgwin 30), but a shorter workbuf works for
|
||||
short inputs: see the README's "Encoder Workbuf Layout" section.
|
||||
short inputs: see the README's "Encoder Workbuf Layout" section. Up to 15
|
||||
raw dictionaries can be attached (like libbrotlienc's
|
||||
`BrotliEncoderAttachPreparedDictionary`), see the README's "Encoder
|
||||
Dictionaries" section. They live at the start of the workbuf, with their
|
||||
prepared hash tables, and `workbuf_len` grows accordingly.
|
||||
@@ -1257,7 +1257,7 @@ run_dict_configs(const char* name, const uint8_t* data, size_t len, uint32_t q,
|
||||
|
||||
// check_smaller checks that, with the current dictionaries (which should
|
||||
// hold much of data), libbrotlienc's output (which Wuffs' output matches) is
|
||||
// smaller than without them.
|
||||
// smaller than without them (unless that is already tiny).
|
||||
static void //
|
||||
check_smaller(const char* name, const uint8_t* data, size_t len, uint32_t q) {
|
||||
config c = make_config(q, 22);
|
||||
@@ -1272,7 +1272,9 @@ check_smaller(const char* name, const uint8_t* data, size_t len, uint32_t q) {
|
||||
}
|
||||
g_num_dicts = saved;
|
||||
g_num_smaller_checks++;
|
||||
if (err || (with_len >= without_len)) {
|
||||
// Tiny outputs (e.g. for long runs of zeroes) cannot shrink.
|
||||
if (err || (with_len > without_len) ||
|
||||
((without_len >= 64) && (with_len >= without_len))) {
|
||||
g_num_mismatches++;
|
||||
printf("NOT SMALLER %s: q=%u dicts=%s: %zu bytes with, %zu without: %s\n",
|
||||
name, q, g_dict_desc, with_len, without_len, err ? err : "");
|
||||
|
||||
+167
-7
@@ -154,8 +154,7 @@ invariants for memory safety but Wuffs requires explicit bounds checks.
|
||||
- Metadata callbacks (metadata blocks are currently skipped). See the
|
||||
"Metadata Blocks" section below.
|
||||
- Fast paths.
|
||||
- Encoder quality levels 10 and 11 and encoder dictionaries (see the
|
||||
"Encoder" section below).
|
||||
- Encoder quality levels 10 and 11 (see the "Encoder" section below).
|
||||
|
||||
|
||||
## Encoder
|
||||
@@ -182,7 +181,8 @@ literal context modeling heuristics, `BrotliBuildMetaBlockGreedy`,
|
||||
`"#unsupported encoder quality"` (and `workbuf_len` is 0). Large-window
|
||||
brotli (see "Encoder Large Window" below) works at qualities 3 to 9 with
|
||||
lgwin 10 to 30. Like the C code, qualities 0 to 2 ignore the large-window
|
||||
quirk.
|
||||
quirk. Raw (LZ77 prefix) dictionaries (see "Encoder Dictionaries" below) work
|
||||
at every quality, and, like the C code, qualities 0 and 1 ignore them.
|
||||
|
||||
The source files are:
|
||||
|
||||
@@ -217,6 +217,13 @@ The source files are:
|
||||
`hash_composite_inc.h` (H35, H55 and H65, which combine H3, H54 or H6 with
|
||||
a rolling hasher) and `hash_rolling_inc.h` (`HROLLING_FAST` and
|
||||
`HROLLING`).
|
||||
- `encode_dictionary.wuffs`: `compound_dictionary.c`
|
||||
(`CreatePreparedDictionary`, `AttachPreparedDictionary`), the raw
|
||||
dictionary cases of `BrotliEncoderPrepareDictionary` and
|
||||
`BrotliEncoderAttachPreparedDictionary` (as `add_dictionary`), the
|
||||
compound dictionary search of `hash.h` (`FindCompoundDictionaryMatch`,
|
||||
`LookupCompoundDictionaryMatch`) and the compound dictionary branch of
|
||||
`ExtendLastCommand`.
|
||||
- `encode_backward_references.wuffs`: `backward_references.c` and
|
||||
`backward_references_inc.h` (`BrotliCreateBackwardReferences`),
|
||||
`command.h` and `ComputeDistanceCode`.
|
||||
@@ -266,6 +273,10 @@ In addition to the `base.io_transformer` methods, the encoder has:
|
||||
another request is pending.
|
||||
- `max_compressed_size(input_size: u64) u64`: mirrors
|
||||
`BrotliEncoderMaxCompressedSize` (0 means overflow).
|
||||
- `add_dictionary!(dict_type: u32, dict: roslice u8, workbuf: slice u8)
|
||||
status` and `add_dictionary_workbuf_len(dict_len: u64) u64`: attach a raw
|
||||
dictionary, like `BrotliEncoderPrepareDictionary` plus
|
||||
`BrotliEncoderAttachPreparedDictionary`. See "Encoder Dictionaries" below.
|
||||
|
||||
`dst_history_retain_length` is 0: the encoder never re-reads `dst`.
|
||||
|
||||
@@ -277,7 +288,8 @@ Each quirk mirrors one `BrotliEncoderParameter`, with the value semantics of
|
||||
otherwise `set_quirk` returns `"#bad argument"`). Like the C code, they are
|
||||
sanitized (`SanitizeParams`, `ComputeLgBlock`, `ChooseDistanceParams`) at the
|
||||
first `transform_io` call, after which `set_quirk` returns
|
||||
`"#bad call sequence"`.
|
||||
`"#bad call sequence"`. Quirks and dictionaries (`add_dictionary`) can be set
|
||||
in any order before that first call.
|
||||
|
||||
| Quirk | C parameter |
|
||||
| ---------------------------------------- | ---------------------------------------------- |
|
||||
@@ -309,6 +321,12 @@ size hint is replaced by the C code's `UpdateSizeHint` estimate.
|
||||
|
||||
### Encoder Workbuf Layout
|
||||
|
||||
Any attached dictionaries come first (see "Encoder Dictionaries" below): the
|
||||
workbuf's first `D` bytes hold them (with their prepared hash tables) and
|
||||
everything described in this section starts at offset `D` (so all of the
|
||||
offsets and lengths below are relative to `workbuf[D ..]`, and `workbuf_len`
|
||||
is `D` more than the lengths given here). `D` is 0 without dictionaries.
|
||||
|
||||
At qualities 0 and 1, the workbuf is the C code's metablock storage (its
|
||||
`GetBrotliStorage` buffer): `workbuf_len` is `(2 << lgwin) + 503 + 16` bytes
|
||||
(the storage for a `1 << lgwin` byte block plus 16 bytes of slack for 8-byte
|
||||
@@ -511,6 +529,135 @@ truncated for windows up to `1 << 30`. The rolling hashers' table holds
|
||||
32-bit positions, like the C code's.
|
||||
|
||||
|
||||
### Encoder Dictionaries
|
||||
|
||||
`encoder.add_dictionary!(dict_type: base.u32, dict: roslice base.u8, workbuf:
|
||||
slice base.u8) base.status` mirrors the C API's
|
||||
`BrotliEncoderPrepareDictionary(BROTLI_SHARED_DICTIONARY_RAW, ...)`
|
||||
followed by `BrotliEncoderAttachPreparedDictionary`: it attaches a raw (LZ77
|
||||
prefix) dictionary, as the next chunk of the C code's compound dictionary.
|
||||
The decoder must be given the same dictionaries, in the same order (with
|
||||
this package's `decoder.add_dictionary` or libbrotlidec's
|
||||
`BrotliDecoderAttachDictionary`, as `DICTIONARY_TYPE_RAW` /
|
||||
`BROTLI_SHARED_DICTIONARY_RAW`).
|
||||
|
||||
Ordering: like quirks, dictionaries must be added before the first
|
||||
`transform_io` call (the C API's "before the first
|
||||
`BrotliEncoderCompressStream`"). Quirks and dictionaries can be set in any
|
||||
order (preparing a raw dictionary does not depend on any parameter: the
|
||||
`quality` argument of `BrotliEncoderPrepareDictionary` is unused for raw
|
||||
dictionaries). Call `workbuf_len` after the last `add_dictionary` and
|
||||
`set_quirk` call.
|
||||
|
||||
Like the decoder, the encoder copies each dictionary into the workbuf, so
|
||||
every `add_dictionary` and `transform_io` call must be passed the same
|
||||
workbuf (or at least one whose first bytes hold the same contents). Each
|
||||
dictionary is followed by its prepared hash tables (the C code's
|
||||
`PreparedDictionary`), which `add_dictionary` builds exactly like
|
||||
`CreatePreparedDictionary` and `CreatePreparedDictionaryWithParams` in
|
||||
`compound_dictionary.c`, so that the compound dictionary search finds the
|
||||
same candidates in the same order:
|
||||
|
||||
- The hash table size depends only on the dictionary length `n`: 17 bucket
|
||||
bits and 7 slot bits for `n` up to 2 MiB, one more of each for each
|
||||
doubling of `n` (18 and 8 up to 4 MiB, etc), up to 22 and 12 (for more than
|
||||
32 MiB). Each position `i` with `i + 7 < n` is hashed (the low 40 bits of
|
||||
its 8 little-endian bytes, times `kPreparedDictionaryHashMul64Long`), with
|
||||
at most 32 positions per bucket (the most recent ones), and the bucket
|
||||
limits are lowered per slot so that each slot has fewer than 65535 items.
|
||||
- Its layout, after the `n` dictionary bytes, is `slot_offsets` (`1 <<
|
||||
slot_bits` u32le values), `heads` (`1 << bucket_bits` u16le values, 0xFFFF
|
||||
for an empty bucket) and `items` (`num_items` u32le values, each a
|
||||
dictionary position, with the high bit marking the end of a bucket's
|
||||
chain), like the C struct's dynamic members.
|
||||
|
||||
So the workbuf layout is:
|
||||
|
||||
[0 .. D) the dictionaries, each one being its n bytes, then
|
||||
(4 << slot_bits) + (2 << bucket_bits) + (4 * num_items)
|
||||
bytes of hash tables. num_items is at most max(0, n - 7).
|
||||
[D .. ) the encoder's other regions (see "Encoder Workbuf Layout").
|
||||
|
||||
Preparing a dictionary needs some scratch space (the C code's temporary
|
||||
"bloated" hasher: `(6 << bucket_bits) + (4 * n)` bytes), which `add_dictionary`
|
||||
puts after the largest possible hash tables. So `add_dictionary`'s workbuf
|
||||
must be at least `add_dictionary_workbuf_len(n)` bytes long: `D` (for the
|
||||
dictionaries added so far) plus `n + (4 << slot_bits) + (2 << bucket_bits) +
|
||||
(4 * max(0, n - 7))` plus that scratch space. For example, for a 1 MiB
|
||||
dictionary that is `D` plus about 9.8 MiB, and for a 40 MiB dictionary `D`
|
||||
plus about 384 MiB. After the call, `D` grows by the dictionary's bytes and
|
||||
actual hash tables (e.g. by at most 5.3 MiB for 1 MiB of dictionary).
|
||||
|
||||
The compound dictionary then affects compression like in the C code:
|
||||
|
||||
- Qualities 0 and 1 (`BrotliEncoderCompressStreamFast`) ignore it.
|
||||
- At qualities 2 to 9, its total size (the C code's `gap`) is added to the
|
||||
static dictionary's base distance (`dictionary_start + gap` in
|
||||
`CreateBackwardReferences`), and distances up to `dictionary_start + gap`
|
||||
(which include compound dictionary references) update the distance cache.
|
||||
`ExtendLastCommand` can extend a copy from the compound dictionary.
|
||||
- Only the hashers that `BrotliCreateBackwardReferences` instantiates with
|
||||
`ENABLE_COMPOUND_DICTIONARY` search the dictionaries (with
|
||||
`LookupCompoundDictionaryMatch`, after the hasher's own search, for each
|
||||
chunk in order): H5, H6, H58, H68, H40, H41, H42, H55 and H65. So
|
||||
qualities 5 to 9 search them, quality 4 only with the large-window H55
|
||||
(lgwin above 24 and a big size hint), and qualities 2 and 3 (H2, H3, H35)
|
||||
and quality 4 with H4 or H54 never do (the C code falls back to its
|
||||
non-dictionary variant for those hashers). The search only finds matches
|
||||
that are within `params.dist.max_distance` (`BROTLI_MAX_DISTANCE`, or
|
||||
`BROTLI_MAX_ALLOWED_DISTANCE` for large-window brotli) and that do not
|
||||
cross a chunk's end.
|
||||
- With `QUIRK_STREAM_OFFSET`, `dictionary_start` is `min(position +
|
||||
stream_offset, max_backward_limit)`, as in the C code (whose output then
|
||||
continues another stream and can only be decoded after it).
|
||||
|
||||
`add_dictionary` statuses (the C code's functions return `NULL` or
|
||||
`BROTLI_FALSE`):
|
||||
|
||||
| Condition | Wuffs status | C code |
|
||||
| ------------------------------------------------------------ | ---------------------------------------------- | ------ |
|
||||
| `transform_io` has been called | `base."#bad call sequence"` | accepted (see below) |
|
||||
| `DICTIONARY_TYPE_SERIALIZED` | `"#unsupported serialized encoder dictionary"` | `BrotliEncoderPrepareDictionary` returns `NULL` |
|
||||
| unknown `dict_type` | `base."#bad argument"` | `BrotliEncoderPrepareDictionary` returns `NULL` |
|
||||
| a 16th dictionary | `"#too many compound dictionaries"` | `BrotliEncoderAttachPreparedDictionary` fails |
|
||||
| more than `0x7FFF_FFFF` bytes of dictionaries in total | `"#too much dictionary data"` | accepted |
|
||||
| `workbuf` shorter than `add_dictionary_workbuf_len(n)` | `base."#bad workbuf length"` | n/a |
|
||||
|
||||
An empty dictionary is accepted (and counts towards the 15), like in the C
|
||||
code. On failure, the encoder's state is unchanged. A `transform_io` call
|
||||
with a workbuf shorter than `D` returns `base."#bad workbuf length"`.
|
||||
|
||||
Differences from the C code:
|
||||
|
||||
- Serialized (shared) dictionaries are rejected, exactly like the default
|
||||
(non-`BROTLI_EXPERIMENTAL`) libbrotlienc build, whose
|
||||
`BrotliEncoderPrepareDictionary` returns `NULL` for them. (The decoder, in
|
||||
contrast, accepts them, like an experimental libbrotlidec build.) With an
|
||||
experimental build, the C encoder would also use their custom static
|
||||
dictionaries; that is not implemented.
|
||||
- The C code's `BrotliEncoderAttachPreparedDictionary` accepts a raw
|
||||
dictionary even after compression has started (it only checks
|
||||
`is_initialized_` for serialized dictionaries). The compound dictionary's
|
||||
size then changes mid-stream, so the static dictionary references before
|
||||
and after that point use different distance bases, and the output is
|
||||
generally not decodable. Here, `add_dictionary` returns `"#bad call
|
||||
sequence"` once `transform_io` has been called.
|
||||
- The C code holds each dictionary's length in a `uint32_t` (and item
|
||||
offsets in 31 bits). This package rejects more than `0x7FFF_FFFF` bytes
|
||||
of dictionaries in total (like its decoder).
|
||||
- When `ExtendLastCommand` extends a copy past the end of a compound
|
||||
dictionary chunk into an empty chunk, the C code reads (one or more
|
||||
bytes) past the end of that empty chunk's (zero-length) data. Here, the
|
||||
extension stops there. This needs an empty dictionary attached after a
|
||||
non-empty one and a copy extension that reaches the non-empty one's end
|
||||
(with empty dictionaries only before the non-empty ones, the behaviors are
|
||||
the same).
|
||||
- Like the C code's `BrotliEncoderPrepareDictionary` with the default
|
||||
allocator, there is no separate "prepared dictionary" object that could be
|
||||
shared between encoders: each encoder prepares its own copy (the C code's
|
||||
"lean" prepared dictionaries reference the caller's bytes instead).
|
||||
|
||||
|
||||
### Encoder Testing
|
||||
|
||||
`wuffs test -mimic std/brotli` checks byte-exact parity with libbrotlienc
|
||||
@@ -529,7 +676,19 @@ processes, `-lw` runs only its large-window grid (lgwin 10 to 30 at
|
||||
qualities 2 to 9, with various chunkings, flushes, size hints and other
|
||||
parameters, each also with the shortest workbuf that should work) and `-big`
|
||||
replaces the inputs with synthetic ones of 3 to 70 MiB with repeats at
|
||||
distances of up to 40 MiB. With `-q=0-9` (the default) it runs 814,706
|
||||
distances of up to 40 MiB. `-dict` replaces the usual grid with a
|
||||
dictionary grid (see "Encoder Dictionaries" above): for each input, seven
|
||||
sets of raw dictionaries (a prefix of the input, the whole input,
|
||||
`test/data/enwik5`, noise plus the input's middle third, 15 pieces of the
|
||||
input, tiny and empty dictionaries, and the input's swapped halves plus some
|
||||
text), with qualities 0 to 9, lgwins, chunkings, flushes, metadata blocks,
|
||||
size hints, stream offsets, large window and other parameters, plus
|
||||
dictionaries of 2 MiB to 40 MiB (every hash table size) with a 1 MiB input.
|
||||
Each dictionary configuration's output is also decoded by libbrotlidec and
|
||||
by this package's decoder (with the same dictionaries), except with a
|
||||
stream offset, and the whole-input and big dictionaries check that the
|
||||
output is smaller than without them (at qualities 5 and above). With
|
||||
`-q=0-9` (the default) and without `-dict` it runs 814,706
|
||||
configurations (96,688 of them in the large-window grid), all byte-exact
|
||||
against libbrotlienc v1.2.1 on x86-64. With `-lw -big` it runs 720 more
|
||||
(on 3, 6 and 12 MiB inputs with the large-window grid and on 24, 40 and
|
||||
@@ -706,10 +865,11 @@ metablock) for as long as it needs the results.
|
||||
|
||||
- Qualities 10 and 11 (Zopfli, the H10 binary tree hasher and
|
||||
`BrotliBuildMetaBlock`, which is already ported in `encode_metablock.wuffs`).
|
||||
- Large-window encoding at qualities 3 and above (it works at quality 2).
|
||||
- `workbuf_len` is a worst case. A future option could take the input length
|
||||
(like the C code's size hint) to give a tighter bound.
|
||||
- Custom and shared dictionaries (`BrotliEncoderAttachPreparedDictionary`).
|
||||
- Compound dictionary matches at qualities 10 and 11 (they need Zopfli).
|
||||
- Serialized (shared) dictionaries, which only an experimental
|
||||
(`BROTLI_EXPERIMENTAL`) libbrotlienc build supports.
|
||||
- Changing parameters mid-stream.
|
||||
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
|
||||
#include "brotli/decode.h"
|
||||
#include "brotli/encode.h"
|
||||
#include "brotli/shared_dictionary.h"
|
||||
|
||||
// mimic_brotli_decode_error_code is like mimic_brotli_decode but also stores
|
||||
// the libbrotlidec error code (a BrotliDecoderErrorCode value) in
|
||||
@@ -91,6 +92,20 @@ mimic_brotli_decode(wuffs_base__io_buffer* dst,
|
||||
//
|
||||
// Each BrotliEncoderCompressStream call gets at most dst_chunk bytes of dst
|
||||
// (or all of dst's writable bytes, if dst_chunk is zero).
|
||||
const char* //
|
||||
mimic_brotli_encode_with_dictionaries(wuffs_base__io_buffer* dst,
|
||||
wuffs_base__io_buffer* src,
|
||||
const uint32_t* param_keys,
|
||||
const uint32_t* param_values,
|
||||
size_t num_params,
|
||||
size_t num_dicts,
|
||||
const uint32_t* dict_types,
|
||||
const wuffs_base__slice_u8* dicts,
|
||||
const uint32_t* step_ops,
|
||||
const size_t* step_lens,
|
||||
size_t num_steps,
|
||||
size_t dst_chunk);
|
||||
|
||||
const char* //
|
||||
mimic_brotli_encode(wuffs_base__io_buffer* dst,
|
||||
wuffs_base__io_buffer* src,
|
||||
@@ -101,15 +116,62 @@ mimic_brotli_encode(wuffs_base__io_buffer* dst,
|
||||
const size_t* step_lens,
|
||||
size_t num_steps,
|
||||
size_t dst_chunk) {
|
||||
return mimic_brotli_encode_with_dictionaries(
|
||||
dst, src, param_keys, param_values, num_params, 0, NULL, NULL, step_ops,
|
||||
step_lens, num_steps, dst_chunk);
|
||||
}
|
||||
|
||||
// mimic_brotli_encode_with_dictionaries is like mimic_brotli_encode but it
|
||||
// also prepares (with BrotliEncoderPrepareDictionary) and attaches (with
|
||||
// BrotliEncoderAttachPreparedDictionary), after setting the parameters and
|
||||
// before the first step, num_dicts dictionaries. Each dict_types[i] is a
|
||||
// BrotliSharedDictionaryType value: 0 for BROTLI_SHARED_DICTIONARY_RAW and 1
|
||||
// for BROTLI_SHARED_DICTIONARY_SERIALIZED. The quality passed to
|
||||
// BrotliEncoderPrepareDictionary is the last BROTLI_PARAM_QUALITY param (or
|
||||
// the default, 11), although the C code ignores it for raw dictionaries.
|
||||
//
|
||||
// libbrotlienc only supports serialized dictionaries when built with
|
||||
// BROTLI_EXPERIMENTAL (which is not the default). When preparing or
|
||||
// attaching a dictionary fails, this function returns
|
||||
// mimic_brotli_prepare_dictionary_failed_message or
|
||||
// mimic_brotli_attach_prepared_dictionary_failed_message.
|
||||
const char mimic_brotli_prepare_dictionary_failed_message[] =
|
||||
"libbrotlienc: BrotliEncoderPrepareDictionary failed";
|
||||
const char mimic_brotli_attach_prepared_dictionary_failed_message[] =
|
||||
"libbrotlienc: BrotliEncoderAttachPreparedDictionary failed";
|
||||
|
||||
#define MIMIC_BROTLI_ENCODE_MAX_DICTS 64
|
||||
|
||||
const char* //
|
||||
mimic_brotli_encode_with_dictionaries(wuffs_base__io_buffer* dst,
|
||||
wuffs_base__io_buffer* src,
|
||||
const uint32_t* param_keys,
|
||||
const uint32_t* param_values,
|
||||
size_t num_params,
|
||||
size_t num_dicts,
|
||||
const uint32_t* dict_types,
|
||||
const wuffs_base__slice_u8* dicts,
|
||||
const uint32_t* step_ops,
|
||||
const size_t* step_lens,
|
||||
size_t num_steps,
|
||||
size_t dst_chunk) {
|
||||
if (BrotliEncoderVersion() != 0x1002001) {
|
||||
return "libbrotlienc: unexpected version (want v1.2.1)";
|
||||
} else if (num_dicts > MIMIC_BROTLI_ENCODE_MAX_DICTS) {
|
||||
return "mimic_brotli_encode: too many dictionaries";
|
||||
}
|
||||
BrotliEncoderPreparedDictionary* prepared[MIMIC_BROTLI_ENCODE_MAX_DICTS];
|
||||
size_t num_prepared = 0;
|
||||
BrotliEncoderState* state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
|
||||
if (!state) {
|
||||
return "libbrotlienc: BrotliEncoderCreateInstance failed";
|
||||
}
|
||||
const char* ret = NULL;
|
||||
int quality = 11;
|
||||
for (size_t j = 0; j < num_params; j++) {
|
||||
if (param_keys[j] == BROTLI_PARAM_QUALITY) {
|
||||
quality = (int)(param_values[j]);
|
||||
}
|
||||
if (!BrotliEncoderSetParameter(state,
|
||||
(BrotliEncoderParameter)(param_keys[j]),
|
||||
param_values[j])) {
|
||||
@@ -117,6 +179,20 @@ mimic_brotli_encode(wuffs_base__io_buffer* dst,
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
for (size_t j = 0; j < num_dicts; j++) {
|
||||
prepared[j] = BrotliEncoderPrepareDictionary(
|
||||
(BrotliSharedDictionaryType)(dict_types[j]), dicts[j].len,
|
||||
dicts[j].ptr, quality, NULL, NULL, NULL);
|
||||
if (!prepared[j]) {
|
||||
ret = mimic_brotli_prepare_dictionary_failed_message;
|
||||
goto cleanup;
|
||||
}
|
||||
num_prepared++;
|
||||
if (!BrotliEncoderAttachPreparedDictionary(state, prepared[j])) {
|
||||
ret = mimic_brotli_attach_prepared_dictionary_failed_message;
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < num_steps; i++) {
|
||||
size_t avail_in = step_lens[i];
|
||||
@@ -156,13 +232,14 @@ mimic_brotli_encode(wuffs_base__io_buffer* dst,
|
||||
|
||||
cleanup:
|
||||
BrotliEncoderDestroyInstance(state);
|
||||
for (size_t j = 0; j < num_prepared; j++) {
|
||||
BrotliEncoderDestroyPreparedDictionary(prepared[j]);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
// ---------------- Large Window and Dictionaries
|
||||
|
||||
#include "brotli/shared_dictionary.h"
|
||||
|
||||
// mimic_brotli_decode_with_options is like mimic_brotli_decode_error_code but
|
||||
// it also sets BROTLI_DECODER_PARAM_LARGE_WINDOW (if large_window is true) and
|
||||
// attaches dictionaries (with BrotliDecoderAttachDictionary). Each
|
||||
|
||||
+682
-13
@@ -854,6 +854,20 @@ brotli_enc_quirk_key(uint32_t param) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const char* //
|
||||
wuffs_brotli_encode_plan_with_dictionaries(wuffs_base__io_buffer* dst,
|
||||
wuffs_base__io_buffer* src,
|
||||
const uint32_t* param_keys,
|
||||
const uint32_t* param_values,
|
||||
size_t num_params,
|
||||
size_t num_dicts,
|
||||
const uint32_t* dict_types,
|
||||
const wuffs_base__slice_u8* dicts,
|
||||
const uint32_t* step_ops,
|
||||
const size_t* step_lens,
|
||||
size_t num_steps,
|
||||
size_t dst_chunk);
|
||||
|
||||
// wuffs_brotli_encode_plan is the Wuffs equivalent of mimic_brotli_encode
|
||||
// (in test/c/mimiclib/brotli.c).
|
||||
const char* //
|
||||
@@ -866,6 +880,29 @@ wuffs_brotli_encode_plan(wuffs_base__io_buffer* dst,
|
||||
const size_t* step_lens,
|
||||
size_t num_steps,
|
||||
size_t dst_chunk) {
|
||||
return wuffs_brotli_encode_plan_with_dictionaries(
|
||||
dst, src, param_keys, param_values, num_params, 0, NULL, NULL, step_ops,
|
||||
step_lens, num_steps, dst_chunk);
|
||||
}
|
||||
|
||||
// wuffs_brotli_encode_plan_with_dictionaries is the Wuffs equivalent of
|
||||
// mimic_brotli_encode_with_dictionaries: it adds (with add_dictionary) the
|
||||
// num_dicts dictionaries after setting the quirks and before the first step.
|
||||
// Each dict_types[i] is WUFFS_BROTLI__DICTIONARY_TYPE_RAW or
|
||||
// WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED.
|
||||
const char* //
|
||||
wuffs_brotli_encode_plan_with_dictionaries(wuffs_base__io_buffer* dst,
|
||||
wuffs_base__io_buffer* src,
|
||||
const uint32_t* param_keys,
|
||||
const uint32_t* param_values,
|
||||
size_t num_params,
|
||||
size_t num_dicts,
|
||||
const uint32_t* dict_types,
|
||||
const wuffs_base__slice_u8* dicts,
|
||||
const uint32_t* step_ops,
|
||||
const size_t* step_lens,
|
||||
size_t num_steps,
|
||||
size_t dst_chunk) {
|
||||
// The encoder struct is large (more than 1 MiB), mostly arrays that do not
|
||||
// need zero-initializing, so don't use wuffs_brotli__encoder__alloc.
|
||||
wuffs_brotli__encoder* enc = malloc(sizeof__wuffs_brotli__encoder());
|
||||
@@ -891,23 +928,45 @@ wuffs_brotli_encode_plan(wuffs_base__io_buffer* dst,
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
wuffs_base__range_ii_u64 workbuf_len =
|
||||
wuffs_brotli__encoder__workbuf_len(enc);
|
||||
wuffs_base__slice_u8 workbuf = g_work_slice_u8;
|
||||
// Large-window brotli at lgwin 30 needs more than 2 GiB.
|
||||
if ((workbuf_len.max_incl > (3ull << 30)) ||
|
||||
(workbuf_len.max_incl > SIZE_MAX)) {
|
||||
ret = "workbuf_len is too large";
|
||||
goto cleanup;
|
||||
} else if (workbuf_len.max_incl > g_work_slice_u8.len) {
|
||||
big_workbuf = malloc((size_t)workbuf_len.max_incl);
|
||||
if (!big_workbuf) {
|
||||
ret = "malloc failed";
|
||||
workbuf.len = 0;
|
||||
size_t big_workbuf_len = 0;
|
||||
for (size_t j = 0; j <= num_dicts; j++) {
|
||||
// Each add_dictionary call needs add_dictionary_workbuf_len bytes and
|
||||
// transform_io needs workbuf_len bytes, with the same dictionary bytes
|
||||
// at the start.
|
||||
uint64_t n = (j < num_dicts)
|
||||
? wuffs_brotli__encoder__add_dictionary_workbuf_len(
|
||||
enc, dicts[j].len)
|
||||
: wuffs_brotli__encoder__workbuf_len(enc).max_incl;
|
||||
// Large-window brotli at lgwin 30 needs more than 2 GiB.
|
||||
if ((n > (3ull << 30)) || (n > SIZE_MAX)) {
|
||||
ret = "workbuf_len is too large";
|
||||
goto cleanup;
|
||||
} else if ((n > g_work_slice_u8.len) && (n > big_workbuf_len)) {
|
||||
uint8_t* p = realloc(big_workbuf, (size_t)n);
|
||||
if (!p) {
|
||||
ret = "malloc failed";
|
||||
goto cleanup;
|
||||
}
|
||||
if (!big_workbuf && workbuf.len) {
|
||||
// Keep the dictionaries added so far.
|
||||
memcpy(p, g_work_slice_u8.ptr, workbuf.len);
|
||||
}
|
||||
big_workbuf = p;
|
||||
big_workbuf_len = (size_t)n;
|
||||
workbuf.ptr = big_workbuf;
|
||||
}
|
||||
workbuf.len = (size_t)n;
|
||||
if (j < num_dicts) {
|
||||
wuffs_base__status status = wuffs_brotli__encoder__add_dictionary(
|
||||
enc, dict_types[j], dicts[j], workbuf);
|
||||
if (status.repr) {
|
||||
ret = status.repr;
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
workbuf.ptr = big_workbuf;
|
||||
}
|
||||
workbuf.len = (size_t)workbuf_len.max_incl;
|
||||
|
||||
for (size_t i = 0; i < num_steps; i++) {
|
||||
size_t end = src->meta.ri + step_lens[i];
|
||||
@@ -3456,6 +3515,610 @@ test_mimic_brotli_encode_large_window_far() {
|
||||
|
||||
#endif // WUFFS_MIMIC
|
||||
|
||||
// ---------------- Brotli Encoder Dictionary Tests
|
||||
|
||||
// brotli_enc_load_file returns a malloc'ed copy of the named file (of up to
|
||||
// 1 MiB) and sets *len to its length.
|
||||
uint8_t* //
|
||||
brotli_enc_load_file(const char* filename, size_t* len) {
|
||||
uint8_t* buf = malloc(1 << 20);
|
||||
if (!buf) {
|
||||
return NULL;
|
||||
}
|
||||
wuffs_base__io_buffer b = ((wuffs_base__io_buffer){
|
||||
.data = wuffs_base__make_slice_u8(buf, 1 << 20),
|
||||
});
|
||||
if (read_file(&b, filename)) {
|
||||
free(buf);
|
||||
return NULL;
|
||||
}
|
||||
*len = b.meta.wi;
|
||||
return buf;
|
||||
}
|
||||
|
||||
// brotli_enc_config_has_param returns whether c sets the key param to a
|
||||
// non-zero value.
|
||||
bool //
|
||||
brotli_enc_config_has_param(const brotli_enc_config* c, uint32_t key) {
|
||||
bool ret = false;
|
||||
for (size_t j = 0; j < c->num_params; j++) {
|
||||
if (c->param_keys[j] == key) {
|
||||
ret = c->param_values[j] != 0;
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
// wuffs_brotli_decode_with_raw_dictionaries decodes src to dst with Wuffs'
|
||||
// std/brotli decoder, with the given raw dictionaries attached.
|
||||
const char* //
|
||||
wuffs_brotli_decode_with_raw_dictionaries(wuffs_base__io_buffer* dst,
|
||||
wuffs_base__io_buffer* src,
|
||||
bool large_window,
|
||||
size_t num_dicts,
|
||||
const wuffs_base__slice_u8* dicts) {
|
||||
wuffs_brotli__decoder dec;
|
||||
CHECK_STATUS("initialize",
|
||||
wuffs_brotli__decoder__initialize(
|
||||
&dec, sizeof dec, WUFFS_VERSION,
|
||||
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
|
||||
CHECK_STATUS("set_quirk",
|
||||
wuffs_brotli__decoder__set_quirk(
|
||||
&dec, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, large_window));
|
||||
size_t total = 0;
|
||||
for (size_t j = 0; j < num_dicts; j++) {
|
||||
total += dicts[j].len;
|
||||
}
|
||||
size_t n = total + (large_window ? (1 << 30) : (1 << 24)) + 542;
|
||||
uint8_t* p = malloc(n);
|
||||
if (!p) {
|
||||
return "malloc failed";
|
||||
}
|
||||
wuffs_base__slice_u8 workbuf = wuffs_base__make_slice_u8(p, n);
|
||||
const char* ret = NULL;
|
||||
for (size_t j = 0; j < num_dicts; j++) {
|
||||
ret = wuffs_brotli__decoder__add_dictionary(
|
||||
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, dicts[j], workbuf)
|
||||
.repr;
|
||||
if (ret) {
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
if (wuffs_brotli__decoder__workbuf_len(&dec).max_incl > n) {
|
||||
ret = "decoder workbuf_len is too large";
|
||||
goto cleanup;
|
||||
}
|
||||
ret = wuffs_brotli__decoder__transform_io(&dec, dst, src, workbuf).repr;
|
||||
cleanup:
|
||||
free(p);
|
||||
return ret;
|
||||
}
|
||||
|
||||
// do_test_brotli_encode_dictionary encodes data[:data_len] with the plan's
|
||||
// config c and the given raw dictionaries. Unless the config sets a stream
|
||||
// offset (whose output is the continuation of another stream), it checks
|
||||
// that Wuffs' decoder (and, if mimic is true, libbrotlidec), with the same
|
||||
// dictionaries, decodes the result back to data. If mimic is true, it also
|
||||
// checks that libbrotlienc's output is the same. It sets *compressed_len.
|
||||
const char* //
|
||||
do_test_brotli_encode_dictionary(const char* name,
|
||||
const brotli_enc_config* c,
|
||||
const uint8_t* data,
|
||||
size_t data_len,
|
||||
size_t num_dicts,
|
||||
const wuffs_base__slice_u8* dicts,
|
||||
bool mimic,
|
||||
size_t* compressed_len) {
|
||||
uint32_t dict_types[16] = {0};
|
||||
for (size_t j = 0; j < 16; j++) {
|
||||
dict_types[j] = WUFFS_BROTLI__DICTIONARY_TYPE_RAW;
|
||||
}
|
||||
if (num_dicts > 16) {
|
||||
return "too many dictionaries";
|
||||
}
|
||||
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
|
||||
.data = g_src_slice_u8,
|
||||
});
|
||||
CHECK_STRING(make_brotli_enc_plan(&src, c, data, data_len));
|
||||
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
|
||||
.data = g_have_slice_u8,
|
||||
});
|
||||
wuffs_base__io_buffer src0 = src;
|
||||
const char* status = wuffs_brotli_encode_plan_with_dictionaries(
|
||||
&have, &src0, c->param_keys, c->param_values, c->num_params, num_dicts,
|
||||
dict_types, dicts, g_brotli_enc_step_ops, g_brotli_enc_step_lens,
|
||||
g_brotli_enc_num_steps, c->dst_chunk);
|
||||
if (status) {
|
||||
RETURN_FAIL("%s: wuffs encode: %s", name, status);
|
||||
}
|
||||
have.meta.closed = true;
|
||||
*compressed_len = have.meta.wi;
|
||||
bool lw = brotli_enc_config_has_param(c, BROTLI_ENC_PARAM_LARGE_WINDOW);
|
||||
bool so = brotli_enc_config_has_param(c, BROTLI_ENC_PARAM_STREAM_OFFSET);
|
||||
|
||||
for (int k = 0; k < (mimic ? 3 : 1); k++) {
|
||||
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
|
||||
.data = g_want_slice_u8,
|
||||
});
|
||||
wuffs_base__io_buffer have1 = have;
|
||||
if (k == 0) {
|
||||
if (so) {
|
||||
continue;
|
||||
}
|
||||
status = wuffs_brotli_decode_with_raw_dictionaries(&want, &have1, lw,
|
||||
num_dicts, dicts);
|
||||
#ifdef WUFFS_MIMIC
|
||||
} else if (k == 1) {
|
||||
if (so) {
|
||||
continue;
|
||||
}
|
||||
int error_code = 0;
|
||||
status = mimic_brotli_decode_with_options(
|
||||
&want, &have1, lw, num_dicts, dict_types, dicts, &error_code);
|
||||
} else {
|
||||
wuffs_base__io_buffer src1 = src;
|
||||
status = mimic_brotli_encode_with_dictionaries(
|
||||
&want, &src1, c->param_keys, c->param_values, c->num_params,
|
||||
num_dicts, dict_types, dicts, g_brotli_enc_step_ops,
|
||||
g_brotli_enc_step_lens, g_brotli_enc_num_steps, c->dst_chunk);
|
||||
if (status) {
|
||||
RETURN_FAIL("%s: mimic encode: %s", name, status);
|
||||
}
|
||||
status = check_io_buffers_equal("", &have, &want);
|
||||
if (status) {
|
||||
RETURN_FAIL("%s: %s", name, status);
|
||||
}
|
||||
continue;
|
||||
#endif // WUFFS_MIMIC
|
||||
}
|
||||
const char* which = (k == 0) ? "wuffs" : "mimic";
|
||||
if (status) {
|
||||
RETURN_FAIL("%s: %s decode: %s", name, which, status);
|
||||
} else if (have1.meta.ri != have1.meta.wi) {
|
||||
RETURN_FAIL("%s: %s decode: trailing data", name, which);
|
||||
} else if (want.meta.wi != data_len) {
|
||||
RETURN_FAIL("%s: %s decoded length: have %zu, want %zu", name, which,
|
||||
want.meta.wi, data_len);
|
||||
} else if (memcmp(want.data.ptr, data, data_len)) {
|
||||
RETURN_FAIL("%s: %s decoded bytes differ", name, which);
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_brotli_encode_add_dictionary() {
|
||||
CHECK_FOCUS(__func__);
|
||||
static const uint8_t text[] = "the quick brown fox jumps over the lazy dog";
|
||||
wuffs_base__slice_u8 dict =
|
||||
wuffs_base__make_slice_u8((uint8_t*)text, sizeof text - 1);
|
||||
wuffs_base__slice_u8 workbuf = g_work_slice_u8;
|
||||
|
||||
wuffs_brotli__encoder enc;
|
||||
CHECK_STATUS("initialize",
|
||||
wuffs_brotli__encoder__initialize(
|
||||
&enc, sizeof enc, WUFFS_VERSION,
|
||||
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
|
||||
CHECK_STATUS("set_quirk", wuffs_brotli__encoder__set_quirk(
|
||||
&enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, 5));
|
||||
uint64_t wlen0 = wuffs_brotli__encoder__workbuf_len(&enc).max_incl;
|
||||
|
||||
// Serialized dictionaries are rejected (like libbrotlienc's default,
|
||||
// non-experimental build) and so are unknown dictionary types.
|
||||
const char* have = wuffs_brotli__encoder__add_dictionary(
|
||||
&enc, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, dict,
|
||||
workbuf)
|
||||
.repr;
|
||||
if (have != wuffs_brotli__error__unsupported_serialized_encoder_dictionary) {
|
||||
RETURN_FAIL("serialized: have \"%s\"", have);
|
||||
}
|
||||
have = wuffs_brotli__encoder__add_dictionary(&enc, 2, dict, workbuf).repr;
|
||||
if (have != wuffs_base__error__bad_argument) {
|
||||
RETURN_FAIL("unknown type: have \"%s\"", have);
|
||||
}
|
||||
|
||||
// The add_dictionary_workbuf_len formula: for a dictionary of n bytes (and
|
||||
// 17 bucket bits and 7 slot bits, for n up to 2 MiB), n bytes, (4 << 7)
|
||||
// slot offsets, (2 << 17) heads and up to 4 * (n - 7) items, then (6 <<
|
||||
// 17) + (4 * n) bytes of scratch space.
|
||||
uint64_t want_wlen =
|
||||
43 + (4 << 7) + (2 << 17) + (4 * 36) + (6 << 17) + (4 * 43);
|
||||
uint64_t have_wlen =
|
||||
wuffs_brotli__encoder__add_dictionary_workbuf_len(&enc, dict.len);
|
||||
if (have_wlen != want_wlen) {
|
||||
RETURN_FAIL("add_dictionary_workbuf_len: have %" PRIu64 ", want %" PRIu64,
|
||||
have_wlen, want_wlen);
|
||||
}
|
||||
workbuf.len = (size_t)(have_wlen - 1);
|
||||
have = wuffs_brotli__encoder__add_dictionary(
|
||||
&enc, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, dict, workbuf)
|
||||
.repr;
|
||||
if (have != wuffs_base__error__bad_workbuf_length) {
|
||||
RETURN_FAIL("short workbuf: have \"%s\"", have);
|
||||
}
|
||||
if (wuffs_brotli__encoder__workbuf_len(&enc).max_incl != wlen0) {
|
||||
RETURN_FAIL("workbuf_len changed after failed add_dictionary calls");
|
||||
}
|
||||
|
||||
// 15 dictionaries (the C code's SHARED_BROTLI_MAX_COMPOUND_DICTS), one of
|
||||
// them empty, can be added, but not 16.
|
||||
uint64_t dict_end = 0;
|
||||
for (int i = 0; i < 16; i++) {
|
||||
wuffs_base__slice_u8 d = dict;
|
||||
if (i == 3) {
|
||||
d.len = 0;
|
||||
}
|
||||
workbuf.len = g_work_slice_u8.len;
|
||||
uint64_t n = wuffs_brotli__encoder__add_dictionary_workbuf_len(&enc, d.len);
|
||||
if (n != (dict_end + ((i == 3) ? ((4 << 7) + (8 << 17)) : want_wlen))) {
|
||||
RETURN_FAIL("i=%d: add_dictionary_workbuf_len: have %" PRIu64, i, n);
|
||||
}
|
||||
have = wuffs_brotli__encoder__add_dictionary(
|
||||
&enc, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, d, workbuf)
|
||||
.repr;
|
||||
if (i == 15) {
|
||||
if (have != wuffs_brotli__error__too_many_compound_dictionaries) {
|
||||
RETURN_FAIL("i=%d: have \"%s\"", i, have);
|
||||
}
|
||||
break;
|
||||
} else if (have) {
|
||||
RETURN_FAIL("i=%d: have \"%s\"", i, have);
|
||||
}
|
||||
// The dictionary bytes, (4 << 7) slot offsets, (2 << 17) heads and 4
|
||||
// bytes per item. Each of the (43 - 7) positions of "the quick brown fox
|
||||
// etc" has a distinct hash, so there are 36 items.
|
||||
dict_end += (i == 3) ? ((4 << 7) + (2 << 17))
|
||||
: (43 + (4 << 7) + (2 << 17) + (4 * 36));
|
||||
uint64_t wlen = wuffs_brotli__encoder__workbuf_len(&enc).max_incl;
|
||||
if (wlen != (wlen0 + dict_end)) {
|
||||
RETURN_FAIL("i=%d: workbuf_len: have %" PRIu64 ", want %" PRIu64, i,
|
||||
wlen, wlen0 + dict_end);
|
||||
}
|
||||
}
|
||||
|
||||
// Quirks can still be set after add_dictionary, until transform_io.
|
||||
CHECK_STATUS("set_quirk", wuffs_brotli__encoder__set_quirk(
|
||||
&enc, WUFFS_BROTLI__QUIRK_LGWIN, 16));
|
||||
uint64_t wlen = wuffs_brotli__encoder__workbuf_len(&enc).max_incl;
|
||||
if (wlen <= dict_end) {
|
||||
RETURN_FAIL("workbuf_len: have %" PRIu64, wlen);
|
||||
}
|
||||
|
||||
// A workbuf shorter than the dictionaries fails.
|
||||
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
|
||||
.data = wuffs_base__make_slice_u8((uint8_t*)text, sizeof text - 1),
|
||||
.meta = wuffs_base__make_io_buffer_meta(sizeof text - 1, 0, 0, true),
|
||||
});
|
||||
wuffs_base__io_buffer dst = ((wuffs_base__io_buffer){
|
||||
.data = g_have_slice_u8,
|
||||
});
|
||||
// (An error disables the encoder, so try this on a copy.)
|
||||
wuffs_brotli__encoder* copy = malloc(sizeof enc);
|
||||
if (!copy) {
|
||||
return "malloc failed";
|
||||
}
|
||||
memcpy(copy, &enc, sizeof enc);
|
||||
workbuf.len = (size_t)(dict_end - 1);
|
||||
have = wuffs_brotli__encoder__transform_io(copy, &dst, &src, workbuf).repr;
|
||||
free(copy);
|
||||
if (have != wuffs_base__error__bad_workbuf_length) {
|
||||
RETURN_FAIL("transform_io (short workbuf): have \"%s\"", have);
|
||||
}
|
||||
workbuf.len = (size_t)wlen;
|
||||
CHECK_STATUS("transform_io",
|
||||
wuffs_brotli__encoder__transform_io(&enc, &dst, &src, workbuf));
|
||||
if (dst.meta.wi >= (sizeof text - 1)) {
|
||||
RETURN_FAIL("compressed length: have %zu", dst.meta.wi);
|
||||
}
|
||||
|
||||
// After transform_io, add_dictionary (and set_quirk) fail. (The C code's
|
||||
// BrotliEncoderAttachPreparedDictionary accepts a raw dictionary at any
|
||||
// time, but attaching one after compression has started changes the
|
||||
// compound dictionary's size mid-stream, which is not supported here.)
|
||||
have = wuffs_brotli__encoder__add_dictionary(
|
||||
&enc, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, dict, workbuf)
|
||||
.repr;
|
||||
if (have != wuffs_base__error__bad_call_sequence) {
|
||||
RETURN_FAIL("after transform_io: have \"%s\"", have);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// brotli_enc_dict_set fills dicts with one of the dictionary test sets (for
|
||||
// the alice29.txt, asyoulik.txt and romeo.txt files a, b and r, each of
|
||||
// length a_len, b_len and r_len) and returns how many dictionaries it has.
|
||||
size_t //
|
||||
brotli_enc_dict_set(int which,
|
||||
wuffs_base__slice_u8* dicts,
|
||||
uint8_t* a,
|
||||
size_t a_len,
|
||||
uint8_t* b,
|
||||
size_t b_len,
|
||||
uint8_t* r,
|
||||
size_t r_len) {
|
||||
switch (which) {
|
||||
case 0: // One small dictionary.
|
||||
dicts[0] = wuffs_base__make_slice_u8(r, r_len);
|
||||
return 1;
|
||||
case 1: // The first half of alice29.txt.
|
||||
dicts[0] = wuffs_base__make_slice_u8(a, a_len / 2);
|
||||
return 1;
|
||||
case 2: // Several dictionaries, including tiny and empty ones.
|
||||
dicts[0] = wuffs_base__make_slice_u8(r, 0);
|
||||
dicts[1] = wuffs_base__make_slice_u8(b + 1000, 5);
|
||||
dicts[2] = wuffs_base__make_slice_u8(a + (a_len / 2), a_len / 4);
|
||||
dicts[3] = wuffs_base__make_slice_u8(b, 4000);
|
||||
dicts[4] = wuffs_base__make_slice_u8(r + 100, 300);
|
||||
return 5;
|
||||
case 3: // 15 (the maximum number of) pieces of alice29.txt.
|
||||
for (size_t j = 0; j < 15; j++) {
|
||||
dicts[j] = wuffs_base__make_slice_u8(a + (j * 10000), 9000);
|
||||
}
|
||||
return 15;
|
||||
}
|
||||
dicts[0] = wuffs_base__make_slice_u8(b, b_len);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const char* //
|
||||
do_test_brotli_encode_dictionary_grid(bool mimic) {
|
||||
size_t a_len = 0;
|
||||
size_t b_len = 0;
|
||||
size_t r_len = 0;
|
||||
uint8_t* a = brotli_enc_load_file("test/data/brotli/alice29.txt", &a_len);
|
||||
uint8_t* b = brotli_enc_load_file("test/data/brotli/asyoulik.txt", &b_len);
|
||||
uint8_t* r = brotli_enc_load_file("test/data/romeo.txt", &r_len);
|
||||
const char* ret = NULL;
|
||||
if (!a || !b || !r || (a_len < 150000)) {
|
||||
ret = "could not load the test files";
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
// Each config is (lgwin, chunking, flush, metadata, extra param, value).
|
||||
static const uint32_t configs[][6] = {
|
||||
{22, BROTLI_ENC_CHUNK_SINGLE, BROTLI_ENC_FLUSH_NONE, 0, 0, 0},
|
||||
{16, BROTLI_ENC_CHUNK_RANDOM, BROTLI_ENC_FLUSH_MIDDLE, 0, 0, 0},
|
||||
{10, BROTLI_ENC_CHUNK_RANDOM, BROTLI_ENC_FLUSH_EVERY_CHUNK, 0, 0, 0},
|
||||
{18, BROTLI_ENC_CHUNK_RANDOM, BROTLI_ENC_FLUSH_MIDDLE,
|
||||
BROTLI_ENC_METADATA_MIDDLE, 0, 0},
|
||||
{22, BROTLI_ENC_CHUNK_RANDOM, BROTLI_ENC_FLUSH_MIDDLE, 0,
|
||||
BROTLI_ENC_PARAM_STREAM_OFFSET, 1000},
|
||||
{24, BROTLI_ENC_CHUNK_SINGLE, BROTLI_ENC_FLUSH_NONE, 0,
|
||||
BROTLI_ENC_PARAM_STREAM_OFFSET, 1 << 30},
|
||||
{26, BROTLI_ENC_CHUNK_RANDOM, BROTLI_ENC_FLUSH_MIDDLE, 0,
|
||||
BROTLI_ENC_PARAM_LARGE_WINDOW, 1},
|
||||
{22, BROTLI_ENC_CHUNK_SINGLE, BROTLI_ENC_FLUSH_NONE, 0,
|
||||
BROTLI_ENC_PARAM_SIZE_HINT, 1 << 20},
|
||||
};
|
||||
// The input is the second half of alice29.txt, except for dictionary set
|
||||
// 4 (all of asyoulik.txt), whose input is romeo.txt.
|
||||
for (int set = 0; (set < 5) && !ret; set++) {
|
||||
wuffs_base__slice_u8 dicts[16];
|
||||
size_t num_dicts = brotli_enc_dict_set(set, dicts, a, a_len, b, b_len, r,
|
||||
r_len);
|
||||
const uint8_t* data = (set == 4) ? r : (a + (a_len / 2));
|
||||
size_t data_len = (set == 4) ? r_len : (a_len - (a_len / 2));
|
||||
for (uint32_t q = 0; (q <= 9) && !ret; q++) {
|
||||
for (size_t i = 0; (i < WUFFS_TESTLIB_ARRAY_SIZE(configs)) && !ret;
|
||||
i++) {
|
||||
if (!mimic && (i >= 2) && (q & 1)) {
|
||||
continue;
|
||||
}
|
||||
brotli_enc_config c = make_brotli_enc_config(q, configs[i][0]);
|
||||
if (configs[i][4]) {
|
||||
brotli_enc_config__add_param(&c, configs[i][4], configs[i][5]);
|
||||
}
|
||||
c.chunking = configs[i][1];
|
||||
c.flush = configs[i][2];
|
||||
c.metadata_position = configs[i][3];
|
||||
c.metadata_len = configs[i][3] ? 100 : 0;
|
||||
c.seed = (set << 16) | (q << 8) | i;
|
||||
char s[256];
|
||||
snprintf(s, sizeof s, "set=%d q=%u config=%zu", set, q, i);
|
||||
size_t have_len = 0;
|
||||
ret = do_test_brotli_encode_dictionary(s, &c, data, data_len,
|
||||
num_dicts, dicts, mimic,
|
||||
&have_len);
|
||||
if (ret || (set != 1) || (q < 5) || (i != 0)) {
|
||||
continue;
|
||||
}
|
||||
// Encoding the second half of alice29.txt with the first half as a
|
||||
// dictionary should be at least 5% better than without it.
|
||||
size_t without_len = 0;
|
||||
ret = do_test_brotli_encode_dictionary(s, &c, data, data_len, 0, NULL,
|
||||
false, &without_len);
|
||||
if (!ret && ((have_len * 100) >= (without_len * 95))) {
|
||||
snprintf(g_fail_msg, sizeof g_fail_msg,
|
||||
"%s: compressed length: have %zu, without dictionary %zu",
|
||||
s, have_len, without_len);
|
||||
ret = g_fail_msg;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cleanup:
|
||||
free(a);
|
||||
free(b);
|
||||
free(r);
|
||||
return ret;
|
||||
}
|
||||
|
||||
const char* //
|
||||
do_test_brotli_encode_dictionary_big(bool mimic) {
|
||||
// Dictionaries of exactly 2 MiB (the largest with 17 bucket bits), one
|
||||
// byte more (18 bucket bits) and 5 and 33 MiB (19 and 22 bucket bits).
|
||||
// Each holds, at various offsets, pieces of the 1 MiB input.
|
||||
static const size_t sizes[] = {2 << 20, (2 << 20) + 1, (5 << 20) + 3,
|
||||
(33 << 20) + 5};
|
||||
const size_t len = 1 << 20;
|
||||
uint8_t* data = malloc(len);
|
||||
uint8_t* dict = malloc(sizes[3]);
|
||||
const char* ret = NULL;
|
||||
if (!data || !dict) {
|
||||
ret = "malloc failed";
|
||||
goto cleanup;
|
||||
}
|
||||
brotli_enc_fill_random(data, len, 0xDA7A);
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
// Make it compressible: mostly lowercase letters.
|
||||
data[i] = 'a' + (data[i] % 13);
|
||||
}
|
||||
for (size_t k = 0; (k < WUFFS_TESTLIB_ARRAY_SIZE(sizes)) && !ret; k++) {
|
||||
size_t dlen = sizes[k];
|
||||
brotli_enc_fill_random(dict, dlen, 0xD1C7 + k);
|
||||
for (size_t j = 0; j < 3; j++) {
|
||||
size_t at = (j == 0) ? 999 : (j == 1) ? (dlen / 2) : (dlen - (65 << 10));
|
||||
memcpy(dict + at, data + (j * (300 << 10)), 64 << 10);
|
||||
}
|
||||
wuffs_base__slice_u8 dicts[2] = {
|
||||
wuffs_base__make_slice_u8(dict, dlen),
|
||||
wuffs_base__make_slice_u8(data + (len - 1000), 1000),
|
||||
};
|
||||
static const uint32_t qs[] = {2, 4, 5, 9};
|
||||
for (size_t i = 0; (i < WUFFS_TESTLIB_ARRAY_SIZE(qs)) && !ret; i++) {
|
||||
// Without mimic, and for the largest dictionary, a smaller grid.
|
||||
if ((!mimic && ((k & 1) == 0)) || ((!mimic || (k == 3)) && (i != 2))) {
|
||||
continue;
|
||||
}
|
||||
brotli_enc_config c = make_brotli_enc_config(qs[i], 24);
|
||||
if (i & 1) {
|
||||
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_SIZE_HINT, 1 << 20);
|
||||
c.chunking = BROTLI_ENC_CHUNK_RANDOM;
|
||||
c.flush = BROTLI_ENC_FLUSH_MIDDLE;
|
||||
}
|
||||
c.seed = (k << 8) | i;
|
||||
char s[256];
|
||||
snprintf(s, sizeof s, "dictionary of %zu bytes: q=%u", dlen, qs[i]);
|
||||
size_t have_len = 0;
|
||||
ret = do_test_brotli_encode_dictionary(s, &c, data, len, 2, dicts,
|
||||
mimic, &have_len);
|
||||
// The three 64 KiB pieces (of 3.7 bits of entropy per byte) should be
|
||||
// found at quality 5 and above, even when they are more than 16 MiB
|
||||
// (lgwin 24's window) into the dictionary.
|
||||
size_t want_max = 0;
|
||||
if (!ret && (qs[i] >= 5)) {
|
||||
c = make_brotli_enc_config(qs[i], 24);
|
||||
ret = do_test_brotli_encode_dictionary(s, &c, data, len, 0, NULL,
|
||||
false, &want_max);
|
||||
want_max -= 3 * (24 << 10);
|
||||
if (!ret && (have_len > want_max)) {
|
||||
snprintf(g_fail_msg, sizeof g_fail_msg,
|
||||
"%s: compressed length: have %zu, want at most %zu", s,
|
||||
have_len, want_max);
|
||||
ret = g_fail_msg;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cleanup:
|
||||
free(data);
|
||||
free(dict);
|
||||
return ret;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_brotli_encode_dictionary_big() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_test_brotli_encode_dictionary_big(false);
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_brotli_encode_dictionary_grid() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_test_brotli_encode_dictionary_grid(false);
|
||||
}
|
||||
|
||||
#ifdef WUFFS_MIMIC
|
||||
|
||||
const char* //
|
||||
test_mimic_brotli_encode_dictionary_big() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_test_brotli_encode_dictionary_big(true);
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_mimic_brotli_encode_dictionary_grid() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_test_brotli_encode_dictionary_grid(true);
|
||||
}
|
||||
|
||||
// test_mimic_brotli_encode_dictionary_rejections checks that the dictionaries
|
||||
// that libbrotlienc (a default, non-experimental build) rejects are also
|
||||
// rejected by Wuffs, and that an empty dictionary is accepted by both.
|
||||
const char* //
|
||||
test_mimic_brotli_encode_dictionary_rejections() {
|
||||
CHECK_FOCUS(__func__);
|
||||
static const uint8_t text[] = "the quick brown fox jumps over the lazy dog";
|
||||
// A serialized dictionary (an LZ77 prefix only), which an experimental
|
||||
// libbrotlienc build would accept.
|
||||
static const uint8_t serialized[] = {0x91, 0x00, 0x03, 'f', 'o', 'x', 0x00,
|
||||
0x00};
|
||||
static const struct {
|
||||
size_t num_dicts;
|
||||
uint32_t type; // The last dictionary's type.
|
||||
size_t len; // The last dictionary's length.
|
||||
const char* want_wuffs;
|
||||
const char* want_mimic;
|
||||
} cases[] = {
|
||||
{1, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, sizeof serialized,
|
||||
wuffs_brotli__error__unsupported_serialized_encoder_dictionary,
|
||||
mimic_brotli_prepare_dictionary_failed_message},
|
||||
{16, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 9,
|
||||
wuffs_brotli__error__too_many_compound_dictionaries,
|
||||
mimic_brotli_attach_prepared_dictionary_failed_message},
|
||||
{15, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 9, NULL, NULL},
|
||||
{1, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 0, NULL, NULL},
|
||||
{3, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 0, NULL, NULL},
|
||||
};
|
||||
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(cases); i++) {
|
||||
uint32_t dict_types[16];
|
||||
wuffs_base__slice_u8 dicts[16];
|
||||
for (size_t j = 0; j < cases[i].num_dicts; j++) {
|
||||
dict_types[j] = WUFFS_BROTLI__DICTIONARY_TYPE_RAW;
|
||||
dicts[j] = wuffs_base__make_slice_u8((uint8_t*)text + j, 20);
|
||||
}
|
||||
size_t last = cases[i].num_dicts - 1;
|
||||
dict_types[last] = cases[i].type;
|
||||
dicts[last] = wuffs_base__make_slice_u8(
|
||||
(uint8_t*)((cases[i].type == WUFFS_BROTLI__DICTIONARY_TYPE_RAW)
|
||||
? text
|
||||
: serialized),
|
||||
cases[i].len);
|
||||
|
||||
brotli_enc_config c = make_brotli_enc_config(5, 22);
|
||||
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
|
||||
.data = g_src_slice_u8,
|
||||
});
|
||||
CHECK_STRING(make_brotli_enc_plan(&src, &c, text, sizeof text - 1));
|
||||
wuffs_base__io_buffer dsts[2] = {
|
||||
((wuffs_base__io_buffer){.data = g_have_slice_u8}),
|
||||
((wuffs_base__io_buffer){.data = g_want_slice_u8}),
|
||||
};
|
||||
for (int k = 0; k < 2; k++) {
|
||||
wuffs_base__io_buffer src1 = src;
|
||||
const char* have =
|
||||
(k ? mimic_brotli_encode_with_dictionaries
|
||||
: wuffs_brotli_encode_plan_with_dictionaries)(
|
||||
&dsts[k], &src1, c.param_keys, c.param_values, c.num_params,
|
||||
cases[i].num_dicts, dict_types, dicts, g_brotli_enc_step_ops,
|
||||
g_brotli_enc_step_lens, g_brotli_enc_num_steps, c.dst_chunk);
|
||||
const char* want = k ? cases[i].want_mimic : cases[i].want_wuffs;
|
||||
if (have != want) {
|
||||
RETURN_FAIL("cases[%zu] (%s): have \"%s\", want \"%s\"", i,
|
||||
k ? "mimic" : "wuffs", have ? have : "",
|
||||
want ? want : "");
|
||||
}
|
||||
}
|
||||
if (!cases[i].want_wuffs) {
|
||||
CHECK_STRING(check_io_buffers_equal("", &dsts[0], &dsts[1]));
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#endif // WUFFS_MIMIC
|
||||
|
||||
// ---------------- Brotli Benches
|
||||
|
||||
const char* //
|
||||
@@ -3526,6 +4189,9 @@ proc g_tests[] = {
|
||||
|
||||
// Encoder tests.
|
||||
|
||||
test_wuffs_brotli_encode_add_dictionary,
|
||||
test_wuffs_brotli_encode_dictionary_big,
|
||||
test_wuffs_brotli_encode_dictionary_grid,
|
||||
test_wuffs_brotli_encode_interface,
|
||||
test_wuffs_brotli_encode_large_window_far,
|
||||
test_wuffs_brotli_encode_large_window_files,
|
||||
@@ -3537,6 +4203,9 @@ proc g_tests[] = {
|
||||
|
||||
#ifdef WUFFS_MIMIC
|
||||
|
||||
test_mimic_brotli_encode_dictionary_big,
|
||||
test_mimic_brotli_encode_dictionary_grid,
|
||||
test_mimic_brotli_encode_dictionary_rejections,
|
||||
test_mimic_brotli_encode_large_window_far,
|
||||
test_mimic_brotli_encode_large_window_files,
|
||||
test_mimic_brotli_encode_parity_corpus,
|
||||
|
||||
Reference in new issue
Block a user