diff --git a/fuzz/c/std/brotli_fuzzer.c b/fuzz/c/std/brotli_fuzzer.c new file mode 100644 index 00000000..96e66cf3 --- /dev/null +++ b/fuzz/c/std/brotli_fuzzer.c @@ -0,0 +1,213 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// ---------------- + +// Silence the nested slash-star warning for the next comment's command line. +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcomment" + +/* +This fuzzer (the fuzz function) is typically run indirectly, by a framework +such as https://github.com/google/oss-fuzz calling LLVMFuzzerTestOneInput. + +When working on the fuzz implementation, or as a coherence check, defining +WUFFS_CONFIG__FUZZLIB_MAIN will let you manually run fuzz over a set of files: + +gcc -DWUFFS_CONFIG__FUZZLIB_MAIN brotli_fuzzer.c +./a.out ../../../test/data/brotli/* +rm -f ./a.out + +It should print "PASS", amongst other information, and exit(0). +*/ + +#pragma clang diagnostic pop + +// Wuffs ships as a "single file C library" or "header file library" as per +// https://github.com/nothings/stb/blob/master/docs/stb_howto.txt +// +// To use that single file as a "foo.c"-like implementation, instead of a +// "foo.h"-like header, #define WUFFS_IMPLEMENTATION before #include'ing or +// compiling it. +#define WUFFS_IMPLEMENTATION + +#if defined(WUFFS_CONFIG__FUZZLIB_MAIN) +// Defining the WUFFS_CONFIG__STATIC_FUNCTIONS macro is optional, but when +// combined with WUFFS_IMPLEMENTATION, it demonstrates making all of Wuffs' +// functions have static storage. +// +// This can help the compiler ignore or discard unused code, which can produce +// faster compiles and smaller binaries. Other motivations are discussed in the +// "ALLOW STATIC IMPLEMENTATION" section of +// https://raw.githubusercontent.com/nothings/stb/master/docs/stb_howto.txt +#define WUFFS_CONFIG__STATIC_FUNCTIONS +#endif // defined(WUFFS_CONFIG__FUZZLIB_MAIN) + +// Defining the WUFFS_CONFIG__MODULE* macros are optional, but it lets users of +// release/c/etc.c choose which parts of Wuffs to build. That file contains the +// entire Wuffs standard library, implementing a variety of codecs and file +// formats. Without this macro definition, an optimizing compiler or linker may +// very well discard Wuffs code for unused codecs, but listing the Wuffs +// modules we use makes that process explicit. Preprocessing means that such +// code simply isn't compiled. +#define WUFFS_CONFIG__MODULES +#define WUFFS_CONFIG__MODULE__BASE +#define WUFFS_CONFIG__MODULE__BROTLI + +// If building this program in an environment that doesn't easily accommodate +// relative includes, you can use the script/inline-c-relative-includes.go +// program to generate a stand-alone C file. +#include "../../../release/c/wuffs-unsupported-snapshot.c" +#include "../fuzzlib/fuzzlib.c" + +// 64 KiB. +#define DST_BUFFER_ARRAY_SIZE 65536 + +// Wuffs allows either statically or dynamically allocated work buffers. This +// program exercises static allocation. +// +// We hard-code 64 MiB, as WUFFS_BROTLI__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE +// is over 3 GiB and even the large-window workbuf_len is over 1 GiB. Using +// those can cause "relocation truncated to fit: R_X86_64_PC32 against symbol +// etc" compiler errors. Large-window streams with a window of 64 MiB or more +// are rejected with "#base: bad workbuf length", but smaller ones are fine. +#define WORK_BUFFER_ARRAY_SIZE 67108864 +#if WORK_BUFFER_ARRAY_SIZE > 0 +uint8_t g_work_buffer_array[WORK_BUFFER_ARRAY_SIZE]; +#else +// Not all C/C++ compilers support 0-length arrays. +uint8_t g_work_buffer_array[1]; +#endif + +// g_raw_dictionary is a raw (LZ77 prefix) dictionary. +static const uint8_t g_raw_dictionary[] = + "The quick brown fox jumps over the lazy dog. " + "Wuffs"; + +// g_serialized_dictionary is a small serialized shared dictionary. It has an +// LZ77 prefix, a custom word list and a custom transform list (with +// BROTLI_TRANSFORM_SHIFT_FIRST and BROTLI_TRANSFORM_SHIFT_ALL transforms) and +// two context-selected word and transform list combinations. +static const uint8_t g_serialized_dictionary[] = { + 0x91, 0x00, 0x11, 0x77, 0x75, 0x66, 0x66, 0x73, 0x20, 0x62, 0x72, 0x6F, + 0x74, 0x6C, 0x69, 0x20, 0x66, 0x75, 0x7A, 0x7A, 0x01, 0x01, 0x01, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x6F, 0x76, 0x65, 0x72, 0x6C, 0x61, 0x7A, 0x79, 0x71, 0x75, 0x69, + 0x63, 0x6B, 0x62, 0x72, 0x6F, 0x77, 0x6E, 0x77, 0x69, 0x7A, 0x61, 0x72, + 0x64, 0x73, 0x71, 0x75, 0x69, 0x65, 0x74, 0x6C, 0x79, 0x01, 0x06, 0x00, + 0x01, 0x20, 0x02, 0xC3, 0xA9, 0x00, 0x06, 0x02, 0x00, 0x02, 0x02, 0x15, + 0x00, 0x01, 0x16, 0x02, 0x02, 0x0B, 0x01, 0x00, 0x0E, 0x00, 0x02, 0x15, + 0x02, 0x00, 0x00, 0x41, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x80, 0x02, 0x00, 0x00, 0x01, 0x01, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, +}; + +const char* // +fuzz(wuffs_base__io_buffer* src, uint64_t hash) { + wuffs_brotli__decoder dec; + wuffs_base__status status = wuffs_brotli__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + (hash & 1) ? WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED : 0); + if (!wuffs_base__status__is_ok(&status)) { + return wuffs_base__status__message(&status); + } + + // Enable large-window brotli for half of the inputs. + wuffs_brotli__decoder__set_quirk(&dec, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, + (hash >> 1) & 1); + + wuffs_base__slice_u8 workbuf = + wuffs_base__make_slice_u8(g_work_buffer_array, WORK_BUFFER_ARRAY_SIZE); + + // Attach dictionaries for some of the inputs. + switch ((hash >> 2) & 7) { + case 1: + status = wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, + wuffs_base__make_slice_u8((uint8_t*)g_raw_dictionary, + sizeof g_raw_dictionary - 1), + workbuf); + break; + case 2: + status = wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, + wuffs_base__make_slice_u8((uint8_t*)g_serialized_dictionary, + sizeof g_serialized_dictionary), + workbuf); + break; + case 3: + status = wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, + wuffs_base__make_slice_u8((uint8_t*)g_serialized_dictionary, + sizeof g_serialized_dictionary), + workbuf); + if (wuffs_base__status__is_ok(&status)) { + status = wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, + wuffs_base__make_slice_u8((uint8_t*)g_raw_dictionary, + sizeof g_raw_dictionary - 1), + workbuf); + } + break; + } + if (!wuffs_base__status__is_ok(&status)) { + return wuffs_base__status__message(&status); + } + wuffs_base__range_ii_u64 workbuf_len = + wuffs_brotli__decoder__workbuf_len(&dec); + if (workbuf.len > workbuf_len.max_incl) { + workbuf.len = (size_t)workbuf_len.max_incl; + } + + static uint8_t dst_buffer[DST_BUFFER_ARRAY_SIZE]; + wuffs_base__io_buffer dst = + wuffs_base__ptr_u8__writer(&dst_buffer[0], DST_BUFFER_ARRAY_SIZE); + + // Use random-ish I/O limits for some of the inputs. + bool limited = ((hash >> 5) & 3) == 0; + size_t wlimit = 1 + ((hash >> 8) & 0xFFFF); + size_t rlimit = 1 + ((hash >> 24) & 0xFFF); + + while (true) { + dst.meta.wi = 0; + wuffs_base__io_buffer limited_dst = dst; + wuffs_base__io_buffer limited_src = *src; + if (limited) { + if (limited_dst.data.len > wlimit) { + limited_dst.data.len = wlimit; + } + if ((limited_src.meta.wi - limited_src.meta.ri) > rlimit) { + limited_src.meta.wi = limited_src.meta.ri + rlimit; + limited_src.meta.closed = false; + } + } + status = wuffs_brotli__decoder__transform_io(&dec, &limited_dst, + &limited_src, workbuf); + src->meta.ri = limited_src.meta.ri; + if (status.repr == wuffs_base__suspension__short_read) { + if (limited && (src->meta.ri < src->meta.wi)) { + continue; + } + break; + } else if (status.repr != wuffs_base__suspension__short_write) { + break; + } + if (limited_dst.meta.wi == 0) { + fprintf(stderr, "wuffs_brotli__decoder__transform_io made no progress\n"); + intentional_segfault(); + } + } + return wuffs_base__status__message(&status); +} diff --git a/fuzz/c/std/seed_corpora.txt b/fuzz/c/std/seed_corpora.txt index 450d383d..4499525c 100644 --- a/fuzz/c/std/seed_corpora.txt +++ b/fuzz/c/std/seed_corpora.txt @@ -15,6 +15,7 @@ # https://github.com/google/oss-fuzz/blob/master/projects/wuffs/Dockerfile bmp: test/data/*.bmp ../bmpsuite_corpus/*.bmp +brotli: test/data/brotli/*.br test/data/brotli/*.compressed* bzip2: test/data/*.bz2 cbor: test/data/*.cbor gif: test/data/*.gif test/data/artificial-gif/*.gif diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 1228c742..5c8d7faf 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -8497,6 +8497,7 @@ struct wuffs_bmp__decoder__struct { extern const char wuffs_brotli__error__bad_block_length[]; extern const char wuffs_brotli__error__bad_block_switch[]; extern const char wuffs_brotli__error__bad_code_length_code_lengths[]; +extern const char wuffs_brotli__error__bad_compound_dictionary[]; extern const char wuffs_brotli__error__bad_context_map_repeat[]; extern const char wuffs_brotli__error__bad_dictionary[]; extern const char wuffs_brotli__error__bad_distance[]; @@ -8507,18 +8508,32 @@ extern const char wuffs_brotli__error__bad_huffman_space[]; extern const char wuffs_brotli__error__bad_metablock_length[]; extern const char wuffs_brotli__error__bad_padding[]; extern const char wuffs_brotli__error__bad_reserved_bit[]; +extern const char wuffs_brotli__error__bad_shared_dictionary[]; extern const char wuffs_brotli__error__bad_simple_huffman_alphabet[]; extern const char wuffs_brotli__error__bad_simple_huffman_symbol[]; extern const char wuffs_brotli__error__bad_transform[]; extern const char wuffs_brotli__error__bad_window_bits[]; +extern const char wuffs_brotli__error__too_many_compound_dictionaries[]; +extern const char wuffs_brotli__error__too_much_dictionary_data[]; extern const char wuffs_brotli__error__truncated_input[]; extern const char wuffs_brotli__error__unsupported_large_window[]; +extern const char wuffs_brotli__error__unsupported_multiple_custom_static_dictionaries[]; // ---------------- Public Consts #define WUFFS_BROTLI__DECODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE 0u -#define WUFFS_BROTLI__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE 16777758u +#define WUFFS_BROTLI__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE 3221226013u + +#define WUFFS_BROTLI__DECODER_WORKBUF_LEN_STANDARD 16777758u + +#define WUFFS_BROTLI__DECODER_WORKBUF_LEN_LARGE_WINDOW 1073742366u + +#define WUFFS_BROTLI__DICTIONARY_TYPE_RAW 0u + +#define WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED 1u + +#define WUFFS_BROTLI__QUIRK_LARGE_WINDOW 716673024u // ---------------- Struct Declarations @@ -8603,6 +8618,14 @@ wuffs_brotli__decoder__transform_io( wuffs_base__io_buffer* a_src, wuffs_base__slice_u8 a_workbuf); +WUFFS_BASE__GENERATED_C_CODE +WUFFS_BASE__MAYBE_STATIC wuffs_base__status +wuffs_brotli__decoder__add_dictionary( + wuffs_brotli__decoder* self, + uint32_t a_dict_type, + wuffs_base__slice_u8 a_dict, + wuffs_base__slice_u8 a_workbuf); + #ifdef __cplusplus } // extern "C" #endif @@ -8665,10 +8688,32 @@ struct wuffs_brotli__decoder__struct { bool f_implicit_zero_distance; uint32_t f_insert_length; uint32_t f_copy_length; + bool f_quirk_large_window; + bool f_large_window; + bool f_started; + uint64_t f_dict_total; + uint32_t f_num_chunks; + uint32_t f_chunk_starts[15]; + uint32_t f_chunk_offsets[16]; + bool f_sd_context_based; + uint32_t f_sd_num_dictionaries; + uint32_t f_sd_num_word_lists; + uint32_t f_sd_num_transform_lists; + uint8_t f_sd_words[64]; + uint8_t f_sd_transforms[64]; + uint8_t f_sd_context_map[64]; + uint64_t f_sd_prefix_pos; + uint64_t f_sd_prefix_size; + uint32_t f_stage_num_word_lists; + uint32_t f_stage_num_transform_lists; + uint32_t f_stage_num_dictionaries; + bool f_stage_context_based; + uint64_t f_parse_pos; uint32_t p_transform_io; uint32_t p_do_transform_io; uint32_t p_read_bits; + uint32_t p_read_bits32; uint32_t p_read_symbol; uint32_t p_decode_window_bits; uint32_t p_decode_var_len_uint8; @@ -8688,7 +8733,7 @@ struct wuffs_brotli__decoder__struct { } private_impl; struct { - uint16_t f_huffs[671890]; + uint16_t f_huffs[678034]; uint8_t f_context_maps[17408]; uint8_t f_context_modes[256]; uint8_t f_code_lengths[704]; @@ -8699,7 +8744,18 @@ struct wuffs_brotli__decoder__struct { uint8_t f_mtf[256]; uint8_t f_dist_extra_bits[1128]; uint32_t f_dist_offset[1128]; - uint8_t f_transformed_word[64]; + uint8_t f_transformed_word[1024]; + uint8_t f_wl_size_bits[64][32]; + uint32_t f_wl_offsets[64][32]; + uint32_t f_wl_data[64]; + uint32_t f_tl_prefix_suffix[64]; + uint16_t f_tl_map[64][256]; + uint32_t f_tl_num_transforms[64]; + uint32_t f_tl_transforms[64]; + uint32_t f_tl_params[64]; + uint8_t f_stage_words[64]; + uint8_t f_stage_transforms[64]; + uint8_t f_stage_context_map[64]; struct { uint64_t scratch; @@ -8709,6 +8765,10 @@ struct wuffs_brotli__decoder__struct { uint32_t v_bits; uint32_t v_n_bits; } s_read_bits; + struct { + uint32_t v_n; + uint32_t v_lo; + } s_read_bits32; struct { uint32_t v_hoff; uint32_t v_bits; @@ -8758,6 +8818,7 @@ struct wuffs_brotli__decoder__struct { uint32_t v_n2; uint32_t v_npostfix; uint32_t v_ndirect; + uint32_t v_alphabet_max; uint32_t v_alphabet; uint32_t v_i; uint32_t v_k; @@ -8786,6 +8847,8 @@ struct wuffs_brotli__decoder__struct { uint32_t v_i; uint32_t v_n; uint32_t v_k; + uint32_t v_address; + uint32_t v_chunk; } s_process_commands; struct { uint32_t v_code; @@ -8885,6 +8948,14 @@ struct wuffs_brotli__decoder__struct { return wuffs_brotli__decoder__transform_io(this, a_dst, a_src, a_workbuf); } + inline wuffs_base__status + add_dictionary( + uint32_t a_dict_type, + wuffs_base__slice_u8 a_dict, + wuffs_base__slice_u8 a_workbuf) { + return wuffs_brotli__decoder__add_dictionary(this, a_dict_type, a_dict, a_workbuf); + } + #endif // __cplusplus }; // struct wuffs_brotli__decoder__struct @@ -40133,6 +40204,7 @@ wuffs_bmp__decoder__process_masks( const char wuffs_brotli__error__bad_block_length[] = "#brotli: bad block length"; const char wuffs_brotli__error__bad_block_switch[] = "#brotli: bad block switch"; const char wuffs_brotli__error__bad_code_length_code_lengths[] = "#brotli: bad code length code lengths"; +const char wuffs_brotli__error__bad_compound_dictionary[] = "#brotli: bad compound dictionary"; const char wuffs_brotli__error__bad_context_map_repeat[] = "#brotli: bad context map repeat"; const char wuffs_brotli__error__bad_dictionary[] = "#brotli: bad dictionary"; const char wuffs_brotli__error__bad_distance[] = "#brotli: bad distance"; @@ -40143,12 +40215,16 @@ const char wuffs_brotli__error__bad_huffman_space[] = "#brotli: bad Huffman spac const char wuffs_brotli__error__bad_metablock_length[] = "#brotli: bad metablock length"; const char wuffs_brotli__error__bad_padding[] = "#brotli: bad padding"; const char wuffs_brotli__error__bad_reserved_bit[] = "#brotli: bad reserved bit"; +const char wuffs_brotli__error__bad_shared_dictionary[] = "#brotli: bad shared dictionary"; const char wuffs_brotli__error__bad_simple_huffman_alphabet[] = "#brotli: bad simple Huffman alphabet"; const char wuffs_brotli__error__bad_simple_huffman_symbol[] = "#brotli: bad simple Huffman symbol"; const char wuffs_brotli__error__bad_transform[] = "#brotli: bad transform"; const char wuffs_brotli__error__bad_window_bits[] = "#brotli: bad window bits"; +const char wuffs_brotli__error__too_many_compound_dictionaries[] = "#brotli: too many compound dictionaries"; +const char wuffs_brotli__error__too_much_dictionary_data[] = "#brotli: too much dictionary data"; const char wuffs_brotli__error__truncated_input[] = "#brotli: truncated input"; const char wuffs_brotli__error__unsupported_large_window[] = "#brotli: unsupported large window"; +const char wuffs_brotli__error__unsupported_multiple_custom_static_dictionaries[] = "#brotli: unsupported multiple custom static dictionaries"; const char wuffs_brotli__error__internal_error_inconsistent_bit_reader[] = "#brotli: internal error: inconsistent bit reader"; const char wuffs_brotli__error__internal_error_inconsistent_huffman_table[] = "#brotli: internal error: inconsistent Huffman table"; const char wuffs_brotli__error__internal_error_inconsistent_ring_buffer[] = "#brotli: internal error: inconsistent ring buffer"; @@ -56513,6 +56589,8 @@ WUFFS_BROTLI__DICTIONARY[122784] WUFFS_BASE__POTENTIALLY_UNUSED = { 164u, 175u, 224u, 164u, 164u, 224u, 164u, 190u, }; +#define WUFFS_BROTLI__DICT_TOTAL_MAX 2147483647u + #define WUFFS_BROTLI__WORKBUF_SLACK 542u #define WUFFS_BROTLI__LIT_HUFFS_BASE 0u @@ -56525,26 +56603,28 @@ WUFFS_BROTLI__DICTIONARY[122784] WUFFS_BASE__POTENTIALLY_UNUSED = { #define WUFFS_BROTLI__DIST_HUFFS_BASE 438272u -#define WUFFS_BROTLI__DIST_HUFFS_STRIDE 896u +#define WUFFS_BROTLI__DIST_HUFFS_STRIDE 920u -#define WUFFS_BROTLI__BLOCK_TYPE_HUFFS_BASE 667648u +#define WUFFS_BROTLI__BLOCK_TYPE_HUFFS_BASE 673792u #define WUFFS_BROTLI__BLOCK_TYPE_HUFFS_STRIDE 632u -#define WUFFS_BROTLI__BLOCK_LEN_HUFFS_BASE 669544u +#define WUFFS_BROTLI__BLOCK_LEN_HUFFS_BASE 675688u #define WUFFS_BROTLI__BLOCK_LEN_HUFFS_STRIDE 396u -#define WUFFS_BROTLI__CONTEXT_MAP_HUFFS_BASE 670732u +#define WUFFS_BROTLI__CONTEXT_MAP_HUFFS_BASE 676876u -#define WUFFS_BROTLI__CODE_LENGTH_HUFFS_BASE 671378u +#define WUFFS_BROTLI__CODE_LENGTH_HUFFS_BASE 677522u -#define WUFFS_BROTLI__CODE_LENGTH_CODE_HUFFS_BASE 671634u +#define WUFFS_BROTLI__CODE_LENGTH_CODE_HUFFS_BASE 677778u -#define WUFFS_BROTLI__HUFFS_LEN 671890u +#define WUFFS_BROTLI__HUFFS_LEN 678034u #define WUFFS_BROTLI__DIST_CONTEXT_MAP_BASE 16384u +#define WUFFS_BROTLI__QUIRKS_BASE 716673024u + // ---------------- Private Initializer Prototypes // ---------------- Private Function Prototypes @@ -56566,7 +56646,8 @@ static wuffs_base__status wuffs_brotli__decoder__do_transform_io( wuffs_brotli__decoder* self, wuffs_base__io_buffer* a_src, - wuffs_base__slice_u8 a_workbuf); + wuffs_base__slice_u8 a_workbuf, + wuffs_base__slice_u8 a_dictbuf); WUFFS_BASE__GENERATED_C_CODE static wuffs_base__empty_struct @@ -56585,6 +56666,13 @@ wuffs_brotli__decoder__read_bits( wuffs_base__io_buffer* a_src, uint32_t a_n); +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__decoder__read_bits32( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + uint32_t a_n); + WUFFS_BASE__GENERATED_C_CODE static wuffs_base__status wuffs_brotli__decoder__read_symbol( @@ -56646,6 +56734,116 @@ wuffs_brotli__decoder__copy_uncompressed( wuffs_base__io_buffer* a_src, wuffs_base__slice_u8 a_workbuf); +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__decoder__add_chunk( + wuffs_brotli__decoder* self, + uint64_t a_start, + uint64_t a_size); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__decoder__parse_shared_dictionary( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_dict, + uint64_t a_base_offset); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__decoder__read_varint32( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_dict, + uint64_t a_pos); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__decoder__parse_word_list( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_dict, + uint64_t a_pos, + uint64_t a_base_offset, + uint32_t a_list); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__decoder__parse_transforms_list( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_dict, + uint64_t a_pos, + uint64_t a_base_offset, + uint32_t a_list); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__copy_dictionary_word( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_dictbuf, + uint32_t a_address, + uint32_t a_len, + uint32_t a_ctx); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__word_list_size_bits( + const wuffs_brotli__decoder* self, + uint32_t a_wsel, + uint32_t a_len); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__word_list_offset( + const wuffs_brotli__decoder* self, + uint32_t a_wsel, + uint32_t a_len); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__transform_list_length( + const wuffs_brotli__decoder* self, + uint32_t a_tsel); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__transform_dictionary_word( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_dictbuf, + uint32_t a_wsel, + uint32_t a_tsel, + uint32_t a_offset, + uint32_t a_len, + uint32_t a_transform_idx); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__append_affix( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_dictbuf, + uint32_t a_tsel, + uint32_t a_id, + uint32_t a_idx); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__shift( + wuffs_brotli__decoder* self, + uint32_t a_p, + uint32_t a_word_len, + uint32_t a_param); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__to_upper_case( + wuffs_brotli__decoder* self, + uint32_t a_p, + uint32_t a_end); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__byte_at( + const wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_s, + uint64_t a_i); + WUFFS_BASE__GENERATED_C_CODE static wuffs_base__status wuffs_brotli__decoder__read_huffman_code( @@ -56704,7 +56902,15 @@ static wuffs_base__status wuffs_brotli__decoder__decode_compressed_metablock( wuffs_brotli__decoder* self, wuffs_base__io_buffer* a_src, - wuffs_base__slice_u8 a_workbuf); + wuffs_base__slice_u8 a_workbuf, + wuffs_base__slice_u8 a_dictbuf); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__calculate_distance_code_limit( + const wuffs_brotli__decoder* self, + uint32_t a_npostfix, + uint32_t a_ndirect); WUFFS_BASE__GENERATED_C_CODE static wuffs_base__status @@ -56750,7 +56956,8 @@ static wuffs_base__status wuffs_brotli__decoder__process_commands( wuffs_brotli__decoder* self, wuffs_base__io_buffer* a_src, - wuffs_base__slice_u8 a_workbuf); + wuffs_base__slice_u8 a_workbuf, + wuffs_base__slice_u8 a_dictbuf); WUFFS_BASE__GENERATED_C_CODE static wuffs_base__empty_struct @@ -56769,20 +56976,6 @@ wuffs_brotli__decoder__take_distance_from_ring_buffer( wuffs_brotli__decoder* self, uint32_t a_code); -WUFFS_BASE__GENERATED_C_CODE -static uint32_t -wuffs_brotli__decoder__copy_dictionary_word( - wuffs_brotli__decoder* self, - uint32_t a_address, - uint32_t a_len); - -WUFFS_BASE__GENERATED_C_CODE -static uint32_t -wuffs_brotli__decoder__to_upper_case( - wuffs_brotli__decoder* self, - uint32_t a_p, - uint32_t a_end); - // ---------------- VTables const wuffs_base__io_transformer__func_ptrs @@ -56886,6 +57079,9 @@ wuffs_brotli__decoder__get_quirk( return 0; } + if ((a_key == 716673024u) && self->private_impl.f_quirk_large_window) { + return 1u; + } return 0u; } @@ -56907,6 +57103,13 @@ wuffs_brotli__decoder__set_quirk( : wuffs_base__error__initialize_not_called); } + if (a_key == 716673024u) { + if (self->private_impl.f_started) { + return wuffs_base__make_status(wuffs_base__error__bad_call_sequence); + } + self->private_impl.f_quirk_large_window = (a_value > 0u); + return wuffs_base__make_status(NULL); + } return wuffs_base__make_status(wuffs_base__error__unsupported_option); } @@ -56941,7 +57144,13 @@ wuffs_brotli__decoder__workbuf_len( return wuffs_base__utility__empty_range_ii_u64(); } - return wuffs_base__utility__make_range_ii_u64(16777758u, 16777758u); + uint64_t v_n = 0; + + v_n = 16777758u; + if (self->private_impl.f_quirk_large_window) { + v_n = 1073742366u; + } + return wuffs_base__utility__make_range_ii_u64((v_n + self->private_impl.f_dict_total), (v_n + self->private_impl.f_dict_total)); } // -------- func brotli.decoder.transform_io @@ -56981,16 +57190,28 @@ wuffs_brotli__decoder__transform_io( WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; while (true) { + if (self->private_impl.f_dict_total > ((uint64_t)(a_workbuf.len))) { + status = wuffs_base__make_status(wuffs_base__error__bad_workbuf_length); + goto exit; + } { - wuffs_base__status t_0 = wuffs_brotli__decoder__do_transform_io(self, a_src, a_workbuf); + wuffs_base__status t_0 = wuffs_brotli__decoder__do_transform_io(self, a_src, wuffs_base__slice_u8__subslice_i(a_workbuf, self->private_impl.f_dict_total), wuffs_base__slice_u8__subslice_j(a_workbuf, self->private_impl.f_dict_total)); v_status = t_0; } - wuffs_brotli__decoder__flush(self, a_dst, a_workbuf); + if (self->private_impl.f_dict_total > ((uint64_t)(a_workbuf.len))) { + status = wuffs_base__make_status(wuffs_base__error__bad_workbuf_length); + goto exit; + } + wuffs_brotli__decoder__flush(self, a_dst, wuffs_base__slice_u8__subslice_i(a_workbuf, self->private_impl.f_dict_total)); if (wuffs_base__status__is_ok(&v_status)) { while ( ! wuffs_brotli__decoder__is_fully_flushed(self)) { status = wuffs_base__make_status(wuffs_base__suspension__short_write); WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(1); - wuffs_brotli__decoder__flush(self, a_dst, a_workbuf); + if (self->private_impl.f_dict_total > ((uint64_t)(a_workbuf.len))) { + status = wuffs_base__make_status(wuffs_base__error__bad_workbuf_length); + goto exit; + } + wuffs_brotli__decoder__flush(self, a_dst, wuffs_base__slice_u8__subslice_i(a_workbuf, self->private_impl.f_dict_total)); } status = wuffs_base__make_status(NULL); goto ok; @@ -57093,7 +57314,8 @@ static wuffs_base__status wuffs_brotli__decoder__do_transform_io( wuffs_brotli__decoder* self, wuffs_base__io_buffer* a_src, - wuffs_base__slice_u8 a_workbuf) { + wuffs_base__slice_u8 a_workbuf, + wuffs_base__slice_u8 a_dictbuf) { wuffs_base__status status = wuffs_base__make_status(NULL); uint64_t v_w = 0; @@ -57188,7 +57410,7 @@ wuffs_brotli__decoder__do_transform_io( a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); } WUFFS_BASE__COROUTINE_SUSPENSION_POINT(5); - status = wuffs_brotli__decoder__decode_compressed_metablock(self, a_src, a_workbuf); + status = wuffs_brotli__decoder__decode_compressed_metablock(self, a_src, a_workbuf, a_dictbuf); if (a_src) { iop_a_src = a_src->data.ptr + a_src->meta.ri; } @@ -57241,6 +57463,7 @@ wuffs_brotli__decoder__initialize_codes( self->private_impl.f_bits = 0u; self->private_impl.f_n_bits = 0u; + self->private_impl.f_large_window = false; self->private_impl.f_pos = 0u; self->private_impl.f_rb_size = 0u; self->private_impl.f_rb_flushed = 0u; @@ -57253,7 +57476,7 @@ wuffs_brotli__decoder__initialize_codes( self->private_impl.f_dist_rb_idx = 0u; v_i = 0u; while (v_i < 256u) { - self->private_data.f_huffs[(671634u + v_i)] = ((uint16_t)(((uint16_t)(((uint16_t)(WUFFS_BROTLI__CODE_LENGTH_PREFIX_VALUE[(v_i & 15u)])) << 4u)) | ((uint16_t)(WUFFS_BROTLI__CODE_LENGTH_PREFIX_LENGTH[(v_i & 15u)])))); + self->private_data.f_huffs[(677778u + v_i)] = ((uint16_t)(((uint16_t)(((uint16_t)(WUFFS_BROTLI__CODE_LENGTH_PREFIX_VALUE[(v_i & 15u)])) << 4u)) | ((uint16_t)(WUFFS_BROTLI__CODE_LENGTH_PREFIX_LENGTH[(v_i & 15u)])))); v_i += 1u; } return wuffs_base__make_empty_struct(); @@ -57363,6 +57586,66 @@ wuffs_brotli__decoder__read_bits( return status; } +// -------- func brotli.decoder.read_bits32 + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__decoder__read_bits32( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + uint32_t a_n) { + wuffs_base__status status = wuffs_base__make_status(NULL); + + uint32_t v_n = 0; + uint32_t v_lo = 0; + + uint32_t coro_susp_point = self->private_impl.p_read_bits32; + if (coro_susp_point) { + v_n = self->private_data.s_read_bits32.v_n; + v_lo = self->private_data.s_read_bits32.v_lo; + } + switch (coro_susp_point) { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; + + v_n = a_n; + if (v_n <= 24u) { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); + status = wuffs_brotli__decoder__read_bits(self, a_src, v_n); + if (status.repr) { + goto suspend; + } + status = wuffs_base__make_status(NULL); + goto ok; + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); + status = wuffs_brotli__decoder__read_bits(self, a_src, 16u); + if (status.repr) { + goto suspend; + } + v_lo = (self->private_impl.f_bits_value & 65535u); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(3); + status = wuffs_brotli__decoder__read_bits(self, a_src, (v_n - 16u)); + if (status.repr) { + goto suspend; + } + self->private_impl.f_bits_value = (v_lo | ((uint32_t)(self->private_impl.f_bits_value << 16u))); + + ok: + self->private_impl.p_read_bits32 = 0; + goto exit; + } + + goto suspend; + suspend: + self->private_impl.p_read_bits32 = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; + self->private_data.s_read_bits32.v_n = v_n; + self->private_data.s_read_bits32.v_lo = v_lo; + + goto exit; + exit: + return status; +} + // -------- func brotli.decoder.read_symbol WUFFS_BASE__GENERATED_C_CODE @@ -57406,7 +57689,7 @@ wuffs_brotli__decoder__read_symbol( v_n_bits = self->private_impl.f_n_bits; while (true) { v_i = ((uint32_t)(v_hoff + (v_bits & 255u))); - if (v_i >= 671890u) { + if (v_i >= 678034u) { status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); goto exit; } @@ -57421,7 +57704,7 @@ wuffs_brotli__decoder__read_symbol( } } else if (v_n_bits > 8u) { v_i = ((uint32_t)(((uint32_t)(v_hoff + (v_e >> 4u))) + ((v_bits >> 8u) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u)))); - if (v_i >= 671890u) { + if (v_i >= 678034u) { status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); goto exit; } @@ -57501,6 +57784,7 @@ wuffs_brotli__decoder__decode_window_bits( wuffs_base__status status = wuffs_base__make_status(NULL); uint32_t v_n = 0; + uint32_t v_w = 0; uint32_t coro_susp_point = self->private_impl.p_decode_window_bits; switch (coro_susp_point) { @@ -57530,8 +57814,36 @@ wuffs_brotli__decoder__decode_window_bits( } v_n = (self->private_impl.f_bits_value & 7u); if (v_n == 1u) { - status = wuffs_base__make_status(wuffs_brotli__error__unsupported_large_window); - goto exit; + if ( ! self->private_impl.f_quirk_large_window) { + status = wuffs_base__make_status(wuffs_brotli__error__unsupported_large_window); + goto exit; + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(4); + status = wuffs_brotli__decoder__read_bits(self, a_src, 1u); + if (status.repr) { + goto suspend; + } + if (self->private_impl.f_bits_value == 1u) { + status = wuffs_base__make_status(wuffs_brotli__error__bad_window_bits); + goto exit; + } + self->private_impl.f_started = true; + self->private_impl.f_large_window = true; + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(5); + status = wuffs_brotli__decoder__read_bits(self, a_src, 6u); + if (status.repr) { + goto suspend; + } + v_w = (self->private_impl.f_bits_value & 63u); + if ((v_w < 10u) || (30u < v_w)) { + status = wuffs_base__make_status(wuffs_brotli__error__bad_window_bits); + goto exit; + } + self->private_impl.f_window_bits = v_w; + self->private_impl.f_window_size = (((uint32_t)(1u)) << v_w); + self->private_impl.f_max_backward_distance = (self->private_impl.f_window_size - 16u); + status = wuffs_base__make_status(NULL); + goto ok; } else if (v_n != 0u) { self->private_impl.f_window_bits = (8u + v_n); } else { @@ -57543,10 +57855,10 @@ wuffs_brotli__decoder__decode_window_bits( status = wuffs_base__make_status(wuffs_brotli__error__bad_window_bits); goto exit; } + self->private_impl.f_started = true; self->private_impl.f_window_size = (((uint32_t)(1u)) << self->private_impl.f_window_bits); self->private_impl.f_max_backward_distance = (self->private_impl.f_window_size - 16u); - goto ok; ok: self->private_impl.p_decode_window_bits = 0; goto exit; @@ -57933,6 +58245,924 @@ wuffs_brotli__decoder__copy_uncompressed( return status; } +// -------- func brotli.decoder.add_dictionary + +WUFFS_BASE__GENERATED_C_CODE +WUFFS_BASE__MAYBE_STATIC wuffs_base__status +wuffs_brotli__decoder__add_dictionary( + wuffs_brotli__decoder* self, + uint32_t a_dict_type, + wuffs_base__slice_u8 a_dict, + wuffs_base__slice_u8 a_workbuf) { + if (!self) { + return wuffs_base__make_status(wuffs_base__error__bad_receiver); + } + if (self->private_impl.magic != WUFFS_BASE__MAGIC) { + return wuffs_base__make_status( + (self->private_impl.magic == WUFFS_BASE__DISABLED) + ? wuffs_base__error__disabled_by_previous_error + : wuffs_base__error__initialize_not_called); + } + + wuffs_base__status v_status = wuffs_base__make_status(NULL); + uint64_t v_n = 0; + uint64_t v_lo = 0; + uint64_t v_hi = 0; + uint32_t v_i = 0; + + if (self->private_impl.f_started) { + return wuffs_base__make_status(wuffs_base__error__bad_call_sequence); + } + v_n = ((uint64_t)(a_dict.len)); + if (a_dict_type == 0u) { + if (self->private_impl.f_num_chunks >= 15u) { + return wuffs_base__make_status(wuffs_brotli__error__too_many_compound_dictionaries); + } + } else if (a_dict_type == 1u) { + v_status = wuffs_brotli__decoder__parse_shared_dictionary(self, a_dict, self->private_impl.f_dict_total); + if ( ! wuffs_base__status__is_ok(&v_status)) { + return wuffs_private_impl__status__ensure_not_a_suspension(v_status); + } + } else { + return wuffs_base__make_status(wuffs_base__error__bad_argument); + } + if (v_n > 2147483647u) { + return wuffs_base__make_status(wuffs_brotli__error__too_much_dictionary_data); + } + v_lo = self->private_impl.f_dict_total; + v_hi = (v_lo + v_n); + if (v_hi > 2147483647u) { + return wuffs_base__make_status(wuffs_brotli__error__too_much_dictionary_data); + } else if (v_hi > ((uint64_t)(a_workbuf.len))) { + return wuffs_base__make_status(wuffs_base__error__bad_workbuf_length); + } + if (v_lo > v_hi) { + return wuffs_base__make_status(wuffs_brotli__error__too_much_dictionary_data); + } + wuffs_private_impl__slice_u8__copy_from_slice(wuffs_base__slice_u8__subslice_ij(a_workbuf, v_lo, v_hi), a_dict); + if (a_dict_type == 0u) { + wuffs_brotli__decoder__add_chunk(self, v_lo, v_n); + } else { + if (self->private_impl.f_sd_prefix_size > 0u) { + wuffs_brotli__decoder__add_chunk(self, wuffs_base__u64__sat_add(v_lo, self->private_impl.f_sd_prefix_pos), self->private_impl.f_sd_prefix_size); + } + self->private_impl.f_sd_num_word_lists = self->private_impl.f_stage_num_word_lists; + self->private_impl.f_sd_num_transform_lists = self->private_impl.f_stage_num_transform_lists; + if ((self->private_impl.f_stage_num_word_lists > 0u) || (self->private_impl.f_stage_num_transform_lists > 0u)) { + self->private_impl.f_sd_context_based = self->private_impl.f_stage_context_based; + self->private_impl.f_sd_num_dictionaries = self->private_impl.f_stage_num_dictionaries; + v_i = 0u; + while (v_i < 64u) { + self->private_impl.f_sd_words[v_i] = wuffs_base__u8__min(self->private_data.f_stage_words[v_i], 64u); + self->private_impl.f_sd_transforms[v_i] = wuffs_base__u8__min(self->private_data.f_stage_transforms[v_i], 64u); + self->private_impl.f_sd_context_map[v_i] = ((uint8_t)(self->private_data.f_stage_context_map[v_i] & 63u)); + v_i += 1u; + } + } else { + self->private_impl.f_sd_context_based = false; + self->private_impl.f_sd_num_dictionaries = 1u; + self->private_impl.f_sd_words[0u] = 0u; + self->private_impl.f_sd_transforms[0u] = 0u; + } + } + if (v_hi <= 2147483647u) { + self->private_impl.f_dict_total = v_hi; + } + return wuffs_base__make_status(NULL); +} + +// -------- func brotli.decoder.add_chunk + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__decoder__add_chunk( + wuffs_brotli__decoder* self, + uint64_t a_start, + uint64_t a_size) { + uint32_t v_k = 0; + + v_k = self->private_impl.f_num_chunks; + if (v_k >= 15u) { + return wuffs_base__make_empty_struct(); + } + self->private_impl.f_chunk_starts[v_k] = ((uint32_t)(a_start)); + self->private_impl.f_chunk_offsets[(v_k + 1u)] = ((uint32_t)(self->private_impl.f_chunk_offsets[v_k] + ((uint32_t)(a_size)))); + self->private_impl.f_num_chunks = (v_k + 1u); + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.decoder.parse_shared_dictionary + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__decoder__parse_shared_dictionary( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_dict, + uint64_t a_base_offset) { + uint64_t v_pos = 0; + uint64_t v_chunk_size = 0; + uint64_t v_prefix_pos = 0; + uint32_t v_num_prefix = 0; + uint32_t v_nwl = 0; + uint32_t v_ntl = 0; + uint32_t v_ndict = 0; + bool v_is_custom = false; + bool v_has_custom = false; + uint32_t v_i = 0; + uint32_t v_v = 0; + uint32_t v_v2 = 0; + wuffs_base__status v_status = wuffs_base__make_status(NULL); + + if (((uint64_t)(a_dict.len)) < 2u) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } else if ((a_dict.ptr[0u] != 145u) || (a_dict.ptr[1u] != 0u)) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_pos = 2u; + v_chunk_size = wuffs_brotli__decoder__read_varint32(self, a_dict, v_pos); + if (v_chunk_size >= 4294967296u) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_pos = self->private_impl.f_parse_pos; + v_prefix_pos = v_pos; + v_num_prefix = 0u; + if (v_chunk_size != 0u) { + if (v_chunk_size > 1073741823u) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_num_prefix = 1u; + if (wuffs_base__u64__sat_add(v_pos, v_chunk_size) > ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + wuffs_private_impl__u64__sat_add_indirect(&v_pos, v_chunk_size); + } + if (wuffs_base__u64__sat_add(v_pos, 2u) > ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_is_custom = false; + if (v_pos < ((uint64_t)(a_dict.len))) { + if (a_dict.ptr[v_pos] > 0u) { + v_is_custom = true; + } + } + if (wuffs_base__u64__sat_add(v_pos, 1u) < ((uint64_t)(a_dict.len))) { + if (a_dict.ptr[wuffs_base__u64__sat_add(v_pos, 1u)] > 0u) { + v_is_custom = true; + } + } + if ((v_num_prefix + self->private_impl.f_num_chunks) > 15u) { + return wuffs_base__make_status(wuffs_brotli__error__too_many_compound_dictionaries); + } + v_has_custom = ((self->private_impl.f_sd_num_word_lists > 0u) || (self->private_impl.f_sd_num_transform_lists > 0u)); + if (v_has_custom && v_is_custom) { + return wuffs_base__make_status(wuffs_brotli__error__unsupported_multiple_custom_static_dictionaries); + } + if (v_pos >= ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_nwl = ((uint32_t)(a_dict.ptr[v_pos])); + wuffs_private_impl__u64__sat_add_indirect(&v_pos, 1u); + if (v_nwl > 64u) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_i = 0u; + while (v_i < v_nwl) { + v_status = wuffs_brotli__decoder__parse_word_list(self, + a_dict, + v_pos, + a_base_offset, + v_i); + if ( ! wuffs_base__status__is_ok(&v_status)) { + return wuffs_private_impl__status__ensure_not_a_suspension(v_status); + } + v_pos = self->private_impl.f_parse_pos; + v_i += 1u; + } + if (v_pos >= ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_ntl = ((uint32_t)(a_dict.ptr[v_pos])); + wuffs_private_impl__u64__sat_add_indirect(&v_pos, 1u); + if (v_ntl > 64u) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_i = 0u; + while (v_i < v_ntl) { + v_status = wuffs_brotli__decoder__parse_transforms_list(self, + a_dict, + v_pos, + a_base_offset, + v_i); + if ( ! wuffs_base__status__is_ok(&v_status)) { + return wuffs_private_impl__status__ensure_not_a_suspension(v_status); + } + v_pos = self->private_impl.f_parse_pos; + v_i += 1u; + } + self->private_impl.f_stage_num_word_lists = wuffs_base__u32__min(v_nwl, 64u); + self->private_impl.f_stage_num_transform_lists = wuffs_base__u32__min(v_ntl, 64u); + self->private_impl.f_stage_num_dictionaries = 1u; + self->private_impl.f_stage_context_based = false; + v_i = 0u; + while (v_i < 64u) { + self->private_data.f_stage_words[v_i] = 0u; + self->private_data.f_stage_transforms[v_i] = 0u; + self->private_data.f_stage_context_map[v_i] = 0u; + v_i += 1u; + } + if ((v_nwl != 0u) || (v_ntl != 0u)) { + if (v_pos >= ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_ndict = ((uint32_t)(a_dict.ptr[v_pos])); + wuffs_private_impl__u64__sat_add_indirect(&v_pos, 1u); + if ((v_ndict == 0u) || (v_ndict > 64u)) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + self->private_impl.f_stage_num_dictionaries = wuffs_base__u32__min(v_ndict, 64u); + v_i = 0u; + while (v_i < v_ndict) { + if (v_pos >= ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_v = ((uint32_t)(a_dict.ptr[v_pos])); + wuffs_private_impl__u64__sat_add_indirect(&v_pos, 1u); + if (v_v > v_nwl) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + if (v_pos >= ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_v2 = ((uint32_t)(a_dict.ptr[v_pos])); + wuffs_private_impl__u64__sat_add_indirect(&v_pos, 1u); + if (v_v2 > v_ntl) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + if (v_v == v_nwl) { + self->private_data.f_stage_words[v_i] = 0u; + } else { + self->private_data.f_stage_words[v_i] = ((uint8_t)(((v_v & 63u) + 1u))); + } + if (v_v2 == v_ntl) { + self->private_data.f_stage_transforms[v_i] = 0u; + } else { + self->private_data.f_stage_transforms[v_i] = ((uint8_t)(((v_v2 & 63u) + 1u))); + } + v_i += 1u; + } + if (v_pos >= ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_v = ((uint32_t)(a_dict.ptr[v_pos])); + wuffs_private_impl__u64__sat_add_indirect(&v_pos, 1u); + if (v_v > 1u) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + self->private_impl.f_stage_context_based = (v_v == 1u); + if (self->private_impl.f_stage_context_based) { + v_i = 0u; + while (v_i < 64u) { + if (v_pos >= ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_v = ((uint32_t)(a_dict.ptr[v_pos])); + wuffs_private_impl__u64__sat_add_indirect(&v_pos, 1u); + if (v_v >= v_ndict) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + self->private_data.f_stage_context_map[v_i] = ((uint8_t)((v_v & 63u))); + v_i += 1u; + } + } + } + self->private_impl.f_sd_prefix_pos = v_prefix_pos; + self->private_impl.f_sd_prefix_size = v_chunk_size; + return wuffs_base__make_status(NULL); +} + +// -------- func brotli.decoder.read_varint32 + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__decoder__read_varint32( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_dict, + uint64_t a_pos) { + uint64_t v_pos = 0; + uint32_t v_num = 0; + uint64_t v_b = 0; + uint64_t v_result = 0; + + v_pos = a_pos; + v_result = 0u; + v_num = 0u; + while (true) { + if (v_pos >= ((uint64_t)(a_dict.len))) { + return 4294967296u; + } + v_b = ((uint64_t)(a_dict.ptr[v_pos])); + wuffs_private_impl__u64__sat_add_indirect(&v_pos, 1u); + if ((v_num == 4u) && (v_b > 15u)) { + return 4294967296u; + } + v_result |= ((v_b & 127u) << (7u * v_num)); + if (v_b < 128u) { + break; + } + if (v_num >= 4u) { + return 4294967296u; + } + v_num += 1u; + } + self->private_impl.f_parse_pos = v_pos; + return (v_result & 4294967295u); +} + +// -------- func brotli.decoder.parse_word_list + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__decoder__parse_word_list( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_dict, + uint64_t a_pos, + uint64_t a_base_offset, + uint32_t a_list) { + uint64_t v_pos = 0; + uint32_t v_len = 0; + uint8_t v_v = 0; + uint32_t v_offset = 0; + uint64_t v_wpos = 0; + + v_pos = a_pos; + if (wuffs_base__u64__sat_add(v_pos, 28u) > ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_len = 0u; + while (v_len < 32u) { + v_v = 0u; + if (v_len >= 4u) { + if (wuffs_base__u64__sat_add(v_pos, ((uint64_t)((v_len - 4u)))) < ((uint64_t)(a_dict.len))) { + v_v = a_dict.ptr[wuffs_base__u64__sat_add(v_pos, ((uint64_t)((v_len - 4u))))]; + } + if (v_v > 15u) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + } + self->private_data.f_wl_size_bits[a_list][v_len] = v_v; + v_len += 1u; + } + wuffs_private_impl__u64__sat_add_indirect(&v_pos, 28u); + v_offset = 0u; + v_len = 0u; + while (v_len < 32u) { + self->private_data.f_wl_offsets[a_list][v_len] = v_offset; + v_v = self->private_data.f_wl_size_bits[a_list][v_len]; + if (v_v != 0u) { + v_offset += (v_len << ((uint8_t)(v_v & 15u))); + } + v_len += 1u; + } + v_wpos = wuffs_base__u64__sat_add(a_base_offset, v_pos); + self->private_data.f_wl_data[a_list] = ((uint32_t)(v_wpos)); + wuffs_private_impl__u64__sat_add_indirect(&v_pos, ((uint64_t)(v_offset))); + if (v_pos > ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + self->private_impl.f_parse_pos = v_pos; + return wuffs_base__make_status(NULL); +} + +// -------- func brotli.decoder.parse_transforms_list + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__decoder__parse_transforms_list( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_dict, + uint64_t a_pos, + uint64_t a_base_offset, + uint32_t a_list) { + uint64_t v_pos = 0; + uint64_t v_data_length = 0; + uint64_t v_ps_start = 0; + uint64_t v_offset = 0; + uint64_t v_stringlet_len = 0; + uint32_t v_count = 0; + uint32_t v_nt = 0; + uint32_t v_t = 0; + uint64_t v_tpos = 0; + uint64_t v_j = 0; + uint32_t v_prefix_id = 0; + uint32_t v_t_type = 0; + uint32_t v_suffix_id = 0; + bool v_has_params = false; + uint32_t v_p0 = 0; + uint32_t v_p1 = 0; + + v_pos = a_pos; + if (v_pos >= ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + if (wuffs_base__u64__sat_add(v_pos, 2u) > ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_data_length = (((uint64_t)(wuffs_brotli__decoder__byte_at(self, a_dict, v_pos))) | (((uint64_t)(wuffs_brotli__decoder__byte_at(self, a_dict, wuffs_base__u64__sat_add(v_pos, 1u)))) << 8u)); + wuffs_private_impl__u64__sat_add_indirect(&v_pos, 2u); + if (v_data_length < 1u) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_ps_start = v_pos; + if (wuffs_base__u64__sat_add(v_pos, v_data_length) >= ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_offset = 0u; + v_count = 0u; + while (true) { + v_j = wuffs_base__u64__sat_add(v_ps_start, v_offset); + v_stringlet_len = 0u; + if (v_j < ((uint64_t)(a_dict.len))) { + v_stringlet_len = ((uint64_t)(a_dict.ptr[v_j])); + } + if (v_count >= 256u) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + self->private_data.f_tl_map[a_list][v_count] = ((uint16_t)(v_offset)); + v_count += 1u; + wuffs_private_impl__u64__sat_add_indirect(&v_offset, 1u); + if (v_stringlet_len == 0u) { + if (v_offset == v_data_length) { + break; + } + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + if (v_count > 255u) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + wuffs_private_impl__u64__sat_add_indirect(&v_offset, v_stringlet_len); + if (v_offset >= v_data_length) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + } + wuffs_private_impl__u64__sat_add_indirect(&v_pos, v_data_length); + self->private_data.f_tl_prefix_suffix[a_list] = ((uint32_t)(wuffs_base__u64__sat_add(a_base_offset, v_ps_start))); + v_nt = 0u; + if (v_pos < ((uint64_t)(a_dict.len))) { + v_nt = ((uint32_t)(a_dict.ptr[v_pos])); + } + wuffs_private_impl__u64__sat_add_indirect(&v_pos, 1u); + self->private_data.f_tl_num_transforms[a_list] = v_nt; + v_tpos = v_pos; + self->private_data.f_tl_transforms[a_list] = ((uint32_t)(wuffs_base__u64__sat_add(a_base_offset, v_tpos))); + wuffs_private_impl__u64__sat_add_indirect(&v_pos, (((uint64_t)(v_nt)) * 3u)); + if (v_pos > ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_has_params = false; + v_t = 0u; + while (v_t < v_nt) { + v_j = wuffs_base__u64__sat_add(v_tpos, (((uint64_t)(v_t)) * 3u)); + v_prefix_id = wuffs_brotli__decoder__byte_at(self, a_dict, v_j); + v_t_type = wuffs_brotli__decoder__byte_at(self, a_dict, wuffs_base__u64__sat_add(v_j, 1u)); + v_suffix_id = wuffs_brotli__decoder__byte_at(self, a_dict, wuffs_base__u64__sat_add(v_j, 2u)); + if (v_prefix_id >= v_count) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } else if (v_t_type >= 23u) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } else if (v_suffix_id >= v_count) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + if ((v_t_type == 21u) || (v_t_type == 22u)) { + v_has_params = true; + } + v_t += 1u; + } + self->private_data.f_tl_params[a_list] = 0u; + if (v_has_params) { + self->private_data.f_tl_params[a_list] = ((uint32_t)(wuffs_base__u64__sat_add(a_base_offset, v_pos))); + v_j = v_pos; + wuffs_private_impl__u64__sat_add_indirect(&v_pos, (((uint64_t)(v_nt)) * 2u)); + if (v_pos > ((uint64_t)(a_dict.len))) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + v_t = 0u; + while (v_t < v_nt) { + v_t_type = wuffs_brotli__decoder__byte_at(self, a_dict, wuffs_base__u64__sat_add(v_tpos, ((((uint64_t)(v_t)) * 3u) + 1u))); + if ((v_t_type != 21u) && (v_t_type != 22u)) { + v_p0 = wuffs_brotli__decoder__byte_at(self, a_dict, wuffs_base__u64__sat_add(v_j, (((uint64_t)(v_t)) * 2u))); + v_p1 = wuffs_brotli__decoder__byte_at(self, a_dict, wuffs_base__u64__sat_add(v_j, ((((uint64_t)(v_t)) * 2u) + 1u))); + if ((v_p0 != 0u) || (v_p1 != 0u)) { + return wuffs_base__make_status(wuffs_brotli__error__bad_shared_dictionary); + } + } + v_t += 1u; + } + } + self->private_impl.f_parse_pos = v_pos; + return wuffs_base__make_status(NULL); +} + +// -------- func brotli.decoder.copy_dictionary_word + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__copy_dictionary_word( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_dictbuf, + uint32_t a_address, + uint32_t a_len, + uint32_t a_ctx) { + uint32_t v_dict_id = 0; + uint32_t v_wsel = 0; + uint32_t v_tsel = 0; + uint32_t v_shift = 0; + uint32_t v_nt = 0; + uint32_t v_address = 0; + uint32_t v_transform_idx = 0; + uint32_t v_word_idx = 0; + uint32_t v_offset = 0; + uint32_t v_dist_remaining = 0; + uint32_t v_d2 = 0; + uint32_t v_w2 = 0; + uint32_t v_t2 = 0; + uint32_t v_shift2 = 0; + uint32_t v_num = 0; + + v_dict_id = 0u; + if (self->private_impl.f_sd_context_based) { + v_dict_id = ((uint32_t)(((uint8_t)(self->private_impl.f_sd_context_map[a_ctx] & 63u)))); + } + v_wsel = ((uint32_t)(self->private_impl.f_sd_words[v_dict_id])); + v_tsel = ((uint32_t)(self->private_impl.f_sd_transforms[v_dict_id])); + v_shift = wuffs_brotli__decoder__word_list_size_bits(self, v_wsel, a_len); + v_nt = wuffs_brotli__decoder__transform_list_length(self, v_tsel); + v_address = a_address; + v_transform_idx = (v_address >> v_shift); + if (((v_transform_idx >= v_nt) || (v_shift == 0u)) && (self->private_impl.f_sd_num_dictionaries > 1u)) { + v_dist_remaining = ((uint32_t)(v_address - ((uint32_t)(((((uint32_t)(1u)) << v_shift) & 4294967294u) * v_nt)))); + v_d2 = 0u; + while (v_d2 < self->private_impl.f_sd_num_dictionaries) { + v_w2 = ((uint32_t)(self->private_impl.f_sd_words[v_d2])); + v_shift2 = wuffs_brotli__decoder__word_list_size_bits(self, v_w2, a_len); + if ((v_d2 != v_dict_id) && (v_shift2 != 0u)) { + v_t2 = ((uint32_t)(self->private_impl.f_sd_transforms[v_d2])); + v_num = ((uint32_t)(((((uint32_t)(1u)) << v_shift2) & 4294967294u) * wuffs_brotli__decoder__transform_list_length(self, v_t2))); + if (v_dist_remaining < v_num) { + v_wsel = v_w2; + v_tsel = v_t2; + v_address = v_dist_remaining; + v_shift = v_shift2; + v_nt = wuffs_brotli__decoder__transform_list_length(self, v_tsel); + v_transform_idx = (v_address >> v_shift); + break; + } + v_dist_remaining -= v_num; + } + v_d2 += 1u; + } + } + if (v_shift == 0u) { + return 4294967294u; + } else if (v_transform_idx >= v_nt) { + return 4294967295u; + } + v_word_idx = (v_address & ((((uint32_t)(1u)) << v_shift) - 1u)); + v_offset = ((uint32_t)(wuffs_brotli__decoder__word_list_offset(self, v_wsel, a_len) + ((uint32_t)(v_word_idx * a_len)))); + v_num = wuffs_brotli__decoder__transform_dictionary_word(self, + a_dictbuf, + v_wsel, + v_tsel, + v_offset, + a_len, + wuffs_base__u32__min(v_transform_idx, 254u)); + return v_num; +} + +// -------- func brotli.decoder.word_list_size_bits + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__word_list_size_bits( + const wuffs_brotli__decoder* self, + uint32_t a_wsel, + uint32_t a_len) { + if (a_wsel == 0u) { + return WUFFS_BROTLI__DICTIONARY_SIZE_BITS_BY_LENGTH[a_len]; + } + return ((uint32_t)(((uint8_t)(self->private_data.f_wl_size_bits[((a_wsel - 1u) & 63u)][a_len] & 15u)))); +} + +// -------- func brotli.decoder.word_list_offset + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__word_list_offset( + const wuffs_brotli__decoder* self, + uint32_t a_wsel, + uint32_t a_len) { + if (a_wsel == 0u) { + return WUFFS_BROTLI__DICTIONARY_OFFSETS_BY_LENGTH[a_len]; + } + return self->private_data.f_wl_offsets[((a_wsel - 1u) & 63u)][a_len]; +} + +// -------- func brotli.decoder.transform_list_length + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__transform_list_length( + const wuffs_brotli__decoder* self, + uint32_t a_tsel) { + if (a_tsel == 0u) { + return 121u; + } + return wuffs_base__u32__min(self->private_data.f_tl_num_transforms[((a_tsel - 1u) & 63u)], 255u); +} + +// -------- func brotli.decoder.transform_dictionary_word + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__transform_dictionary_word( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_dictbuf, + uint32_t a_wsel, + uint32_t a_tsel, + uint32_t a_offset, + uint32_t a_len, + uint32_t a_transform_idx) { + uint32_t v_idx = 0; + uint32_t v_t = 0; + uint32_t v_k = 0; + uint64_t v_tpos = 0; + uint32_t v_prefix_id = 0; + uint32_t v_t_type = 0; + uint32_t v_suffix_id = 0; + uint32_t v_param = 0; + uint32_t v_word_start = 0; + uint32_t v_word_len = 0; + uint32_t v_skip = 0; + uint32_t v_i = 0; + uint64_t v_j = 0; + uint32_t v_remaining = 0; + uint32_t v_step = 0; + + v_k = 0u; + v_tpos = 0u; + if (a_tsel == 0u) { + v_t = wuffs_base__u32__min(a_transform_idx, 120u); + v_prefix_id = ((uint32_t)(WUFFS_BROTLI__TRANSFORM_PREFIX_ID[v_t])); + v_t_type = ((uint32_t)(WUFFS_BROTLI__TRANSFORM_TYPE[v_t])); + v_suffix_id = ((uint32_t)(WUFFS_BROTLI__TRANSFORM_SUFFIX_ID[v_t])); + } else { + v_k = ((a_tsel - 1u) & 63u); + v_tpos = (((uint64_t)(self->private_data.f_tl_transforms[v_k])) + (((uint64_t)(a_transform_idx)) * 3u)); + v_prefix_id = wuffs_brotli__decoder__byte_at(self, a_dictbuf, v_tpos); + v_t_type = wuffs_brotli__decoder__byte_at(self, a_dictbuf, (v_tpos + 1u)); + v_suffix_id = wuffs_brotli__decoder__byte_at(self, a_dictbuf, (v_tpos + 2u)); + } + v_idx = wuffs_brotli__decoder__append_affix(self, + a_dictbuf, + a_tsel, + v_prefix_id, + 0u); + v_word_len = a_len; + v_skip = 0u; + if (v_t_type <= 9u) { + if (v_t_type < a_len) { + v_word_len = ((uint32_t)(a_len - v_t_type)); + } else { + v_word_len = 0u; + } + } else if ((12u <= v_t_type) && (v_t_type <= 20u)) { + v_skip = (v_t_type - 11u); + if (v_skip < a_len) { + v_word_len = ((uint32_t)(a_len - v_skip)); + } else { + v_word_len = 0u; + } + } + v_word_start = v_idx; + v_i = 0u; + while (v_i < v_word_len) { + if (v_idx >= 1024u) { + break; + } + if (a_wsel == 0u) { + v_j = ((uint64_t)(((uint32_t)(((uint32_t)(a_offset + v_skip)) + v_i)))); + if (v_j < 122784u) { + self->private_data.f_transformed_word[v_idx] = WUFFS_BROTLI__DICTIONARY[v_j]; + } + } else { + v_j = (((uint64_t)(self->private_data.f_wl_data[((a_wsel - 1u) & 63u)])) + ((uint64_t)(((uint32_t)(((uint32_t)(a_offset + v_skip)) + v_i))))); + if (v_j < ((uint64_t)(a_dictbuf.len))) { + self->private_data.f_transformed_word[v_idx] = a_dictbuf.ptr[v_j]; + } + } + v_idx += 1u; + v_i += 1u; + } + if (v_t_type == 10u) { + wuffs_brotli__decoder__to_upper_case(self, v_word_start, v_idx); + } else if (v_t_type == 11u) { + v_i = v_word_start; + while (v_i < v_idx) { + v_step = wuffs_brotli__decoder__to_upper_case(self, v_i, v_idx); + v_i += v_step; + } + } else if ((v_t_type == 21u) || (v_t_type == 22u)) { + v_j = (((uint64_t)(self->private_data.f_tl_params[v_k])) + (((uint64_t)(a_transform_idx)) * 2u)); + v_param = (wuffs_brotli__decoder__byte_at(self, a_dictbuf, v_j) | (wuffs_brotli__decoder__byte_at(self, a_dictbuf, (v_j + 1u)) << 8u)); + if (v_t_type == 21u) { + wuffs_brotli__decoder__shift(self, v_word_start, ((uint32_t)(v_idx - v_word_start)), v_param); + } else { + v_i = v_word_start; + v_remaining = ((uint32_t)(v_idx - v_word_start)); + while (v_remaining > 0u) { + v_step = wuffs_brotli__decoder__shift(self, v_i, v_remaining, v_param); + if (v_step >= v_remaining) { + break; + } + v_i += v_step; + v_remaining -= v_step; + } + } + } + v_idx = wuffs_brotli__decoder__append_affix(self, + a_dictbuf, + a_tsel, + v_suffix_id, + v_idx); + return v_idx; +} + +// -------- func brotli.decoder.append_affix + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__append_affix( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_dictbuf, + uint32_t a_tsel, + uint32_t a_id, + uint32_t a_idx) { + uint32_t v_idx = 0; + uint32_t v_affix = 0; + uint32_t v_affix_len = 0; + uint32_t v_i = 0; + uint32_t v_j = 0; + uint32_t v_k = 0; + uint64_t v_pos = 0; + + v_idx = a_idx; + if (a_tsel == 0u) { + v_affix = WUFFS_BROTLI__PREFIX_SUFFIX_MAP[wuffs_base__u32__min(a_id, 49u)]; + v_affix_len = ((uint32_t)(WUFFS_BROTLI__PREFIX_SUFFIX[v_affix])); + v_i = 0u; + while (v_i < v_affix_len) { + v_j = ((uint32_t)(((uint32_t)(v_affix + 1u)) + v_i)); + if ((v_j >= 217u) || (v_idx >= 1024u)) { + break; + } + self->private_data.f_transformed_word[v_idx] = WUFFS_BROTLI__PREFIX_SUFFIX[v_j]; + v_idx += 1u; + v_i += 1u; + } + return v_idx; + } + v_k = ((a_tsel - 1u) & 63u); + v_pos = (((uint64_t)(self->private_data.f_tl_prefix_suffix[v_k])) + ((uint64_t)(self->private_data.f_tl_map[v_k][a_id]))); + v_affix_len = 0u; + if (v_pos < ((uint64_t)(a_dictbuf.len))) { + v_affix_len = ((uint32_t)(a_dictbuf.ptr[v_pos])); + } + v_i = 0u; + while (v_i < v_affix_len) { + v_pos += 1u; + if ((v_pos >= ((uint64_t)(a_dictbuf.len))) || (v_idx >= 1024u)) { + break; + } + self->private_data.f_transformed_word[v_idx] = a_dictbuf.ptr[v_pos]; + v_idx += 1u; + v_i += 1u; + } + return v_idx; +} + +// -------- func brotli.decoder.shift + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__shift( + wuffs_brotli__decoder* self, + uint32_t a_p, + uint32_t a_word_len, + uint32_t a_param) { + uint32_t v_p = 0; + uint32_t v_scalar = 0; + uint32_t v_c0 = 0; + uint32_t v_c1 = 0; + uint32_t v_c2 = 0; + uint32_t v_c3 = 0; + + if (a_p > 1019u) { + return 1u; + } + v_p = a_p; + v_scalar = ((uint32_t)((a_param & 32767u) + ((uint32_t)(16777216u - (a_param & 32768u))))); + v_c0 = ((uint32_t)(self->private_data.f_transformed_word[v_p])); + if (v_c0 < 128u) { + v_scalar += v_c0; + self->private_data.f_transformed_word[v_p] = ((uint8_t)((v_scalar & 127u))); + return 1u; + } else if (v_c0 < 192u) { + return 1u; + } else if (v_c0 < 224u) { + if (a_word_len < 2u) { + return 1u; + } + v_c1 = ((uint32_t)(self->private_data.f_transformed_word[(v_p + 1u)])); + v_scalar += ((v_c1 & 63u) | ((v_c0 & 31u) << 6u)); + self->private_data.f_transformed_word[v_p] = ((uint8_t)((192u | ((v_scalar >> 6u) & 31u)))); + self->private_data.f_transformed_word[(v_p + 1u)] = ((uint8_t)(((v_c1 & 192u) | (v_scalar & 63u)))); + return 2u; + } else if (v_c0 < 240u) { + if (a_word_len < 3u) { + return a_word_len; + } + v_c1 = ((uint32_t)(self->private_data.f_transformed_word[(v_p + 1u)])); + v_c2 = ((uint32_t)(self->private_data.f_transformed_word[(v_p + 2u)])); + v_scalar += ((v_c2 & 63u) | ((v_c1 & 63u) << 6u) | ((v_c0 & 15u) << 12u)); + self->private_data.f_transformed_word[v_p] = ((uint8_t)((224u | ((v_scalar >> 12u) & 15u)))); + self->private_data.f_transformed_word[(v_p + 1u)] = ((uint8_t)(((v_c1 & 192u) | ((v_scalar >> 6u) & 63u)))); + self->private_data.f_transformed_word[(v_p + 2u)] = ((uint8_t)(((v_c2 & 192u) | (v_scalar & 63u)))); + return 3u; + } else if (v_c0 < 248u) { + if (a_word_len < 4u) { + return a_word_len; + } + v_c1 = ((uint32_t)(self->private_data.f_transformed_word[(v_p + 1u)])); + v_c2 = ((uint32_t)(self->private_data.f_transformed_word[(v_p + 2u)])); + v_c3 = ((uint32_t)(self->private_data.f_transformed_word[(v_p + 3u)])); + v_scalar += ((v_c3 & 63u) | + ((v_c2 & 63u) << 6u) | + ((v_c1 & 63u) << 12u) | + ((v_c0 & 7u) << 18u)); + self->private_data.f_transformed_word[v_p] = ((uint8_t)((240u | ((v_scalar >> 18u) & 7u)))); + self->private_data.f_transformed_word[(v_p + 1u)] = ((uint8_t)(((v_c1 & 192u) | ((v_scalar >> 12u) & 63u)))); + self->private_data.f_transformed_word[(v_p + 2u)] = ((uint8_t)(((v_c2 & 192u) | ((v_scalar >> 6u) & 63u)))); + self->private_data.f_transformed_word[(v_p + 3u)] = ((uint8_t)(((v_c3 & 192u) | (v_scalar & 63u)))); + return 4u; + } + return 1u; +} + +// -------- func brotli.decoder.to_upper_case + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__to_upper_case( + wuffs_brotli__decoder* self, + uint32_t a_p, + uint32_t a_end) { + uint8_t v_c = 0; + + if (a_p >= 1024u) { + return 1u; + } + v_c = self->private_data.f_transformed_word[a_p]; + if (v_c < 192u) { + if ((97u <= v_c) && (v_c <= 122u)) { + self->private_data.f_transformed_word[a_p] = ((uint8_t)(v_c ^ 32u)); + } + return 1u; + } + if (v_c < 224u) { + if (((a_p + 1u) < a_end) && ((a_p + 1u) < 1024u)) { + self->private_data.f_transformed_word[(a_p + 1u)] ^= 32u; + } + return 2u; + } + if (((a_p + 2u) < a_end) && ((a_p + 2u) < 1024u)) { + self->private_data.f_transformed_word[(a_p + 2u)] ^= 5u; + } + return 3u; +} + +// -------- func brotli.decoder.byte_at + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__byte_at( + const wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_s, + uint64_t a_i) { + if (a_i < ((uint64_t)(a_s.len))) { + return ((uint32_t)(a_s.ptr[a_i])); + } + return 0u; +} + // -------- func brotli.decoder.read_huffman_code WUFFS_BASE__GENERATED_C_CODE @@ -58157,7 +59387,7 @@ wuffs_brotli__decoder__build_simple_huffman_table( uint32_t v_k = 0; uint32_t v_table_size = 0; - if (a_hoff > 671634u) { + if (a_hoff > 677778u) { return wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); } v_t = a_hoff; @@ -58271,7 +59501,7 @@ wuffs_brotli__decoder__read_code_length_code_lengths( v_i = a_hskip; while (v_i < 18u) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); - status = wuffs_brotli__decoder__read_symbol(self, a_src, 671634u); + status = wuffs_brotli__decoder__read_symbol(self, a_src, 677778u); if (status.repr) { goto suspend; } @@ -58342,7 +59572,7 @@ wuffs_brotli__decoder__build_code_lengths_huffman_table( v_code = ((uint16_t)((v_only_symbol << 4u))); v_i = 0u; while (v_i < 256u) { - self->private_data.f_huffs[(671378u + v_i)] = v_code; + self->private_data.f_huffs[(677522u + v_i)] = v_code; v_i += 1u; } return wuffs_base__make_empty_struct(); @@ -58360,7 +59590,7 @@ wuffs_brotli__decoder__build_code_lengths_huffman_table( v_end = 32u; while (v_end >= v_step) { v_end -= v_step; - self->private_data.f_huffs[(671378u + ((v_r + v_end) & 31u))] = v_code; + self->private_data.f_huffs[(677522u + ((v_r + v_end) & 31u))] = v_code; } v_key += v_key_step; } @@ -58370,7 +59600,7 @@ wuffs_brotli__decoder__build_code_lengths_huffman_table( } v_i = 32u; while (v_i < 256u) { - self->private_data.f_huffs[(671378u + v_i)] = self->private_data.f_huffs[(671378u + (v_i & 31u))]; + self->private_data.f_huffs[(677522u + v_i)] = self->private_data.f_huffs[(677522u + (v_i & 31u))]; v_i += 1u; } return wuffs_base__make_empty_struct(); @@ -58432,7 +59662,7 @@ wuffs_brotli__decoder__read_symbol_code_lengths( v_space = 32768u; while ((v_symbol < v_alphabet_size) && (v_space != 0u)) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); - status = wuffs_brotli__decoder__read_symbol(self, a_src, 671378u); + status = wuffs_brotli__decoder__read_symbol(self, a_src, 677522u); if (status.repr) { goto suspend; } @@ -58603,7 +59833,7 @@ wuffs_brotli__decoder__build_huffman_table( while ((v_end >= v_step) && (v_step > 0u)) { v_end -= v_step; v_i = ((uint32_t)(((uint32_t)(a_hoff + v_r)) + v_end)); - if (v_i >= 671890u) { + if (v_i >= 678034u) { return wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); } self->private_data.f_huffs[v_i] = v_code; @@ -58619,7 +59849,7 @@ wuffs_brotli__decoder__build_huffman_table( while (v_i < 256u) { v_n = ((uint32_t)(a_hoff + v_i)); v_cur = ((uint32_t)(a_hoff + (v_i & ((uint32_t)(((uint32_t)(v_table_size)) - 1u))))); - if ((v_n >= 671890u) || (v_cur >= 671890u)) { + if ((v_n >= 678034u) || (v_cur >= 678034u)) { return wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); } self->private_data.f_huffs[v_n] = self->private_data.f_huffs[v_cur]; @@ -58656,7 +59886,7 @@ wuffs_brotli__decoder__build_huffman_table( v_sub_key = ((uint32_t)(WUFFS_BROTLI__REVERSE_BITS[(v_key & 255u)])); v_key += v_key_step; v_i = ((uint32_t)(a_hoff + v_sub_key)); - if (v_i >= 671890u) { + if (v_i >= 678034u) { return wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); } self->private_data.f_huffs[v_i] = ((uint16_t)((((v_cur & 4095u) << 4u) | (((uint32_t)(v_table_bits)) + 8u)))); @@ -58673,7 +59903,7 @@ wuffs_brotli__decoder__build_huffman_table( while ((v_end >= v_step) && (v_step > 0u)) { v_end -= v_step; v_i = ((uint32_t)(((uint32_t)(((uint32_t)(a_hoff + v_cur)) + v_r)) + v_end)); - if (v_i >= 671890u) { + if (v_i >= 678034u) { return wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); } self->private_data.f_huffs[v_i] = v_code; @@ -58695,7 +59925,8 @@ static wuffs_base__status wuffs_brotli__decoder__decode_compressed_metablock( wuffs_brotli__decoder* self, wuffs_base__io_buffer* a_src, - wuffs_base__slice_u8 a_workbuf) { + wuffs_base__slice_u8 a_workbuf, + wuffs_base__slice_u8 a_dictbuf) { wuffs_base__status status = wuffs_base__make_status(NULL); uint32_t v_t = 0; @@ -58705,6 +59936,7 @@ wuffs_brotli__decoder__decode_compressed_metablock( uint32_t v_bits = 0; uint32_t v_npostfix = 0; uint32_t v_ndirect = 0; + uint32_t v_alphabet_max = 0; uint32_t v_alphabet = 0; uint32_t v_i = 0; uint32_t v_k = 0; @@ -58717,6 +59949,7 @@ wuffs_brotli__decoder__decode_compressed_metablock( v_n2 = self->private_data.s_decode_compressed_metablock.v_n2; v_npostfix = self->private_data.s_decode_compressed_metablock.v_npostfix; v_ndirect = self->private_data.s_decode_compressed_metablock.v_ndirect; + v_alphabet_max = self->private_data.s_decode_compressed_metablock.v_alphabet_max; v_alphabet = self->private_data.s_decode_compressed_metablock.v_alphabet; v_i = self->private_data.s_decode_compressed_metablock.v_i; v_k = self->private_data.s_decode_compressed_metablock.v_k; @@ -58739,7 +59972,7 @@ wuffs_brotli__decoder__decode_compressed_metablock( a_src, (v_n + 2u), (v_n + 2u), - (667648u + (v_t * 632u)), + (673792u + (v_t * 632u)), 632u); if (status.repr) { goto suspend; @@ -58749,7 +59982,7 @@ wuffs_brotli__decoder__decode_compressed_metablock( a_src, 26u, 26u, - (669544u + (v_t * 396u)), + (675688u + (v_t * 396u)), 396u); if (status.repr) { goto suspend; @@ -58797,7 +60030,13 @@ wuffs_brotli__decoder__decode_compressed_metablock( goto suspend; } self->private_impl.f_num_dist_htrees = wuffs_base__u32__min(self->private_impl.f_bits_value, 256u); - v_alphabet = (16u + v_ndirect + (((uint32_t)(24u)) << (v_npostfix + 1u))); + if (self->private_impl.f_large_window) { + v_alphabet_max = (16u + v_ndirect + (((uint32_t)(62u)) << (v_npostfix + 1u))); + v_alphabet = wuffs_brotli__decoder__calculate_distance_code_limit(self, v_npostfix, v_ndirect); + } else { + v_alphabet = (16u + v_ndirect + (((uint32_t)(24u)) << (v_npostfix + 1u))); + v_alphabet_max = v_alphabet; + } self->private_impl.f_distance_alphabet_size = v_alphabet; v_k = 0u; while (v_k < self->private_impl.f_num_literal_htrees) { @@ -58832,10 +60071,10 @@ wuffs_brotli__decoder__decode_compressed_metablock( WUFFS_BASE__COROUTINE_SUSPENSION_POINT(11); status = wuffs_brotli__decoder__read_huffman_code(self, a_src, + v_alphabet_max, v_alphabet, - v_alphabet, - (438272u + (v_k * 896u)), - 896u); + (438272u + (v_k * 920u)), + 920u); if (status.repr) { goto suspend; } @@ -58846,7 +60085,7 @@ wuffs_brotli__decoder__decode_compressed_metablock( self->private_impl.f_htree_command = 0u; wuffs_brotli__decoder__calculate_distance_lut(self); WUFFS_BASE__COROUTINE_SUSPENSION_POINT(12); - status = wuffs_brotli__decoder__process_commands(self, a_src, a_workbuf); + status = wuffs_brotli__decoder__process_commands(self, a_src, a_workbuf, a_dictbuf); if (status.repr) { goto suspend; } @@ -58866,6 +60105,7 @@ wuffs_brotli__decoder__decode_compressed_metablock( self->private_data.s_decode_compressed_metablock.v_n2 = v_n2; self->private_data.s_decode_compressed_metablock.v_npostfix = v_npostfix; self->private_data.s_decode_compressed_metablock.v_ndirect = v_ndirect; + self->private_data.s_decode_compressed_metablock.v_alphabet_max = v_alphabet_max; self->private_data.s_decode_compressed_metablock.v_alphabet = v_alphabet; self->private_data.s_decode_compressed_metablock.v_i = v_i; self->private_data.s_decode_compressed_metablock.v_k = v_k; @@ -58875,6 +60115,44 @@ wuffs_brotli__decoder__decode_compressed_metablock( return status; } +// -------- func brotli.decoder.calculate_distance_code_limit + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__calculate_distance_code_limit( + const wuffs_brotli__decoder* self, + uint32_t a_npostfix, + uint32_t a_ndirect) { + uint32_t v_offset = 0; + uint32_t v_ndistbits = 0; + uint32_t v_tmp = 0; + uint32_t v_half = 0; + uint32_t v_group = 0; + uint32_t v_postfix = 0; + uint32_t v_ret = 0; + + v_offset = (((2147483644u - a_ndirect) >> a_npostfix) + 4u); + v_postfix = ((((uint32_t)(1u)) << a_npostfix) - 1u); + v_tmp = (v_offset / 2u); + v_ndistbits = 0u; + while ((v_tmp != 0u) && (v_ndistbits < 32u)) { + v_ndistbits += 1u; + v_tmp >>= 1u; + } + v_ndistbits -= 1u; + v_half = ((v_offset >> (v_ndistbits & 31u)) & 1u); + v_group = (((uint32_t)(((uint32_t)(v_ndistbits - 1u)) << 1u)) | v_half); + if (v_group == 0u) { + return (a_ndirect + 16u); + } + v_group -= 1u; + v_ret = ((uint32_t)(((uint32_t)((((uint32_t)(v_group << a_npostfix)) | v_postfix) + a_ndirect)) + 17u)); + if (v_ret > 544u) { + return 544u; + } + return v_ret; +} + // -------- func brotli.decoder.decode_context_map WUFFS_BASE__GENERATED_C_CODE @@ -58952,7 +60230,7 @@ wuffs_brotli__decoder__decode_context_map( a_src, v_alphabet, v_alphabet, - 670732u, + 676876u, 646u); if (status.repr) { goto suspend; @@ -58960,7 +60238,7 @@ wuffs_brotli__decoder__decode_context_map( v_ci = 0u; while (v_ci < v_size) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT(5); - status = wuffs_brotli__decoder__read_symbol(self, a_src, 670732u); + status = wuffs_brotli__decoder__read_symbol(self, a_src, 676876u); if (status.repr) { goto suspend; } @@ -59088,7 +60366,7 @@ wuffs_brotli__decoder__read_block_length( WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); - status = wuffs_brotli__decoder__read_symbol(self, a_src, (669544u + (a_t * 396u))); + status = wuffs_brotli__decoder__read_symbol(self, a_src, (675688u + (a_t * 396u))); if (status.repr) { goto suspend; } @@ -59148,7 +60426,7 @@ wuffs_brotli__decoder__decode_block_switch( goto exit; } WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); - status = wuffs_brotli__decoder__read_symbol(self, a_src, (667648u + (v_t * 632u))); + status = wuffs_brotli__decoder__read_symbol(self, a_src, (673792u + (v_t * 632u))); if (status.repr) { goto suspend; } @@ -59263,7 +60541,8 @@ static wuffs_base__status wuffs_brotli__decoder__process_commands( wuffs_brotli__decoder* self, wuffs_base__io_buffer* a_src, - wuffs_base__slice_u8 a_workbuf) { + wuffs_base__slice_u8 a_workbuf, + wuffs_base__slice_u8 a_dictbuf) { wuffs_base__status status = wuffs_base__make_status(NULL); uint32_t v_cmd_code = 0; @@ -59279,6 +60558,10 @@ wuffs_brotli__decoder__process_commands( uint32_t v_n = 0; uint32_t v_k = 0; uint8_t v_b = 0; + uint32_t v_compound_size = 0; + uint32_t v_address = 0; + uint32_t v_chunk = 0; + uint64_t v_j = 0; uint32_t coro_susp_point = self->private_impl.p_process_commands; if (coro_susp_point) { @@ -59289,6 +60572,8 @@ wuffs_brotli__decoder__process_commands( v_i = self->private_data.s_process_commands.v_i; v_n = self->private_data.s_process_commands.v_n; v_k = self->private_data.s_process_commands.v_k; + v_address = self->private_data.s_process_commands.v_address; + v_chunk = self->private_data.s_process_commands.v_chunk; } switch (coro_susp_point) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; @@ -59398,31 +60683,78 @@ wuffs_brotli__decoder__process_commands( status = wuffs_base__make_status(wuffs_brotli__error__bad_distance); goto exit; } - if ((v_i < 4u) || (24u < v_i)) { - status = wuffs_base__make_status(wuffs_brotli__error__bad_dictionary); - goto exit; - } - self->private_impl.f_dist_rb_idx += self->private_impl.f_distance_context; - v_n = wuffs_brotli__decoder__copy_dictionary_word(self, ((uint32_t)(((uint32_t)(v_distance - v_max_distance)) - 1u)), wuffs_base__u32__min(v_i, 24u)); - if (v_n == 4294967295u) { - status = wuffs_base__make_status(wuffs_brotli__error__bad_transform); - goto exit; - } else if ((v_n == 0u) && (v_distance <= 120u)) { - status = wuffs_base__make_status(wuffs_brotli__error__bad_transform); - goto exit; - } - self->private_impl.f_mb_remaining -= v_n; - v_k = 0u; - while (v_k < v_n) { - wuffs_brotli__decoder__put_byte(self, a_workbuf, self->private_data.f_transformed_word[(v_k & 63u)]); - v_k += 1u; - if (self->private_impl.f_pos == self->private_impl.f_rb_size) { - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(10); - status = wuffs_brotli__decoder__ring_event(self); - if (status.repr) { - goto suspend; + v_compound_size = self->private_impl.f_chunk_offsets[self->private_impl.f_num_chunks]; + v_address = ((v_distance - v_max_distance) - 1u); + if (v_address < v_compound_size) { + v_address = ((uint32_t)(v_compound_size - (v_address + 1u))); + if ((((uint64_t)(v_address)) + ((uint64_t)(v_i))) > ((uint64_t)(v_compound_size))) { + status = wuffs_base__make_status(wuffs_brotli__error__bad_compound_dictionary); + goto exit; + } + self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)] = v_distance; + self->private_impl.f_dist_rb_idx += 1u; + self->private_impl.f_mb_remaining -= v_i; + v_chunk = 0u; + while (v_i > 0u) { + while (v_chunk < 14u) { + if (v_address < self->private_impl.f_chunk_offsets[(v_chunk + 1u)]) { + break; + } + v_chunk += 1u; + } + v_j = (((uint64_t)(self->private_impl.f_chunk_starts[v_chunk])) + ((uint64_t)(((uint32_t)(v_address - self->private_impl.f_chunk_offsets[v_chunk]))))); + v_b = 0u; + if (v_j < ((uint64_t)(a_dictbuf.len))) { + v_b = a_dictbuf.ptr[v_j]; + } + wuffs_brotli__decoder__put_byte(self, a_workbuf, v_b); + v_address += 1u; + v_i -= 1u; + if (self->private_impl.f_pos == self->private_impl.f_rb_size) { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(10); + status = wuffs_brotli__decoder__ring_event(self); + if (status.repr) { + goto suspend; + } } } + } else if ((4u <= v_i) && (v_i <= 31u)) { + v_p1 = ((uint32_t)(wuffs_brotli__decoder__get_byte(self, a_workbuf, 1u))); + v_p2 = ((uint32_t)(wuffs_brotli__decoder__get_byte(self, a_workbuf, 2u))); + v_mode_offset = (self->private_impl.f_context_mode << 9u); + v_ctx = ((uint32_t)(((uint8_t)(WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | v_p1)] | WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | 256u | v_p2)])))); + self->private_impl.f_dist_rb_idx += self->private_impl.f_distance_context; + v_n = wuffs_brotli__decoder__copy_dictionary_word(self, + a_dictbuf, + ((uint32_t)(v_address - v_compound_size)), + wuffs_base__u32__min(v_i, 31u), + v_ctx); + if (v_n == 4294967294u) { + status = wuffs_base__make_status(wuffs_brotli__error__bad_dictionary); + goto exit; + } else if (v_n == 4294967295u) { + status = wuffs_base__make_status(wuffs_brotli__error__bad_transform); + goto exit; + } else if ((v_n == 0u) && (v_distance <= 120u)) { + status = wuffs_base__make_status(wuffs_brotli__error__bad_transform); + goto exit; + } + self->private_impl.f_mb_remaining -= v_n; + v_k = 0u; + while (v_k < v_n) { + wuffs_brotli__decoder__put_byte(self, a_workbuf, self->private_data.f_transformed_word[(v_k & 1023u)]); + v_k += 1u; + if (self->private_impl.f_pos == self->private_impl.f_rb_size) { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(11); + status = wuffs_brotli__decoder__ring_event(self); + if (status.repr) { + goto suspend; + } + } + } + } else { + status = wuffs_base__make_status(wuffs_brotli__error__bad_dictionary); + goto exit; } } else { self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)] = v_distance; @@ -59433,7 +60765,7 @@ wuffs_brotli__decoder__process_commands( wuffs_brotli__decoder__put_byte(self, a_workbuf, v_b); v_i -= 1u; if (self->private_impl.f_pos == self->private_impl.f_rb_size) { - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(11); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(12); status = wuffs_brotli__decoder__ring_event(self); if (status.repr) { goto suspend; @@ -59462,6 +60794,8 @@ wuffs_brotli__decoder__process_commands( self->private_data.s_process_commands.v_i = v_i; self->private_data.s_process_commands.v_n = v_n; self->private_data.s_process_commands.v_k = v_k; + self->private_data.s_process_commands.v_address = v_address; + self->private_data.s_process_commands.v_chunk = v_chunk; goto exit; exit: @@ -59497,7 +60831,7 @@ wuffs_brotli__decoder__read_distance( WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); - status = wuffs_brotli__decoder__read_symbol(self, a_src, (438272u + (self->private_impl.f_dist_htree_index * 896u))); + status = wuffs_brotli__decoder__read_symbol(self, a_src, (438272u + (self->private_impl.f_dist_htree_index * 920u))); if (status.repr) { goto suspend; } @@ -59514,7 +60848,7 @@ wuffs_brotli__decoder__read_distance( goto exit; } WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); - status = wuffs_brotli__decoder__read_bits(self, a_src, ((uint32_t)(wuffs_base__u8__min(self->private_data.f_dist_extra_bits[v_code], 24u)))); + status = wuffs_brotli__decoder__read_bits32(self, a_src, ((uint32_t)(wuffs_base__u8__min(self->private_data.f_dist_extra_bits[v_code], 32u)))); if (status.repr) { goto suspend; } @@ -59522,7 +60856,7 @@ wuffs_brotli__decoder__read_distance( status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); goto exit; } - self->private_impl.f_distance_code = ((uint32_t)(self->private_data.f_dist_offset[v_code] + ((uint32_t)(self->private_impl.f_bits_value << self->private_impl.f_distance_postfix_bits)))); + self->private_impl.f_distance_code = ((uint32_t)(((uint64_t)(((uint64_t)(self->private_data.f_dist_offset[v_code])) + ((uint64_t)(((uint64_t)(self->private_impl.f_bits_value)) << self->private_impl.f_distance_postfix_bits)))))); ok: self->private_impl.p_read_distance = 0; @@ -59575,135 +60909,6 @@ wuffs_brotli__decoder__take_distance_from_ring_buffer( return wuffs_base__make_empty_struct(); } -// -------- func brotli.decoder.copy_dictionary_word - -WUFFS_BASE__GENERATED_C_CODE -static uint32_t -wuffs_brotli__decoder__copy_dictionary_word( - wuffs_brotli__decoder* self, - uint32_t a_address, - uint32_t a_len) { - uint32_t v_shift = 0; - uint32_t v_word_idx = 0; - uint32_t v_transform_idx = 0; - uint32_t v_offset = 0; - uint32_t v_t = 0; - uint32_t v_t_type = 0; - uint32_t v_affix = 0; - uint32_t v_affix_len = 0; - uint32_t v_idx = 0; - uint32_t v_word_start = 0; - uint32_t v_word_len = 0; - uint32_t v_skip = 0; - uint32_t v_i = 0; - uint32_t v_j = 0; - uint32_t v_step = 0; - - v_shift = WUFFS_BROTLI__DICTIONARY_SIZE_BITS_BY_LENGTH[a_len]; - v_word_idx = (a_address & ((((uint32_t)(1u)) << v_shift) - 1u)); - v_transform_idx = (a_address >> v_shift); - if (v_transform_idx >= 121u) { - return 4294967295u; - } - v_t = wuffs_base__u32__min(v_transform_idx, 120u); - v_offset = ((uint32_t)(WUFFS_BROTLI__DICTIONARY_OFFSETS_BY_LENGTH[a_len] + ((uint32_t)(v_word_idx * a_len)))); - v_idx = 0u; - v_affix = WUFFS_BROTLI__PREFIX_SUFFIX_MAP[WUFFS_BROTLI__TRANSFORM_PREFIX_ID[v_t]]; - v_affix_len = ((uint32_t)(WUFFS_BROTLI__PREFIX_SUFFIX[v_affix])); - v_i = 0u; - while (v_i < v_affix_len) { - v_j = ((uint32_t)(((uint32_t)(v_affix + 1u)) + v_i)); - if ((v_j >= 217u) || (v_idx >= 64u)) { - break; - } - self->private_data.f_transformed_word[v_idx] = WUFFS_BROTLI__PREFIX_SUFFIX[v_j]; - v_idx += 1u; - v_i += 1u; - } - v_t_type = ((uint32_t)(WUFFS_BROTLI__TRANSFORM_TYPE[v_t])); - v_word_len = a_len; - v_skip = 0u; - if (v_t_type <= 9u) { - if (v_t_type < a_len) { - v_word_len = ((uint32_t)(a_len - v_t_type)); - } else { - v_word_len = 0u; - } - } else if ((12u <= v_t_type) && (v_t_type <= 20u)) { - v_skip = (v_t_type - 11u); - if (v_skip < a_len) { - v_word_len = ((uint32_t)(a_len - v_skip)); - } else { - v_word_len = 0u; - } - } - v_word_start = v_idx; - v_i = 0u; - while (v_i < v_word_len) { - v_j = ((uint32_t)(((uint32_t)(v_offset + v_skip)) + v_i)); - if ((v_j >= 122784u) || (v_idx >= 64u)) { - break; - } - self->private_data.f_transformed_word[v_idx] = WUFFS_BROTLI__DICTIONARY[v_j]; - v_idx += 1u; - v_i += 1u; - } - if (v_t_type == 10u) { - wuffs_brotli__decoder__to_upper_case(self, v_word_start, v_idx); - } else if (v_t_type == 11u) { - v_i = v_word_start; - while (v_i < v_idx) { - v_step = wuffs_brotli__decoder__to_upper_case(self, v_i, v_idx); - v_i += v_step; - } - } - v_affix = WUFFS_BROTLI__PREFIX_SUFFIX_MAP[WUFFS_BROTLI__TRANSFORM_SUFFIX_ID[v_t]]; - v_affix_len = ((uint32_t)(WUFFS_BROTLI__PREFIX_SUFFIX[v_affix])); - v_i = 0u; - while (v_i < v_affix_len) { - v_j = ((uint32_t)(((uint32_t)(v_affix + 1u)) + v_i)); - if ((v_j >= 217u) || (v_idx >= 64u)) { - break; - } - self->private_data.f_transformed_word[v_idx] = WUFFS_BROTLI__PREFIX_SUFFIX[v_j]; - v_idx += 1u; - v_i += 1u; - } - return v_idx; -} - -// -------- func brotli.decoder.to_upper_case - -WUFFS_BASE__GENERATED_C_CODE -static uint32_t -wuffs_brotli__decoder__to_upper_case( - wuffs_brotli__decoder* self, - uint32_t a_p, - uint32_t a_end) { - uint8_t v_c = 0; - - if (a_p >= 64u) { - return 1u; - } - v_c = self->private_data.f_transformed_word[a_p]; - if (v_c < 192u) { - if ((97u <= v_c) && (v_c <= 122u)) { - self->private_data.f_transformed_word[a_p] = ((uint8_t)(v_c ^ 32u)); - } - return 1u; - } - if (v_c < 224u) { - if (((a_p + 1u) < a_end) && ((a_p + 1u) < 64u)) { - self->private_data.f_transformed_word[(a_p + 1u)] ^= 32u; - } - return 2u; - } - if (((a_p + 2u) < a_end) && ((a_p + 2u) < 64u)) { - self->private_data.f_transformed_word[(a_p + 2u)] ^= 5u; - } - return 3u; -} - #endif // !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__BROTLI) #if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__BZIP2) diff --git a/std/brotli/README.md b/std/brotli/README.md index 8ebf1a2b..7a8d3d0d 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -49,8 +49,10 @@ decoder does not read any further than that. ## Workbuf Layout -`workbuf_len` is `(1 << 24) + 542` bytes, both minimum and maximum: the -largest (non-large-window) brotli window size plus some slack. The slack +Without the large-window quirk and without attached dictionaries (see the +"Large Window" and "Dictionaries" sections below), `workbuf_len` is `(1 << 24) ++ 542` bytes, both minimum and maximum: the largest (non-large-window) brotli +window size plus some slack. The slack mirrors the C code's `kRingBufferWriteAheadSlack`, which its fast paths use to write (and discard) a few bytes past the ring buffer's end. This implementation does not yet have such fast paths, but it still reserves the @@ -101,15 +103,18 @@ a corresponding Wuffs status: | `BROTLI_DECODER_ERROR_FORMAT_PADDING_2` | `"#bad final padding"` | | `BROTLI_DECODER_ERROR_FORMAT_DISTANCE` | `"#bad distance"` | | `BROTLI_DECODER_ERROR_FORMAT_BLOCK_SWITCH` | `"#bad block switch"` | +| `BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY` | `"#bad compound dictionary"` | | `BROTLI_DECODER_NEEDS_MORE_INPUT` (with closed `src`) | `"#truncated input"` | Notes: - `"#unsupported large window"` is for the large-window escape `WBITS` value - (`0010001`). libbrotlidec (with `BROTLI_DECODER_PARAM_LARGE_WINDOW` not set, - the default) rejects it with `BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS`. For - standard (non-large-window) brotli, no other `WBITS` value is invalid, so - `"#bad window bits"` is not currently produced. + (`0010001`) when the `QUIRK_LARGE_WINDOW` quirk is not set. libbrotlidec + (with `BROTLI_DECODER_PARAM_LARGE_WINDOW` not set, the default) rejects it + with `BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS`. For standard + (non-large-window) brotli, no other `WBITS` value is invalid. With the quirk + set, `"#bad window bits"` is for a large-window header with a non-zero + reserved bit or with window bits outside of `[10 ..= 30]`. - Like the C code, which error an overrun metablock produces (`"#bad block length"` or `"#bad metablock length"`) can depend on how the input is split across `transform_io` calls. When the C code runs out of input, it first @@ -118,8 +123,9 @@ Notes: running out of input. - `"#bad block switch"` needs more than `(1 << 24)` commands in one metablock. It is implemented but not tested. -- The remaining libbrotlidec error codes do not apply: `COMPOUND_DICTIONARY` - and `DICTIONARY_NOT_SET` (custom dictionaries are not supported), +- The remaining libbrotlidec error codes do not apply: `DICTIONARY_NOT_SET` + (it needs a word list without data, which libbrotlidec only has when built + with `BROTLI_EXTERNAL_DICTIONARY_DATA` and not given that data), `INVALID_ARGUMENTS`, the `ALLOC_*` codes (this decoder does not allocate) and `UNREACHABLE`. @@ -145,10 +151,151 @@ invariants for memory safety but Wuffs requires explicit bounds checks. ## TODO -- Large-window brotli (`BROTLI_DECODER_PARAM_LARGE_WINDOW`, window bits up to - 30). The window-related fields are sized for 24 bits. -- Custom, shared and compound dictionaries. -- Metadata callbacks (metadata blocks are currently skipped). +- Metadata callbacks (metadata blocks are currently skipped). See the + "Metadata Blocks" section below. - Fast paths. -- Fuzzers. - An encoder. + + +## Large Window + +Large-window brotli (an incompatible extension of RFC 7932, with window sizes +up to `(1 << 30)` bytes) is enabled by the `QUIRK_LARGE_WINDOW` quirk. It +mirrors libbrotlidec's `BROTLI_DECODER_PARAM_LARGE_WINDOW` parameter: it must +be set before decoding starts (afterwards, `set_quirk` returns `"#bad call +sequence"`, like `BrotliDecoderSetParameter` failing once the decoder has +left its `BROTLI_STATE_UNINITED` state). With the quirk set: + +- a stream that starts with the large-window `WBITS` escape is decoded as a + large-window stream: the escape is followed by a reserved bit (that must be + zero) and 6 window bits (that must be in `[10 ..= 30]`), otherwise the + decoder fails with `"#bad window bits"`. +- a stream that does not start with that escape is decoded as standard brotli, + exactly as without the quirk. +- `workbuf_len` is `(1 << 30) + 542` bytes (plus any dictionaries), the + largest large-window window size plus slack. As before, the workbuf only + needs to be as long as the stream's actual window size plus 542 bytes (plus + any dictionaries). Without the quirk, it is `(1 << 24) + 542` bytes (plus + any dictionaries), as before. + +For a large-window stream, like the C code, the distance alphabet's +`alphabet_size_max` uses `BROTLI_LARGE_MAX_DISTANCE_BITS` (62) and its +`alphabet_size_limit` comes from `BrotliCalculateDistanceCodeLimit` (at most +544 symbols, so that no distance exceeds `BROTLI_MAX_ALLOWED_DISTANCE`). Up to +29 extra distance bits are read (the C code's `BrotliSafeReadBits32`) and the +distance is calculated with 64-bit arithmetic, like the C code's +`brotli_reg_t`. The Huffman table stride for distance trees is sized for 544 +symbols in both modes. + +The `DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE` constant is now the overall +worst case: `(1 << 30) + 542` plus the maximum total dictionary length +(`0x7FFF_FFFF`). The new `DECODER_WORKBUF_LEN_STANDARD` and +`DECODER_WORKBUF_LEN_LARGE_WINDOW` constants give the window-only values. + + +## Dictionaries + +`decoder.add_dictionary!(dict_type: base.u32, dict: roslice base.u8, workbuf: +slice base.u8) base.status` mirrors libbrotlidec's +`BrotliDecoderAttachDictionary`. `dict_type` is `DICTIONARY_TYPE_RAW` (0, the +C code's `BROTLI_SHARED_DICTIONARY_RAW`) or `DICTIONARY_TYPE_SERIALIZED` (1, +`BROTLI_SHARED_DICTIONARY_SERIALIZED`). It must be called before decoding +starts. It can be called multiple times. + +Wuffs structs cannot hold pointers or slices, so `add_dictionary` copies the +dictionary bytes into the workbuf. Every `add_dictionary` and `transform_io` +call must be passed the same workbuf (or at least one whose first bytes hold +the same dictionary bytes). The `add_dictionary` workbuf only needs to be long +enough to hold all of the dictionaries added so far. Call `workbuf_len` after +the last `add_dictionary` call: it grows by each dictionary's length. + +The workbuf layout is: + + [0 .. D) the attached dictionaries, concatenated, where D is + the total length of all successfully added + dictionaries (raw or serialized, in full). + [D .. D + W + 542) the history ring buffer (W = 1 << WBITS) and slack. + +The dictionaries come first because their position should not depend on the +stream's window size, which is only known once decoding has started. + +Raw dictionaries are LZ77 prefix dictionaries. Like the C code (its +`BrotliDecoderCompoundDictionary` and `AttachCompoundDictionary`, +`InitializeCompoundDictionaryCopy` and `CopyFromCompoundDictionary` +functions), at most 15 such "compound dictionary" chunks (raw dictionaries +plus serialized dictionaries' LZ77 prefixes) can be attached. A backward +reference distance `d` beyond the current maximum LZ77 distance `m` (but +within `BROTLI_MAX_ALLOWED_DISTANCE`) refers to the compound dictionary when +`d - m - 1` is less than the compound dictionary's total size `T`, at address +`T - (d - m)`. A copy that would extend past the compound dictionary's end +fails with `"#bad compound dictionary"`. Otherwise, the static dictionary's +address is `d - m - 1 - T`. + +Serialized dictionaries are parsed and validated like +`BrotliSharedDictionaryAttach` (`DecodeSharedDictionary`, +`DryParseDictionary`, `ParseDictionary`, `ParseWordList`, +`ParseTransformsList` and `ParsePrefixSuffixTable` in +`c/common/shared_dictionary.c`): the `0x91 0x00` magic, an optional LZ77 +prefix (at most 1073741823 bytes), up to 64 custom word lists (word lengths 4 +to 31, at most 15 size bits per length), up to 64 custom transform lists (up +to 256 prefix/suffix stringlets, up to 255 transforms, transform types up to +`BROTLI_TRANSFORM_SHIFT_ALL`, with 16-bit parameters for the +`BROTLI_TRANSFORM_SHIFT_FIRST` and `BROTLI_TRANSFORM_SHIFT_ALL` types), up to +64 (word list, transform list) combinations and an optional 64-entry context +map. The parsed metadata lives in fixed-size decoder fields (`wl_etc`, `tl_etc` +and `sd_etc`), with offsets into the copied bytes in the workbuf. Like the C +code: + +- only one attached serialized dictionary can have custom word lists or + transform lists (otherwise `"#unsupported multiple custom static + dictionaries"`), and +- attaching a serialized dictionary without custom word lists or transform + lists (only an LZ77 prefix) resets the static dictionary to the built-in + (RFC 7932) one, even if an earlier serialized dictionary had custom lists. + +Static dictionary references then mirror `ProcessCommandsInternal`: the word +list and transform list are selected by the literal context (`BROTLI_CONTEXT(p1, +p2, context_lookup)`) through the context map (if enabled), with the C code's +fall back to the other (word list, transform list) combinations when the +address is out of range, and words are transformed like +`BrotliTransformDictionaryWord` (including `Shift` for the shift transforms). +Transformed words can be up to 541 bytes long. + +`add_dictionary` statuses (the C code just returns `BROTLI_FALSE`): + +| Condition | Wuffs status | +| -------------------------------------------------------------- | ----------------------------------------------------- | +| decoding has started | `base."#bad call sequence"` | +| unknown `dict_type` | `base."#bad argument"` | +| more than 15 compound dictionary chunks | `"#too many compound dictionaries"` | +| invalid serialized dictionary | `"#bad shared dictionary"` | +| a second serialized dictionary with custom lists | `"#unsupported multiple custom static dictionaries"` | +| more than `0x7FFF_FFFF` bytes of dictionaries in total | `"#too much dictionary data"` | +| `workbuf` shorter than the dictionaries so far plus this one | `base."#bad workbuf length"` | + +Differences from the C code: + +- On failure, `add_dictionary` leaves the decoder unchanged. libbrotlidec's + `BrotliDecoderAttachDictionary` can fail after partially updating its shared + dictionary (e.g. after counting a serialized dictionary's LZ77 prefix or + reading its number of word lists), which can affect later attach calls. +- libbrotlicommon only parses serialized dictionaries when built with + `BROTLI_EXPERIMENTAL`, which is not the default. A default build rejects + every serialized dictionary. This package always accepts them, matching an + experimental build. The mimic tests skip serialized dictionary cases when + the linked libbrotlidec rejects them; they pass, with no skips, when linked + against a `BROTLI_EXPERIMENTAL` build. +- The C code holds the compound dictionary's total size in an `int`. This + package rejects more than `0x7FFF_FFFF` bytes of dictionaries in total + (counting serialized dictionaries in full), where the C code would overflow. +- `BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET` cannot happen (see above). + + +## Metadata Blocks + +libbrotlidec can report metadata blocks' contents through callbacks +(`BrotliDecoderSetMetadataCallbacks`). Wuffs code cannot call back into its +caller, so there is no equivalent: metadata blocks are skipped (validated and +consumed but not reported), exactly as libbrotlidec does without callbacks, +so the decoded output is the same. A future quirk could expose metadata +blocks, e.g. by suspending with a status when one starts. diff --git a/std/brotli/decode_brotli.wuffs b/std/brotli/decode_brotli.wuffs index e6588db7..eccf6bd2 100644 --- a/std/brotli/decode_brotli.wuffs +++ b/std/brotli/decode_brotli.wuffs @@ -20,12 +20,12 @@ // // See README.md for how C error codes map to Wuffs statuses. // -// TODO: support large-window brotli (BROTLI_DECODER_PARAM_LARGE_WINDOW), -// custom / attached / shared dictionaries, metadata callbacks and fast paths. +// TODO: support metadata callbacks and fast paths. pub status "#bad block length" pub status "#bad block switch" pub status "#bad code length code lengths" +pub status "#bad compound dictionary" pub status "#bad context map repeat" pub status "#bad dictionary" pub status "#bad distance" @@ -36,12 +36,16 @@ pub status "#bad Huffman space" pub status "#bad metablock length" pub status "#bad padding" pub status "#bad reserved bit" +pub status "#bad shared dictionary" pub status "#bad simple Huffman alphabet" pub status "#bad simple Huffman symbol" pub status "#bad transform" pub status "#bad window bits" +pub status "#too many compound dictionaries" +pub status "#too much dictionary data" pub status "#truncated input" pub status "#unsupported large window" +pub status "#unsupported multiple custom static dictionaries" pri status "#internal error: inconsistent bit reader" pri status "#internal error: inconsistent Huffman table" @@ -49,10 +53,33 @@ pri status "#internal error: inconsistent ring buffer" pub const DECODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE : base.u64 = 0 -// DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE is the largest (standard, not -// large-window) brotli window size, (1 << 24) bytes, plus WORKBUF_SLACK. The -// workbuf holds the history ring buffer. -pub const DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE : base.u64 = 0x100_021E // 0x100_0000 + 542. +// DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE is the largest workbuf_len value: +// DICT_TOTAL_MAX bytes of attached dictionaries plus the largest +// (large-window) brotli window size, (1 << 30) bytes, plus WORKBUF_SLACK. +// +// Without attached dictionaries and without the QUIRK_LARGE_WINDOW quirk, +// workbuf_len is DECODER_WORKBUF_LEN_STANDARD, which is much smaller. +pub const DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE : base.u64 = 0xC000_021D // 0x7FFF_FFFF + 0x4000_0000 + 542. + +// DECODER_WORKBUF_LEN_STANDARD is the largest standard (not large-window) +// brotli window size, (1 << 24) bytes, plus WORKBUF_SLACK. This is the +// workbuf_len when no dictionaries are attached and the QUIRK_LARGE_WINDOW +// quirk is not set. +pub const DECODER_WORKBUF_LEN_STANDARD : base.u64 = 0x100_021E // 0x100_0000 + 542. + +// DECODER_WORKBUF_LEN_LARGE_WINDOW is the largest large-window brotli window +// size, (1 << 30) bytes, plus WORKBUF_SLACK. +pub const DECODER_WORKBUF_LEN_LARGE_WINDOW : base.u64 = 0x4000_021E // 0x4000_0000 + 542. + +// DICTIONARY_TYPE_RAW and DICTIONARY_TYPE_SERIALIZED are the values for +// decoder.add_dictionary's dict_type argument. They mirror the C code's +// BROTLI_SHARED_DICTIONARY_RAW and BROTLI_SHARED_DICTIONARY_SERIALIZED. +pub const DICTIONARY_TYPE_RAW : base.u32 = 0 +pub const DICTIONARY_TYPE_SERIALIZED : base.u32 = 1 + +// DICT_TOTAL_MAX is the maximum total length of all attached dictionaries. +// The C code holds the compound dictionaries' total size in an int. +pri const DICT_TOTAL_MAX : base.u64 = 0x7FFF_FFFF // WORKBUF_SLACK mirrors kRingBufferWriteAheadSlack in decode.c. The C code's // fast paths can write (and then discard) up to this many bytes past the end @@ -70,7 +97,9 @@ pri const WORKBUF_SLACK : base.u64 = 542 // or the BROTLI_HUFFMAN_MAX_SIZE_ETC constants in huffman.h. There are at most // 256 trees in each of the literal, insert-and-copy and distance groups. // -// TODO: large-window brotli needs a larger distance stride. +// The distance stride is for the largest distance alphabet_size_limit, which +// is 544 (for large-window brotli, BrotliCalculateDistanceCodeLimit with +// NPOSTFIX = 3 and NDIRECT = 120), not 520 (for standard brotli). pri const LIT_HUFFS_BASE : base.u32 = 0 pri const LIT_HUFFS_STRIDE : base.u32 = 632 // 256 + 376. @@ -79,26 +108,26 @@ pri const CMD_HUFFS_BASE : base.u32 = 161792 // 0 + (256 * 632). pri const CMD_HUFFS_STRIDE : base.u32 = 1080 // 704 + 376. pri const DIST_HUFFS_BASE : base.u32 = 438272 // 161792 + (256 * 1080). -pri const DIST_HUFFS_STRIDE : base.u32 = 896 // 520 + 376. +pri const DIST_HUFFS_STRIDE : base.u32 = 920 // 544 + 376. // BLOCK_TYPE_HUFFS_STRIDE is BROTLI_HUFFMAN_MAX_SIZE_258. -pri const BLOCK_TYPE_HUFFS_BASE : base.u32 = 667648 // 438272 + (256 * 896). +pri const BLOCK_TYPE_HUFFS_BASE : base.u32 = 673792 // 438272 + (256 * 920). pri const BLOCK_TYPE_HUFFS_STRIDE : base.u32 = 632 // BLOCK_LEN_HUFFS_STRIDE is BROTLI_HUFFMAN_MAX_SIZE_26. -pri const BLOCK_LEN_HUFFS_BASE : base.u32 = 669544 // 667648 + (3 * 632). +pri const BLOCK_LEN_HUFFS_BASE : base.u32 = 675688 // 673792 + (3 * 632). pri const BLOCK_LEN_HUFFS_STRIDE : base.u32 = 396 // The context map table's size is BROTLI_HUFFMAN_MAX_SIZE_272. -pri const CONTEXT_MAP_HUFFS_BASE : base.u32 = 670732 // 669544 + (3 * 396). +pri const CONTEXT_MAP_HUFFS_BASE : base.u32 = 676876 // 675688 + (3 * 396). // The code length code (and the code length code length code) tables are 5 // (and 4) bits wide, replicated to fill 256 entries, the same as the 8-bit // root tables for every other code. -pri const CODE_LENGTH_HUFFS_BASE : base.u32 = 671378 // 670732 + 646. -pri const CODE_LENGTH_CODE_HUFFS_BASE : base.u32 = 671634 // 671378 + 256. +pri const CODE_LENGTH_HUFFS_BASE : base.u32 = 677522 // 676876 + 646. +pri const CODE_LENGTH_CODE_HUFFS_BASE : base.u32 = 677778 // 677522 + 256. -pri const HUFFS_LEN : base.u32 = 671890 // 671634 + 256. +pri const HUFFS_LEN : base.u32 = 678034 // 677778 + 256. // The two context maps live in one decoder.context_maps array. The literal // context map (up to 256 block types times 64 contexts) comes first, then the @@ -136,11 +165,12 @@ pub struct decoder? implements base.io_transformer( // rb_flushed is how many of the ring buffer's bytes (in the current // trip around the ring) have been written to args.dst. // - // TODO: large-window brotli allows up to 30 window_bits. - window_bits : base.u32[..= 24], - window_size : base.u32[..= 0x100_0000], - max_backward_distance : base.u32[..= 0x100_0000], - rb_size : base.u32[..= 0x100_0000], + // window_bits is at most 24 for standard brotli and at most 30 for + // large-window brotli. + window_bits : base.u32[..= 30], + window_size : base.u32[..= 0x4000_0000], + max_backward_distance : base.u32[..= 0x4000_0000], + rb_size : base.u32[..= 0x4000_0000], pos : base.u32, rb_flushed : base.u32, @@ -168,9 +198,12 @@ pub struct decoder? implements base.io_transformer( distance_postfix_bits : base.u32[..= 3], num_direct_distance_codes : base.u32[..= 120], - distance_alphabet_size : base.u32[..= 520], - num_literal_htrees : base.u32[..= 256], - num_dist_htrees : base.u32[..= 256], + // distance_alphabet_size is the C code's + // distance_hgroup.alphabet_size_limit: at most 520 for standard + // brotli and at most 544 for large-window brotli. + distance_alphabet_size : base.u32[..= 544], + num_literal_htrees : base.u32[..= 256], + num_dist_htrees : base.u32[..= 256], // The C code's context_map_slice and dist_context_map_slice, as // offsets into decoder.context_maps, plus the context_lookup mode. @@ -197,9 +230,71 @@ pub struct decoder? implements base.io_transformer( insert_length : base.u32, copy_length : base.u32, + // quirk_large_window is the QUIRK_LARGE_WINDOW quirk (the C code's + // BROTLI_DECODER_PARAM_LARGE_WINDOW parameter). large_window is + // whether the stream is a large-window stream (the C code's + // s->large_window after DecodeWindowBits), which needs both the quirk + // and the large-window WBITS escape. + quirk_large_window : base.bool, + large_window : base.bool, + + // started is whether the C code's s->state would no longer be + // BROTLI_STATE_UNINITED. Quirks and dictionaries can only be set or + // added before then. + started : base.bool, + + // dict_total is the total length of all attached dictionaries, which + // live at the start of the workbuf: args.workbuf[.. dict_total]. The + // ring buffer follows: args.workbuf[dict_total ..]. + dict_total : base.u64[..= 0x7FFF_FFFF], + + // The compound dictionary (the C code's BrotliDecoderCompoundDictionary + // in state.h). It has num_chunks chunks. The i'th chunk starts at + // args.workbuf[chunk_starts[i]] and spans compound dictionary + // addresses [chunk_offsets[i] .. chunk_offsets[i + 1]]. + // chunk_offsets[num_chunks] is the C code's total_size. + num_chunks : base.u32[..= 15], + chunk_starts : array[15] base.u32, + chunk_offsets : array[16] base.u32, + + // The shared dictionary's static dictionary part (the C code's + // BrotliSharedDictionary in shared_dictionary_internal.h). The C + // code's words[i] and transforms[i] are given by sd_words[i] and + // sd_transforms[i]: 0 means the built-in (RFC 7932) word list or + // transform list and (1 + j) means the j'th custom list (see the + // wl_etc and tl_etc fields). sd_context_based, sd_context_map and + // sd_num_dictionaries are the C code's context_based, context_map and + // num_dictionaries (except that the initial zero value means one). + // sd_num_word_lists and sd_num_transform_lists are the C code's + // num_word_lists and num_transform_lists. + sd_context_based : base.bool, + sd_num_dictionaries : base.u32[..= 64], + sd_num_word_lists : base.u32[..= 64], + sd_num_transform_lists : base.u32[..= 64], + sd_words : array[64] base.u8[..= 64], + sd_transforms : array[64] base.u8[..= 64], + sd_context_map : array[64] base.u8[..= 63], + + // The prefix (LZ77 dictionary chunk) of a serialized shared + // dictionary, as found by parse_shared_dictionary: its position + // within that dictionary's bytes and its length. + sd_prefix_pos : base.u64, + sd_prefix_size : base.u64, + + // More staging fields for parse_shared_dictionary. See also the + // stage_etc arrays. + stage_num_word_lists : base.u32[..= 64], + stage_num_transform_lists : base.u32[..= 64], + stage_num_dictionaries : base.u32[..= 64], + stage_context_based : base.bool, + + // parse_pos is where parse_shared_dictionary's helper functions + // store the new parsing position. + parse_pos : base.u64, + util : base.utility, ) + ( - huffs : array[671890] base.u16, // HUFFS_LEN. + huffs : array[678034] base.u16, // HUFFS_LEN. context_maps : array[17408] base.u8, // 16384 + 1024. context_modes : array[256] base.u8, @@ -221,16 +316,52 @@ pub struct decoder? implements base.io_transformer( dist_offset : array[1128] base.u32, // transformed_word holds a transformed static dictionary word: a - // prefix (up to 8 bytes), a word (up to 24 bytes) and a suffix (up to - // 8 bytes). - transformed_word : array[64] base.u8, + // prefix (up to 255 bytes), a word (up to 31 bytes) and a suffix (up + // to 255 bytes). Built-in transforms' prefixes and suffixes are at + // most 8 bytes and built-in words are at most 24 bytes. + transformed_word : array[1024] base.u8, + + // Custom word lists (the C code's BrotliDictionary words_instances). + // wl_data is the offset (within the workbuf) of the list's words. + wl_size_bits : array[64] array[32] base.u8, + wl_offsets : array[64] array[32] base.u32, + wl_data : array[64] base.u32, + + // Custom transform lists (the C code's BrotliTransforms + // transforms_instances). tl_prefix_suffix, tl_transforms and + // tl_params are offsets (within the workbuf) of the C code's + // prefix_suffix, transforms and params arrays. tl_map is the C code's + // prefix_suffix_map. + tl_prefix_suffix : array[64] base.u32, + tl_map : array[64] array[256] base.u16, + tl_num_transforms : array[64] base.u32, + tl_transforms : array[64] base.u32, + tl_params : array[64] base.u32, + + // Staging areas for parse_shared_dictionary. They are copied to the + // sd_etc fields only if the whole serialized dictionary is valid. + stage_words : array[64] base.u8, + stage_transforms : array[64] base.u8, + stage_context_map : array[64] base.u8, ) pub func decoder.get_quirk(key: base.u32) base.u64 { + if (args.key == QUIRK_LARGE_WINDOW) and this.quirk_large_window { + return 1 + } return 0 } +// set_quirk mirrors BrotliDecoderSetParameter in decode.c, which fails once +// decoding has started. pub func decoder.set_quirk!(key: base.u32, value: base.u64) base.status { + if args.key == QUIRK_LARGE_WINDOW { + if this.started { + return base."#bad call sequence" + } + this.quirk_large_window = args.value > 0 + return ok + } return base."#unsupported option" } @@ -238,10 +369,19 @@ pub func decoder.dst_history_retain_length() base.optional_u63 { return this.util.make_optional_u63(has_value: true, value: 0) } +// workbuf_len is the total length of the dictionaries added so far (see +// add_dictionary) plus the largest window size (for the QUIRK_LARGE_WINDOW +// quirk's current value) plus WORKBUF_SLACK. pub func decoder.workbuf_len() base.range_ii_u64 { + var n : base.u64[..= 0x4000_021E] + + n = DECODER_WORKBUF_LEN_STANDARD + if this.quirk_large_window { + n = DECODER_WORKBUF_LEN_LARGE_WINDOW + } return this.util.make_range_ii_u64( - min_incl: DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE, - max_incl: DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE) + min_incl: n + this.dict_total, + max_incl: n + this.dict_total) } // transform_io wraps do_transform_io, which decodes into the ring buffer (the @@ -252,17 +392,31 @@ pub func decoder.workbuf_len() base.range_ii_u64 { // write" when it needs every decoded byte flushed before it can wrap around // the ring buffer. If this function can flush all of them, it resumes // do_transform_io immediately instead of suspending. +// +// The first this.dict_total bytes of args.workbuf hold the attached +// dictionaries (see add_dictionary). The ring buffer follows them. pub func decoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf: slice base.u8) { var status : base.status while true { - status =? this.do_transform_io?(src: args.src, workbuf: args.workbuf) - this.flush!(dst: args.dst, workbuf: args.workbuf) + if this.dict_total > args.workbuf.length() { + return base."#bad workbuf length" + } + status =? this.do_transform_io?(src: args.src, + workbuf: args.workbuf[this.dict_total ..], + dictbuf: args.workbuf[.. this.dict_total]) + if this.dict_total > args.workbuf.length() { + return base."#bad workbuf length" + } + this.flush!(dst: args.dst, workbuf: args.workbuf[this.dict_total ..]) if status.is_ok() { while not this.is_fully_flushed() { yield? base."$short write" - this.flush!(dst: args.dst, workbuf: args.workbuf) + if this.dict_total > args.workbuf.length() { + return base."#bad workbuf length" + } + this.flush!(dst: args.dst, workbuf: args.workbuf[this.dict_total ..]) } return ok @@ -310,7 +464,7 @@ pri func decoder.flush!(dst: base.io_writer, workbuf: slice base.u8) { // Mirrors the BROTLI_STATE_ETC state machine in BrotliDecoderDecompressStream // in decode.c. -pri func decoder.do_transform_io?(src: base.io_reader, workbuf: slice base.u8) { +pri func decoder.do_transform_io?(src: base.io_reader, workbuf: slice base.u8, dictbuf: roslice base.u8) { var w : base.u64 var aligned : base.bool @@ -358,7 +512,7 @@ pri func decoder.do_transform_io?(src: base.io_reader, workbuf: slice base.u8) { // BROTLI_STATE_UNCOMPRESSED. this.copy_uncompressed?(src: args.src, workbuf: args.workbuf) } else { - this.decode_compressed_metablock?(src: args.src, workbuf: args.workbuf) + this.decode_compressed_metablock?(src: args.src, workbuf: args.workbuf, dictbuf: args.dictbuf) } } @@ -386,6 +540,7 @@ pri func decoder.initialize_codes!() { this.bits = 0 this.n_bits = 0 + this.large_window = false this.pos = 0 this.rb_size = 0 this.rb_flushed = 0 @@ -452,6 +607,23 @@ pri func decoder.read_bits?(src: base.io_reader, n: base.u32[..= 24]) { this.n_bits = n_bits - n } +// read_bits32 is like read_bits but args.n can be up to 32. Mirrors +// BrotliSafeReadBits32 in bit_reader.h. +pri func decoder.read_bits32?(src: base.io_reader, n: base.u32[..= 32]) { + var n : base.u32[..= 32] + var lo : base.u32 + + n = args.n + if n <= 24 { + this.read_bits?(src: args.src, n: n) + return ok + } + this.read_bits?(src: args.src, n: 16) + lo = this.bits_value & 0xFFFF + this.read_bits?(src: args.src, n: n - 16) + this.bits_value = lo | (this.bits_value ~mod<< 16) +} + // read_symbol reads one Huffman-encoded symbol, decoded by the table at // this.huffs[args.table ..], and stores it in this.symbol. It reads input // bytes only when needed. Mirrors SafeReadSymbol and SafeDecodeSymbol in @@ -521,9 +693,11 @@ pri func decoder.jump_to_byte_boundary!() base.bool { return pad == 0 } -// Mirrors DecodeWindowBits in decode.c. +// Mirrors DecodeWindowBits in decode.c, and the BROTLI_STATE_UNINITED and +// BROTLI_STATE_LARGE_WINDOW_BITS states of BrotliDecoderDecompressStream. pri func decoder.decode_window_bits?(src: base.io_reader) { var n : base.u32[..= 7] + var w : base.u32[..= 63] this.read_bits?(src: args.src, n: 1) if this.bits_value == 0 { @@ -537,12 +711,30 @@ pri func decoder.decode_window_bits?(src: base.io_reader) { this.read_bits?(src: args.src, n: 3) n = this.bits_value & 7 if n == 1 { - // The C code would read a "Large Window Brotli" header, if - // BROTLI_DECODER_PARAM_LARGE_WINDOW was set, or otherwise fail - // with BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS. - // - // TODO: support large-window brotli. - return "#unsupported large window" + // The "Large Window Brotli" escape. Without the + // QUIRK_LARGE_WINDOW quirk (the C code's + // BROTLI_DECODER_PARAM_LARGE_WINDOW), the C code fails with + // BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS. + if not this.quirk_large_window { + return "#unsupported large window" + } + this.read_bits?(src: args.src, n: 1) + if this.bits_value == 1 { + return "#bad window bits" + } + // The C code is now in the BROTLI_STATE_LARGE_WINDOW_BITS + // state. + this.started = true + this.large_window = true + this.read_bits?(src: args.src, n: 6) + w = this.bits_value & 63 + if (w < 10) or (30 < w) { + return "#bad window bits" + } + this.window_bits = w + this.window_size = (1 as base.u32) << w + this.max_backward_distance = this.window_size - 16 + return ok } else if n <> 0 { this.window_bits = 8 + n } else { @@ -553,6 +745,8 @@ pri func decoder.decode_window_bits?(src: base.io_reader) { if this.window_bits < 10 { return "#bad window bits" } + // The C code is now in the BROTLI_STATE_INITIALIZE state. + this.started = true this.window_size = (1 as base.u32) << this.window_bits // Maximum distance, see section 9.1. of the spec. BROTLI_WINDOW_GAP is 16. this.max_backward_distance = this.window_size - 16 diff --git a/std/brotli/decode_dictionary.wuffs b/std/brotli/decode_dictionary.wuffs new file mode 100644 index 00000000..65fcfd51 --- /dev/null +++ b/std/brotli/decode_dictionary.wuffs @@ -0,0 +1,955 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// -------- + +// add_dictionary attaches a dictionary, like BrotliDecoderAttachDictionary in +// decode.c. It must be called before decoding starts (before the first +// transform_io call that consumes any input). +// +// args.dict_type is DICTIONARY_TYPE_RAW (a raw LZ77 prefix dictionary) or +// DICTIONARY_TYPE_SERIALIZED (a serialized shared dictionary). +// +// Wuffs structs cannot hold pointers, so the dictionary's bytes are copied +// into the workbuf: they are appended to the dictionaries added so far, which +// live at the start of the workbuf. args.workbuf must therefore be at least +// as long as the total length of all dictionaries added so far, including +// this one, and every later add_dictionary and transform_io call must be +// passed the same workbuf (or at least one whose prefix holds the same +// bytes). After the last add_dictionary call, workbuf_len gives the workbuf +// length that transform_io needs. +// +// On failure, the decoder's state is unchanged. The C code can leave its +// state partially updated when BrotliDecoderAttachDictionary fails. +pub func decoder.add_dictionary!(dict_type: base.u32, dict: roslice base.u8, workbuf: slice base.u8) base.status { + var status : base.status + var n : base.u64 + var lo : base.u64 + var hi : base.u64 + var i : base.u32 + + if this.started { + return base."#bad call sequence" + } + n = args.dict.length() + + if args.dict_type == DICTIONARY_TYPE_RAW { + // Mirrors the BROTLI_SHARED_DICTIONARY_RAW case of + // BrotliSharedDictionaryAttach in shared_dictionary.c and + // AttachCompoundDictionary in decode.c. + if this.num_chunks >= 15 { + return "#too many compound dictionaries" + } + } else if args.dict_type == DICTIONARY_TYPE_SERIALIZED { + // Mirrors DecodeSharedDictionary in shared_dictionary.c. + status = this.parse_shared_dictionary!(dict: args.dict, base_offset: this.dict_total) + if not status.is_ok() { + return status + } + } else { + return base."#bad argument" + } + + // Copy the dictionary bytes into the workbuf. + if n > DICT_TOTAL_MAX { + return "#too much dictionary data" + } + lo = this.dict_total + hi = lo + n + if hi > DICT_TOTAL_MAX { + return "#too much dictionary data" + } else if hi > args.workbuf.length() { + return base."#bad workbuf length" + } + if lo > hi { + return "#too much dictionary data" + } + args.workbuf[lo .. hi].copy_from_slice!(s: args.dict) + + // Commit. + if args.dict_type == DICTIONARY_TYPE_RAW { + this.add_chunk!(start: lo, size: n) + + } else { + if this.sd_prefix_size > 0 { + this.add_chunk!(start: lo ~sat+ this.sd_prefix_pos, size: this.sd_prefix_size) + } + // Mirrors the end of ParseDictionary in shared_dictionary.c. If the + // serialized dictionary has no custom word lists or transform lists, + // the static dictionary is reset to the built-in one, even if an + // earlier serialized dictionary had custom lists. + this.sd_num_word_lists = this.stage_num_word_lists + this.sd_num_transform_lists = this.stage_num_transform_lists + if (this.stage_num_word_lists > 0) or (this.stage_num_transform_lists > 0) { + this.sd_context_based = this.stage_context_based + this.sd_num_dictionaries = this.stage_num_dictionaries + i = 0 + while i < 64 { + this.sd_words[i] = this.stage_words[i].min(no_more_than: 64) + this.sd_transforms[i] = this.stage_transforms[i].min(no_more_than: 64) + this.sd_context_map[i] = this.stage_context_map[i] & 0x3F + i += 1 + } + } else { + this.sd_context_based = false + this.sd_num_dictionaries = 1 + this.sd_words[0] = 0 + this.sd_transforms[0] = 0 + } + } + + if hi <= DICT_TOTAL_MAX { + this.dict_total = hi + } + return ok +} + +// add_chunk adds a chunk to the compound dictionary. Mirrors +// AttachCompoundDictionary in decode.c. +pri func decoder.add_chunk!(start: base.u64, size: base.u64) { + var k : base.u32[..= 15] + + k = this.num_chunks + if k >= 15 { + return nothing + } + this.chunk_starts[k] = (args.start & 0xFFFF_FFFF) as base.u32 + this.chunk_offsets[k + 1] = this.chunk_offsets[k] ~mod+ ((args.size & 0xFFFF_FFFF) as base.u32) + this.num_chunks = k + 1 +} + +// parse_shared_dictionary parses and validates a serialized shared +// dictionary. Mirrors DecodeSharedDictionary, DryParseDictionary and +// ParseDictionary in shared_dictionary.c. +// +// It does not change the decoder's sd_etc fields, only the stage_etc fields +// (and the wl_etc and tl_etc fields, but only when there are no custom word +// lists or transform lists in use). The caller commits them. +// +// args.base_offset is where the serialized dictionary's bytes will be copied +// to, within the workbuf. +pri func decoder.parse_shared_dictionary!(dict: roslice base.u8, base_offset: base.u64) base.status { + var pos : base.u64 + var chunk_size : base.u64 + var prefix_pos : base.u64 + var num_prefix : base.u32[..= 1] + var nwl : base.u32[..= 0xFF] + var ntl : base.u32[..= 0xFF] + var ndict : base.u32[..= 0xFF] + var is_custom : base.bool + var has_custom : base.bool + var i : base.u32 + var v : base.u32[..= 0xFF] + var v2 : base.u32[..= 0xFF] + var status : base.status + + // Check the magic header bytes. + if args.dict.length() < 2 { + return "#bad shared dictionary" + } else if (args.dict[0] <> 0x91) or (args.dict[1] <> 0x00) { + return "#bad shared dictionary" + } + + // DryParseDictionary. + pos = 2 + // LZ77_DICTIONARY_LENGTH. Mirrors ReadVarint32. + chunk_size = this.read_varint32!(dict: args.dict, pos: pos) + if chunk_size >= 0x1_0000_0000 { + return "#bad shared dictionary" + } + pos = this.parse_pos + prefix_pos = pos + num_prefix = 0 + if chunk_size <> 0 { + // "This limitation is not specified but the 32-bit Brotli decoder for + // now". + if chunk_size > 1073_741823 { + return "#bad shared dictionary" + } + num_prefix = 1 + if (pos ~sat+ chunk_size) > args.dict.length() { + return "#bad shared dictionary" + } + pos ~sat+= chunk_size + } + // NUM_WORD_LISTS and NUM_TRANSFORM_LISTS. + if (pos ~sat+ 2) > args.dict.length() { + return "#bad shared dictionary" + } + is_custom = false + if pos < args.dict.length() { + if args.dict[pos] > 0 { + is_custom = true + } + } + if (pos ~sat+ 1) < args.dict.length() { + if args.dict[pos ~sat+ 1] > 0 { + is_custom = true + } + } + + if (num_prefix + this.num_chunks) > 15 { + return "#too many compound dictionaries" + } + // "Cannot combine different static dictionaries, only prefix + // dictionaries". + has_custom = (this.sd_num_word_lists > 0) or (this.sd_num_transform_lists > 0) + if has_custom and is_custom { + return "#unsupported multiple custom static dictionaries" + } + + // ParseDictionary. The LZ77_DICTIONARY_LENGTH bytes were checked above. + + // NUM_WORD_LISTS. + if pos >= args.dict.length() { + return "#bad shared dictionary" + } + nwl = args.dict[pos] as base.u32 + pos ~sat+= 1 + if nwl > 64 { + return "#bad shared dictionary" + } + i = 0 + while i < nwl, + inv nwl <= 64, + { + assert i < 64 via "a < b: a < c; c <= b"(c: nwl) + status = this.parse_word_list!(dict: args.dict, pos: pos, base_offset: args.base_offset, list: i) + if not status.is_ok() { + return status + } + pos = this.parse_pos + i += 1 + } + + // NUM_TRANSFORM_LISTS. + if pos >= args.dict.length() { + return "#bad shared dictionary" + } + ntl = args.dict[pos] as base.u32 + pos ~sat+= 1 + if ntl > 64 { + return "#bad shared dictionary" + } + i = 0 + while i < ntl, + inv ntl <= 64, + { + assert i < 64 via "a < b: a < c; c <= b"(c: ntl) + status = this.parse_transforms_list!(dict: args.dict, pos: pos, base_offset: args.base_offset, list: i) + if not status.is_ok() { + return status + } + pos = this.parse_pos + i += 1 + } + + this.stage_num_word_lists = nwl.min(no_more_than: 64) + this.stage_num_transform_lists = ntl.min(no_more_than: 64) + this.stage_num_dictionaries = 1 + this.stage_context_based = false + i = 0 + while i < 64 { + this.stage_words[i] = 0 + this.stage_transforms[i] = 0 + this.stage_context_map[i] = 0 + i += 1 + } + + if (nwl <> 0) or (ntl <> 0) { + // NUM_DICTIONARIES. + if pos >= args.dict.length() { + return "#bad shared dictionary" + } + ndict = args.dict[pos] as base.u32 + pos ~sat+= 1 + if (ndict == 0) or (ndict > 64) { + return "#bad shared dictionary" + } + this.stage_num_dictionaries = ndict.min(no_more_than: 64) + i = 0 + while i < ndict, + inv ndict <= 64, + { + assert i < 64 via "a < b: a < c; c <= b"(c: ndict) + if pos >= args.dict.length() { + return "#bad shared dictionary" + } + v = args.dict[pos] as base.u32 + pos ~sat+= 1 + if v > nwl { + return "#bad shared dictionary" + } + if pos >= args.dict.length() { + return "#bad shared dictionary" + } + v2 = args.dict[pos] as base.u32 + pos ~sat+= 1 + if v2 > ntl { + return "#bad shared dictionary" + } + // Index nwl (or ntl) means the built-in list, which this.sd_words + // (or this.sd_transforms) represents as 0. + if v == nwl { + this.stage_words[i] = 0 + } else { + this.stage_words[i] = ((v & 0x3F) + 1) as base.u8 + } + if v2 == ntl { + this.stage_transforms[i] = 0 + } else { + this.stage_transforms[i] = ((v2 & 0x3F) + 1) as base.u8 + } + i += 1 + } + + // CONTEXT_ENABLED. Mirrors ReadBool. + if pos >= args.dict.length() { + return "#bad shared dictionary" + } + v = args.dict[pos] as base.u32 + pos ~sat+= 1 + if v > 1 { + return "#bad shared dictionary" + } + this.stage_context_based = v == 1 + + // CONTEXT_MAP. + if this.stage_context_based { + i = 0 + while i < 64 { + if pos >= args.dict.length() { + return "#bad shared dictionary" + } + v = args.dict[pos] as base.u32 + pos ~sat+= 1 + if v >= ndict { + return "#bad shared dictionary" + } + this.stage_context_map[i] = (v & 0x3F) as base.u8 + i += 1 + } + } + } + + this.sd_prefix_pos = prefix_pos + this.sd_prefix_size = chunk_size + return ok +} + +// read_varint32 reads a varint at args.dict[args.pos ..]. It returns a value +// of 0x1_0000_0000 or more on failure. On success, it returns the value and +// stores the new position in this.parse_pos. Mirrors ReadVarint32 in +// shared_dictionary.c. +pri func decoder.read_varint32!(dict: roslice base.u8, pos: base.u64) base.u64 { + var pos : base.u64 + var num : base.u32[..= 4] + var b : base.u64 + var result : base.u64 + + pos = args.pos + result = 0 + num = 0 + while true { + if pos >= args.dict.length() { + return 0x1_0000_0000 + } + b = args.dict[pos] as base.u64 + pos ~sat+= 1 + if (num == 4) and (b > 15) { + return 0x1_0000_0000 + } + result |= (b & 127) << (7 * num) + if b < 128 { + break + } + if num >= 4 { + return 0x1_0000_0000 + } + num += 1 + } + this.parse_pos = pos + return result & 0xFFFF_FFFF +} + +// parse_word_list parses the args.list'th custom word list, at +// args.dict[args.pos ..]. On success, it stores the new position in +// this.parse_pos. Mirrors ParseWordList and BrotliSizeBitsToOffsets in +// shared_dictionary.c. +pri func decoder.parse_word_list!(dict: roslice base.u8, pos: base.u64, base_offset: base.u64, list: base.u32[..= 63]) base.status { + var pos : base.u64 + var len : base.u32 + var v : base.u8 + var offset : base.u32 + var wpos : base.u64 + + pos = args.pos + if (pos ~sat+ 28) > args.dict.length() { + return "#bad shared dictionary" + } + len = 0 + while len < 32 { + v = 0 + if len >= 4 { + if (pos ~sat+ ((len - 4) as base.u64)) < args.dict.length() { + v = args.dict[pos ~sat+ ((len - 4) as base.u64)] + } + // BROTLI_MAX_SIZE_BITS. + if v > 15 { + return "#bad shared dictionary" + } + } + this.wl_size_bits[args.list][len] = v + len += 1 + } + pos ~sat+= 28 + + offset = 0 + len = 0 + while len < 32 { + this.wl_offsets[args.list][len] = offset + v = this.wl_size_bits[args.list][len] + if v <> 0 { + offset ~mod+= len << (v & 15) + } + len += 1 + } + + wpos = args.base_offset ~sat+ pos + this.wl_data[args.list] = (wpos & 0xFFFF_FFFF) as base.u32 + pos ~sat+= offset as base.u64 + if pos > args.dict.length() { + return "#bad shared dictionary" + } + this.parse_pos = pos + return ok +} + +// parse_transforms_list parses the args.list'th custom transform list, at +// args.dict[args.pos ..]. On success, it stores the new position in +// this.parse_pos. Mirrors ParseTransformsList and ParsePrefixSuffixTable +// in shared_dictionary.c. +pri func decoder.parse_transforms_list!(dict: roslice base.u8, pos: base.u64, base_offset: base.u64, list: base.u32[..= 63]) base.status { + var pos : base.u64 + var data_length : base.u64 + var ps_start : base.u64 + var offset : base.u64 + var stringlet_len : base.u64 + var count : base.u32[..= 256] + var nt : base.u32[..= 0xFF] + var t : base.u32 + var tpos : base.u64 + var j : base.u64 + var prefix_id : base.u32[..= 0xFF] + var t_type : base.u32[..= 0xFF] + var suffix_id : base.u32[..= 0xFF] + var has_params : base.bool + var p0 : base.u32 + var p1 : base.u32 + + pos = args.pos + if pos >= args.dict.length() { + return "#bad shared dictionary" + } + + // ParsePrefixSuffixTable. PREFIX_SUFFIX_LENGTH. + if (pos ~sat+ 2) > args.dict.length() { + return "#bad shared dictionary" + } + data_length = (this.byte_at(s: args.dict, i: pos) as base.u64) | + ((this.byte_at(s: args.dict, i: pos ~sat+ 1) as base.u64) << 8) + pos ~sat+= 2 + // It must at least have space for the NUL terminator. + if data_length < 1 { + return "#bad shared dictionary" + } + ps_start = pos + if (pos ~sat+ data_length) >= args.dict.length() { + return "#bad shared dictionary" + } + offset = 0 + count = 0 + while true { + // STRING_LENGTH. + j = ps_start ~sat+ offset + stringlet_len = 0 + if j < args.dict.length() { + stringlet_len = args.dict[j] as base.u64 + } + if count >= 256 { + return "#bad shared dictionary" + } + this.tl_map[args.list][count] = (offset & 0xFFFF) as base.u16 + count += 1 + offset ~sat+= 1 + if stringlet_len == 0 { + if offset == data_length { + break + } + return "#bad shared dictionary" + } + if count > 255 { + return "#bad shared dictionary" + } + offset ~sat+= stringlet_len + if offset >= data_length { + return "#bad shared dictionary" + } + } + pos ~sat+= data_length + this.tl_prefix_suffix[args.list] = ((args.base_offset ~sat+ ps_start) & 0xFFFF_FFFF) as base.u32 + + // NUM_TRANSFORMS. The "(pos + data_length) >= length" check above means + // that this is in bounds. + nt = 0 + if pos < args.dict.length() { + nt = args.dict[pos] as base.u32 + } + pos ~sat+= 1 + this.tl_num_transforms[args.list] = nt + tpos = pos + this.tl_transforms[args.list] = ((args.base_offset ~sat+ tpos) & 0xFFFF_FFFF) as base.u32 + pos ~sat+= (nt as base.u64) * 3 + if pos > args.dict.length() { + return "#bad shared dictionary" + } + + // Check for errors and read extra parameters. + has_params = false + t = 0 + while t < nt { + j = tpos ~sat+ ((t as base.u64) * 3) + prefix_id = this.byte_at(s: args.dict, i: j) + t_type = this.byte_at(s: args.dict, i: j ~sat+ 1) + suffix_id = this.byte_at(s: args.dict, i: j ~sat+ 2) + if prefix_id >= count { + return "#bad shared dictionary" + } else if t_type >= 23 { + // BROTLI_NUM_TRANSFORM_TYPES. + return "#bad shared dictionary" + } else if suffix_id >= count { + return "#bad shared dictionary" + } + // BROTLI_TRANSFORM_SHIFT_FIRST or BROTLI_TRANSFORM_SHIFT_ALL. + if (t_type == 21) or (t_type == 22) { + has_params = true + } + t ~mod+= 1 + } + + this.tl_params[args.list] = 0 + if has_params { + this.tl_params[args.list] = ((args.base_offset ~sat+ pos) & 0xFFFF_FFFF) as base.u32 + j = pos + pos ~sat+= (nt as base.u64) * 2 + if pos > args.dict.length() { + return "#bad shared dictionary" + } + t = 0 + while t < nt { + t_type = this.byte_at(s: args.dict, i: tpos ~sat+ (((t as base.u64) * 3) + 1)) + if (t_type <> 21) and (t_type <> 22) { + p0 = this.byte_at(s: args.dict, i: j ~sat+ ((t as base.u64) * 2)) + p1 = this.byte_at(s: args.dict, i: j ~sat+ (((t as base.u64) * 2) + 1)) + if (p0 <> 0) or (p1 <> 0) { + return "#bad shared dictionary" + } + } + t ~mod+= 1 + } + } + this.parse_pos = pos + return ok +} + +// -------- + +// copy_dictionary_word writes a transformed static dictionary word to +// this.transformed_word and returns its length. It returns 0xFFFF_FFFE for a +// BROTLI_DECODER_ERROR_FORMAT_DICTIONARY error and 0xFFFF_FFFF for a +// BROTLI_DECODER_ERROR_FORMAT_TRANSFORM error. args.address is the C code's +// (distance_code - max_distance - 1 - compound_dictionary_size) and args.ctx +// is the C code's BROTLI_CONTEXT(p1, p2, s->context_lookup). +// +// Mirrors the static dictionary part of ProcessCommandsInternal in decode.c, +// including its selection of a word list and transform list from the shared +// dictionary. +pri func decoder.copy_dictionary_word!(dictbuf: roslice base.u8, address: base.u32, len: base.u32[..= 31], ctx: base.u32[..= 63]) base.u32 { + var dict_id : base.u32[..= 63] + var wsel : base.u32[..= 64] + var tsel : base.u32[..= 64] + var shift : base.u32[..= 24] + var nt : base.u32[..= 255] + var address : base.u32 + var transform_idx : base.u32 + var word_idx : base.u32 + var offset : base.u32 + var dist_remaining : base.u32 + var d2 : base.u32 + var w2 : base.u32[..= 64] + var t2 : base.u32[..= 64] + var shift2 : base.u32[..= 24] + var num : base.u32 + + dict_id = 0 + if this.sd_context_based { + dict_id = (this.sd_context_map[args.ctx] & 0x3F) as base.u32 + } + wsel = this.sd_words[dict_id] as base.u32 + tsel = this.sd_transforms[dict_id] as base.u32 + shift = this.word_list_size_bits(wsel: wsel, len: args.len) + nt = this.transform_list_length(tsel: tsel) + address = args.address + transform_idx = address >> shift + + // If the distance is out of bound, select a next static dictionary if + // there exist multiple. + if ((transform_idx >= nt) or (shift == 0)) and (this.sd_num_dictionaries > 1) { + dist_remaining = address ~mod- ((((1 as base.u32) << shift) & 0xFFFF_FFFE) ~mod* nt) + d2 = 0 + while d2 < this.sd_num_dictionaries { + assert d2 < 64 via "a < b: a < c; c <= b"(c: this.sd_num_dictionaries) + w2 = this.sd_words[d2] as base.u32 + shift2 = this.word_list_size_bits(wsel: w2, len: args.len) + if (d2 <> dict_id) and (shift2 <> 0) { + t2 = this.sd_transforms[d2] as base.u32 + num = (((1 as base.u32) << shift2) & 0xFFFF_FFFE) ~mod* this.transform_list_length(tsel: t2) + if dist_remaining < num { + wsel = w2 + tsel = t2 + address = dist_remaining + shift = shift2 + nt = this.transform_list_length(tsel: tsel) + transform_idx = address >> shift + break + } + dist_remaining -= num + } + d2 += 1 + } + } + + if shift == 0 { + return 0xFFFF_FFFE + } else if transform_idx >= nt { + return 0xFFFF_FFFF + } + word_idx = address & (((1 as base.u32) << shift) - 1) + offset = this.word_list_offset(wsel: wsel, len: args.len) ~mod+ (word_idx ~mod* args.len) + num = this.transform_dictionary_word!(dictbuf: args.dictbuf, + wsel: wsel, + tsel: tsel, + offset: offset, + len: args.len, + transform_idx: transform_idx.min(no_more_than: 254)) + return num +} + +// word_list_size_bits returns the C code's size_bits_by_length[args.len] for +// the word list selected by args.wsel: 0 for the built-in list or (1 + j) for +// the j'th custom list. +pri func decoder.word_list_size_bits(wsel: base.u32[..= 64], len: base.u32[..= 31]) base.u32[..= 24] { + if args.wsel == 0 { + return DICTIONARY_SIZE_BITS_BY_LENGTH[args.len] + } + return (this.wl_size_bits[(args.wsel - 1) & 63][args.len] & 15) as base.u32 +} + +// word_list_offset returns the C code's offsets_by_length[args.len] for the +// word list selected by args.wsel. +pri func decoder.word_list_offset(wsel: base.u32[..= 64], len: base.u32[..= 31]) base.u32 { + if args.wsel == 0 { + return DICTIONARY_OFFSETS_BY_LENGTH[args.len] + } + return this.wl_offsets[(args.wsel - 1) & 63][args.len] +} + +// transform_list_length returns the C code's num_transforms for the +// transform list selected by args.tsel: 0 for the built-in list or (1 + j) +// for the j'th custom list. +pri func decoder.transform_list_length(tsel: base.u32[..= 64]) base.u32[..= 255] { + if args.tsel == 0 { + return NUM_TRANSFORMS + } + return this.tl_num_transforms[(args.tsel - 1) & 63].min(no_more_than: 255) +} + +// transform_dictionary_word writes the transformed word to +// this.transformed_word and returns its length. Mirrors +// BrotliTransformDictionaryWord in transform.c. +pri func decoder.transform_dictionary_word!(dictbuf: roslice base.u8, wsel: base.u32[..= 64], tsel: base.u32[..= 64], offset: base.u32, len: base.u32[..= 31], transform_idx: base.u32[..= 254]) base.u32 { + var idx : base.u32 + var t : base.u32[..= 120] + var k : base.u32[..= 63] + var tpos : base.u64 + var prefix_id : base.u32[..= 0xFF] + var t_type : base.u32[..= 0xFF] + var suffix_id : base.u32[..= 0xFF] + var param : base.u32 + var word_start : base.u32 + var word_len : base.u32 + var skip : base.u32[..= 9] + var i : base.u32 + var j : base.u64 + var remaining : base.u32 + var step : base.u32 + + // Look up the transform's prefix_id, type and suffix_id. + k = 0 + tpos = 0 + if args.tsel == 0 { + t = args.transform_idx.min(no_more_than: 120) + prefix_id = TRANSFORM_PREFIX_ID[t] as base.u32 + t_type = TRANSFORM_TYPE[t] as base.u32 + suffix_id = TRANSFORM_SUFFIX_ID[t] as base.u32 + } else { + k = (args.tsel - 1) & 63 + tpos = (this.tl_transforms[k] as base.u64) + ((args.transform_idx as base.u64) * 3) + prefix_id = this.byte_at(s: args.dictbuf, i: tpos) + t_type = this.byte_at(s: args.dictbuf, i: tpos + 1) + suffix_id = this.byte_at(s: args.dictbuf, i: tpos + 2) + } + + // The prefix. + idx = this.append_affix!(dictbuf: args.dictbuf, tsel: args.tsel, id: prefix_id, idx: 0) + + // The word, possibly with its first or last few bytes omitted. + word_len = args.len + skip = 0 + if t_type <= 9 { + // BROTLI_TRANSFORM_OMIT_LAST_N. + if t_type < args.len { + word_len = args.len ~mod- t_type + } else { + word_len = 0 + } + } else if (12 <= t_type) and (t_type <= 20) { + // BROTLI_TRANSFORM_OMIT_FIRST_N. + skip = t_type - 11 + if skip < args.len { + word_len = args.len ~mod- skip + } else { + word_len = 0 + } + } + word_start = idx + i = 0 + while i < word_len { + if idx >= 1024 { + break + } + if args.wsel == 0 { + j = ((args.offset ~mod+ skip) ~mod+ i) as base.u64 + if j < 122784 { + this.transformed_word[idx] = DICTIONARY[j] + } + } else { + j = (this.wl_data[(args.wsel - 1) & 63] as base.u64) + + (((args.offset ~mod+ skip) ~mod+ i) as base.u64) + if j < args.dictbuf.length() { + this.transformed_word[idx] = args.dictbuf[j] + } + } + idx ~mod+= 1 + i ~mod+= 1 + } + + // Upper-casing and shifting. The C code's ToUpperCase can modify bytes + // after the word's end, but those bytes are either overwritten by the + // suffix or are past the transformed word's length, so we don't bother. + if t_type == 10 { + // BROTLI_TRANSFORM_UPPERCASE_FIRST. + this.to_upper_case!(p: word_start, end: idx) + } else if t_type == 11 { + // BROTLI_TRANSFORM_UPPERCASE_ALL. + i = word_start + while i < idx { + step = this.to_upper_case!(p: i, end: idx) + i ~mod+= step + } + } else if (t_type == 21) or (t_type == 22) { + // BROTLI_TRANSFORM_SHIFT_FIRST or BROTLI_TRANSFORM_SHIFT_ALL. Only + // custom transforms have these types, and their parameters exist. + j = (this.tl_params[k] as base.u64) + ((args.transform_idx as base.u64) * 2) + param = this.byte_at(s: args.dictbuf, i: j) | + (this.byte_at(s: args.dictbuf, i: j + 1) << 8) + if t_type == 21 { + this.shift!(p: word_start, word_len: idx ~mod- word_start, param: param) + } else { + i = word_start + remaining = idx ~mod- word_start + while remaining > 0 { + step = this.shift!(p: i, word_len: remaining, param: param) + if step >= remaining { + break + } + i ~mod+= step + remaining ~mod-= step + } + } + } + + // The suffix. + idx = this.append_affix!(dictbuf: args.dictbuf, tsel: args.tsel, id: suffix_id, idx: idx) + return idx +} + +// append_affix appends the args.id'th prefix or suffix of the transform list +// selected by args.tsel to this.transformed_word, starting at args.idx. It +// returns the new length. +pri func decoder.append_affix!(dictbuf: roslice base.u8, tsel: base.u32[..= 64], id: base.u32[..= 0xFF], idx: base.u32) base.u32 { + var idx : base.u32 + var affix : base.u32[..= 216] + var affix_len : base.u32 + var i : base.u32 + var j : base.u32 + var k : base.u32[..= 63] + var pos : base.u64 + + idx = args.idx + if args.tsel == 0 { + affix = PREFIX_SUFFIX_MAP[args.id.min(no_more_than: 49)] + affix_len = PREFIX_SUFFIX[affix] as base.u32 + i = 0 + while i < affix_len { + j = (affix ~mod+ 1) ~mod+ i + if (j >= 217) or (idx >= 1024) { + break + } + this.transformed_word[idx] = PREFIX_SUFFIX[j] + idx ~mod+= 1 + i ~mod+= 1 + } + return idx + } + + k = (args.tsel - 1) & 63 + pos = (this.tl_prefix_suffix[k] as base.u64) + (this.tl_map[k][args.id] as base.u64) + affix_len = 0 + if pos < args.dictbuf.length() { + affix_len = args.dictbuf[pos] as base.u32 + } + i = 0 + while i < affix_len { + pos ~mod+= 1 + if (pos >= args.dictbuf.length()) or (idx >= 1024) { + break + } + this.transformed_word[idx] = args.dictbuf[pos] + idx ~mod+= 1 + i ~mod+= 1 + } + return idx +} + +// shift applies a shift transform to the (UTF-8 encoded) character at +// this.transformed_word[args.p ..], which is args.word_len bytes long. It +// returns the step: the number of bytes in the character. Mirrors Shift in +// transform.c. +pri func decoder.shift!(p: base.u32, word_len: base.u32, param: base.u32) base.u32 { + var p : base.u32[..= 1019] + var scalar : base.u32 + var c0 : base.u32 + var c1 : base.u32 + var c2 : base.u32 + var c3 : base.u32 + + if args.p > 1019 { + return 1 + } + p = args.p + // Limited sign extension: scalar < (1 << 24). + scalar = (args.param & 0x7FFF) ~mod+ (0x100_0000 ~mod- (args.param & 0x8000)) + c0 = this.transformed_word[p] as base.u32 + if c0 < 0x80 { + // 1-byte rune / 0sssssss / 7 bit scalar (ASCII). + scalar ~mod+= c0 + this.transformed_word[p] = (scalar & 0x7F) as base.u8 + return 1 + } else if c0 < 0xC0 { + // Continuation / 10AAAAAA. + return 1 + } else if c0 < 0xE0 { + // 2-byte rune / 110sssss AAssssss / 11 bit scalar. + if args.word_len < 2 { + return 1 + } + c1 = this.transformed_word[p + 1] as base.u32 + scalar ~mod+= (c1 & 0x3F) | ((c0 & 0x1F) << 6) + this.transformed_word[p + 0] = (0xC0 | ((scalar >> 6) & 0x1F)) as base.u8 + this.transformed_word[p + 1] = ((c1 & 0xC0) | (scalar & 0x3F)) as base.u8 + return 2 + } else if c0 < 0xF0 { + // 3-byte rune / 1110ssss AAssssss BBssssss / 16 bit scalar. + if args.word_len < 3 { + return args.word_len + } + c1 = this.transformed_word[p + 1] as base.u32 + c2 = this.transformed_word[p + 2] as base.u32 + scalar ~mod+= (c2 & 0x3F) | ((c1 & 0x3F) << 6) | ((c0 & 0x0F) << 12) + this.transformed_word[p + 0] = (0xE0 | ((scalar >> 12) & 0x0F)) as base.u8 + this.transformed_word[p + 1] = ((c1 & 0xC0) | ((scalar >> 6) & 0x3F)) as base.u8 + this.transformed_word[p + 2] = ((c2 & 0xC0) | (scalar & 0x3F)) as base.u8 + return 3 + } else if c0 < 0xF8 { + // 4-byte rune / 11110sss AAssssss BBssssss CCssssss / 21 bit scalar. + if args.word_len < 4 { + return args.word_len + } + c1 = this.transformed_word[p + 1] as base.u32 + c2 = this.transformed_word[p + 2] as base.u32 + c3 = this.transformed_word[p + 3] as base.u32 + scalar ~mod+= (c3 & 0x3F) | ((c2 & 0x3F) << 6) | ((c1 & 0x3F) << 12) | ((c0 & 0x07) << 18) + this.transformed_word[p + 0] = (0xF0 | ((scalar >> 18) & 0x07)) as base.u8 + this.transformed_word[p + 1] = ((c1 & 0xC0) | ((scalar >> 12) & 0x3F)) as base.u8 + this.transformed_word[p + 2] = ((c2 & 0xC0) | ((scalar >> 6) & 0x3F)) as base.u8 + this.transformed_word[p + 3] = ((c3 & 0xC0) | (scalar & 0x3F)) as base.u8 + return 4 + } + return 1 +} + +// Mirrors ToUpperCase in transform.c, except that it does not modify bytes at +// or after args.end. It returns the step: the number of bytes in the +// (overly simplified model of a) UTF-8 encoded character. +pri func decoder.to_upper_case!(p: base.u32, end: base.u32) base.u32 { + var c : base.u8 + + if args.p >= 1024 { + return 1 + } + c = this.transformed_word[args.p] + if c < 0xC0 { + if ('a' <= c) and (c <= 'z') { + this.transformed_word[args.p] = c ^ 32 + } + return 1 + } + // An overly simplified uppercasing model for UTF-8. + if c < 0xE0 { + if ((args.p + 1) < args.end) and ((args.p + 1) < 1024) { + this.transformed_word[args.p + 1] ^= 32 + } + return 2 + } + // An arbitrary transform for three byte characters. + if ((args.p + 2) < args.end) and ((args.p + 2) < 1024) { + this.transformed_word[args.p + 2] ^= 5 + } + return 3 +} + +// byte_at returns args.s[args.i], or 0 if args.i is out of bounds. +pri func decoder.byte_at(s: roslice base.u8, i: base.u64) base.u32[..= 0xFF] { + if args.i < args.s.length() { + return args.s[args.i] as base.u32 + } + return 0 +} diff --git a/std/brotli/decode_huffman.wuffs b/std/brotli/decode_huffman.wuffs index ac9ceb7a..90e786bc 100644 --- a/std/brotli/decode_huffman.wuffs +++ b/std/brotli/decode_huffman.wuffs @@ -135,7 +135,7 @@ pri func decoder.read_simple_huffman_code?(src: base.io_reader, alphabet_size_ma // symbol, 1 means 2 symbols, 2 means 3 symbols, 3 means 4 symbols with // lengths [2, 2, 2, 2], 4 means 4 symbols with lengths [1, 2, 3, 3]. pri func decoder.build_simple_huffman_table!(hoff: base.u32, num_symbols: base.u32[..= 4]) base.status { - var t : base.u32[..= 671634] + var t : base.u32[..= 677778] var v : array[4] base.u32[..= 0x3FF] var tmp : base.u32[..= 0x3FF] var i : base.u32 diff --git a/std/brotli/decode_metablock.wuffs b/std/brotli/decode_metablock.wuffs index d5aecf3e..f3d29d85 100644 --- a/std/brotli/decode_metablock.wuffs +++ b/std/brotli/decode_metablock.wuffs @@ -13,17 +13,18 @@ // Mirrors the BROTLI_STATE_HUFFMAN_CODE_0 through // BROTLI_STATE_BEFORE_COMPRESSED_METABLOCK_BODY states of // BrotliDecoderDecompressStream in decode.c, followed by ProcessCommands. -pri func decoder.decode_compressed_metablock?(src: base.io_reader, workbuf: slice base.u8) { - var t : base.u32 - var n : base.u32[..= 256] - var n0 : base.u32[..= 256] - var n2 : base.u32[..= 256] - var bits : base.u32 - var npostfix : base.u32[..= 3] - var ndirect : base.u32[..= 120] - var alphabet : base.u32[..= 520] - var i : base.u32 - var k : base.u32 +pri func decoder.decode_compressed_metablock?(src: base.io_reader, workbuf: slice base.u8, dictbuf: roslice base.u8) { + var t : base.u32 + var n : base.u32[..= 256] + var n0 : base.u32[..= 256] + var n2 : base.u32[..= 256] + var bits : base.u32 + var npostfix : base.u32[..= 3] + var ndirect : base.u32[..= 120] + var alphabet_max : base.u32[..= 1128] + var alphabet : base.u32[..= 544] + var i : base.u32 + var k : base.u32 // BROTLI_STATE_HUFFMAN_CODE_0 ..= BROTLI_STATE_HUFFMAN_CODE_3. t = 0 @@ -75,8 +76,19 @@ pri func decoder.decode_compressed_metablock?(src: base.io_reader, workbuf: slic this.decode_context_map?(src: args.src, size: n2 << 2, offset: DIST_CONTEXT_MAP_BASE) this.num_dist_htrees = this.bits_value.min(no_more_than: 256) - // BROTLI_DISTANCE_ALPHABET_SIZE(npostfix, ndirect, BROTLI_MAX_DISTANCE_BITS). - alphabet = 16 + ndirect + ((24 as base.u32) << (npostfix + 1)) + if this.large_window { + // BROTLI_DISTANCE_ALPHABET_SIZE(npostfix, ndirect, + // BROTLI_LARGE_MAX_DISTANCE_BITS) and the max_alphabet_size of + // BrotliCalculateDistanceCodeLimit(BROTLI_MAX_ALLOWED_DISTANCE, + // npostfix, ndirect). + alphabet_max = 16 + ndirect + ((62 as base.u32) << (npostfix + 1)) + alphabet = this.calculate_distance_code_limit(npostfix: npostfix, ndirect: ndirect) + } else { + // BROTLI_DISTANCE_ALPHABET_SIZE(npostfix, ndirect, + // BROTLI_MAX_DISTANCE_BITS). + alphabet = 16 + ndirect + ((24 as base.u32) << (npostfix + 1)) + alphabet_max = alphabet + } this.distance_alphabet_size = alphabet // BROTLI_STATE_TREE_GROUP. Mirrors HuffmanTreeGroupDecode, three times. @@ -104,7 +116,7 @@ pri func decoder.decode_compressed_metablock?(src: base.io_reader, workbuf: slic while k < this.num_dist_htrees { assert k < 256 via "a < b: a < c; c <= b"(c: this.num_dist_htrees) this.read_huffman_code?(src: args.src, - alphabet_size_max: alphabet, + alphabet_size_max: alphabet_max, alphabet_size_limit: alphabet, hoff: DIST_HUFFS_BASE + (k * DIST_HUFFS_STRIDE), table_size_max: DIST_HUFFS_STRIDE) @@ -117,7 +129,50 @@ pri func decoder.decode_compressed_metablock?(src: base.io_reader, workbuf: slic this.htree_command = 0 this.calculate_distance_lut!() - this.process_commands?(src: args.src, workbuf: args.workbuf) + this.process_commands?(src: args.src, workbuf: args.workbuf, dictbuf: args.dictbuf) +} + +// Mirrors BrotliCalculateDistanceCodeLimit in constants.h, with a +// max_distance of BROTLI_MAX_ALLOWED_DISTANCE (0x7FFF_FFFC). It returns the +// max_alphabet_size, which is at most 544. +pri func decoder.calculate_distance_code_limit(npostfix: base.u32[..= 3], ndirect: base.u32[..= 120]) base.u32[..= 544] { + var offset : base.u32 + var ndistbits : base.u32 + var tmp : base.u32 + var half : base.u32[..= 1] + var group : base.u32 + var postfix : base.u32[..= 7] + var ret : base.u32 + + // The max_distance <= ndirect case cannot happen. + // + // offset is (forbidden_distance - ndirect - 1), where forbidden_distance + // is (max_distance + 1), after removing the postfix and "head-start". + offset = ((0x7FFF_FFFC - args.ndirect) >> args.npostfix) + 4 + postfix = ((1 as base.u32) << args.npostfix) - 1 + // Calculate the number of distance bits (a poor man's log2floor). + tmp = offset / 2 + ndistbits = 0 + while (tmp <> 0) and (ndistbits < 32) { + ndistbits += 1 + tmp >>= 1 + } + // One bit is covered with subrange addressing ("half"). + ndistbits ~mod-= 1 + // Find the subrange. + half = (offset >> (ndistbits & 31)) & 1 + // Calculate the "group" part of the dcode. + group = ((ndistbits ~mod- 1) ~mod<< 1) | half + if group == 0 { + return args.ndirect + 16 + } + // Decrement "group", so it is the last permitted "group". + group -= 1 + ret = (((group ~mod<< args.npostfix) | postfix) ~mod+ args.ndirect) ~mod+ 17 + if ret > 544 { + return 544 + } + return ret } // Mirrors DecodeContextMap in decode.c. The context map is written to @@ -310,7 +365,7 @@ pri func decoder.prepare_literal_decoding!() { pri func decoder.calculate_distance_lut!() { var npostfix : base.u32[..= 3] var ndirect : base.u32[..= 120] - var limit : base.u32[..= 520] + var limit : base.u32[..= 544] var postfix : base.u32[..= 8] var i : base.u32 var j : base.u32 @@ -361,7 +416,7 @@ pri func decoder.calculate_distance_lut!() { } // Mirrors ProcessCommandsInternal in decode.c. -pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8) { +pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, dictbuf: roslice base.u8) { var cmd_code : base.u32[..= 703] var insert_length : base.u32 var p1 : base.u32[..= 0xFF] @@ -375,6 +430,10 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8) var n : base.u32 var k : base.u32 var b : base.u8 + var compound_size : base.u32 + var address : base.u32 + var chunk : base.u32[..= 14] + var j : base.u64 while true { // CommandBegin. @@ -460,39 +519,84 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8) distance = this.distance_code if distance > max_distance { - // Apply a static dictionary reference, since the distance is - // larger than the max LZ77 distance. + // Apply a compound dictionary or static dictionary reference, + // since the distance is larger than the max LZ77 distance. // // The maximum allowed distance is BROTLI_MAX_ALLOWED_DISTANCE. if distance > 0x7FFF_FFFC { return "#bad distance" } - // The C code checks for SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH - // (4) and SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH (31) but the - // RFC 7932 dictionary's size_bits_by_length is zero for lengths - // 25 ..= 31, which also fails with "#bad dictionary". - // - // TODO: support custom dictionaries. - if (i < 4) or (24 < i) { - return "#bad dictionary" - } - this.dist_rb_idx ~mod+= this.distance_context - n = this.copy_dictionary_word!( - address: (distance ~mod- max_distance) ~mod- 1, - len: i.min(no_more_than: 24)) - if n == 0xFFFF_FFFF { - return "#bad transform" - } else if (n == 0) and (distance <= 120) { - return "#bad transform" - } - this.mb_remaining ~mod-= n - k = 0 - while k < n { - this.put_byte!(workbuf: args.workbuf, b: this.transformed_word[k & 63]) - k ~mod+= 1 - if this.pos == this.rb_size { - this.ring_event?() + compound_size = this.chunk_offsets[this.num_chunks] + address = (distance - max_distance) - 1 + + if address < compound_size { + // Mirrors InitializeCompoundDictionaryCopy and + // CopyFromCompoundDictionary in decode.c. This address is the + // C code's (compound_dictionary_size - (s->distance_code - + // s->max_distance)). + address = compound_size ~mod- (address + 1) + if ((address as base.u64) + (i as base.u64)) > (compound_size as base.u64) { + return "#bad compound dictionary" } + // Update the recent distances cache. + this.dist_rb[this.dist_rb_idx & 3] = distance + this.dist_rb_idx ~mod+= 1 + this.mb_remaining ~mod-= i + chunk = 0 + while i > 0 { + while chunk < 14 { + if address < this.chunk_offsets[chunk + 1] { + break + } + chunk += 1 + } + j = (this.chunk_starts[chunk] as base.u64) + + ((address ~mod- this.chunk_offsets[chunk]) as base.u64) + b = 0 + if j < args.dictbuf.length() { + b = args.dictbuf[j] + } + this.put_byte!(workbuf: args.workbuf, b: b) + address ~mod+= 1 + i ~mod-= 1 + if this.pos == this.rb_size { + this.ring_event?() + } + } + + } else if (4 <= i) and (i <= 31) { + // SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH and + // SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH. + p1 = this.get_byte(workbuf: args.workbuf, distance: 1) as base.u32 + p2 = this.get_byte(workbuf: args.workbuf, distance: 2) as base.u32 + mode_offset = this.context_mode << 9 + ctx = (CONTEXT_LOOKUP[mode_offset | p1] | + CONTEXT_LOOKUP[mode_offset | 0x100 | p2]) as base.u32 + // Compensate double distance-ring-buffer roll. + this.dist_rb_idx ~mod+= this.distance_context + n = this.copy_dictionary_word!(dictbuf: args.dictbuf, + address: address ~mod- compound_size, + len: i.min(no_more_than: 31), + ctx: ctx) + if n == 0xFFFF_FFFE { + return "#bad dictionary" + } else if n == 0xFFFF_FFFF { + return "#bad transform" + } else if (n == 0) and (distance <= 120) { + return "#bad transform" + } + this.mb_remaining ~mod-= n + k = 0 + while k < n { + this.put_byte!(workbuf: args.workbuf, b: this.transformed_word[k & 1023]) + k ~mod+= 1 + if this.pos == this.rb_size { + this.ring_event?() + } + } + + } else { + return "#bad dictionary" } } else { @@ -543,12 +647,17 @@ pri func decoder.read_distance?(src: base.io_reader) { if (code >= this.distance_alphabet_size) or (code >= 1128) { return "#internal error: inconsistent Huffman table" } - this.read_bits?(src: args.src, n: this.dist_extra_bits[code].min(no_more_than: 24) as base.u32) + // The C code uses BrotliSafeReadBits32, as large-window brotli can have + // more than 24 extra bits. + this.read_bits32?(src: args.src, n: this.dist_extra_bits[code].min(no_more_than: 32) as base.u32) if code >= 1128 { return "#internal error: inconsistent Huffman table" } - this.distance_code = this.dist_offset[code] ~mod+ - (this.bits_value ~mod<< this.distance_postfix_bits) + // The C code calculates this with brotli_reg_t (64-bit) arithmetic and + // then casts to int. BrotliCalculateDistanceCodeLimit means that the + // result is less than BROTLI_MAX_ALLOWED_DISTANCE. + this.distance_code = (((this.dist_offset[code] as base.u64) ~mod+ + ((this.bits_value as base.u64) ~mod<< this.distance_postfix_bits)) & 0xFFFF_FFFF) as base.u32 } // Mirrors TakeDistanceFromRingBuffer in decode.c. @@ -585,144 +694,3 @@ pri func decoder.take_distance_from_ring_buffer!(code: base.u32[..= 15]) { this.distance_code = v - 3 } } - -// copy_dictionary_word writes a transformed static dictionary word to -// this.transformed_word and returns its length. It returns 0xFFFF_FFFF if the -// transform index is out of range. Mirrors part of ProcessCommandsInternal in -// decode.c and BrotliTransformDictionaryWord in transform.c. -pri func decoder.copy_dictionary_word!(address: base.u32, len: base.u32[..= 24]) base.u32 { - var shift : base.u32[..= 24] - var word_idx : base.u32 - var transform_idx : base.u32 - var offset : base.u32 - var t : base.u32[..= 120] - var t_type : base.u32[..= 22] - var affix : base.u32[..= 216] - var affix_len : base.u32 - var idx : base.u32 - var word_start : base.u32 - var word_len : base.u32 - var skip : base.u32[..= 9] - var i : base.u32 - var j : base.u32 - var step : base.u32 - - shift = DICTIONARY_SIZE_BITS_BY_LENGTH[args.len] - word_idx = args.address & (((1 as base.u32) << shift) - 1) - transform_idx = args.address >> shift - if transform_idx >= NUM_TRANSFORMS { - return 0xFFFF_FFFF - } - t = transform_idx.min(no_more_than: 120) - offset = DICTIONARY_OFFSETS_BY_LENGTH[args.len] ~mod+ (word_idx ~mod* args.len) - - // The prefix. - idx = 0 - affix = PREFIX_SUFFIX_MAP[TRANSFORM_PREFIX_ID[t]] - affix_len = PREFIX_SUFFIX[affix] as base.u32 - i = 0 - while i < affix_len { - j = (affix ~mod+ 1) ~mod+ i - if (j >= 217) or (idx >= 64) { - break - } - this.transformed_word[idx] = PREFIX_SUFFIX[j] - idx ~mod+= 1 - i ~mod+= 1 - } - - // The word, possibly with its first or last few bytes omitted. - t_type = TRANSFORM_TYPE[t] as base.u32 - word_len = args.len - skip = 0 - if t_type <= 9 { - // BROTLI_TRANSFORM_OMIT_LAST_N. - if t_type < args.len { - word_len = args.len ~mod- t_type - } else { - word_len = 0 - } - } else if (12 <= t_type) and (t_type <= 20) { - // BROTLI_TRANSFORM_OMIT_FIRST_N. - skip = t_type - 11 - if skip < args.len { - word_len = args.len ~mod- skip - } else { - word_len = 0 - } - } - word_start = idx - i = 0 - while i < word_len { - j = (offset ~mod+ skip) ~mod+ i - if (j >= 122784) or (idx >= 64) { - break - } - this.transformed_word[idx] = DICTIONARY[j] - idx ~mod+= 1 - i ~mod+= 1 - } - - // Upper-casing. The C code's ToUpperCase can modify bytes after the - // word's end, but those bytes are either overwritten by the suffix or are - // past the transformed word's length, so we don't bother. - if t_type == 10 { - // BROTLI_TRANSFORM_UPPERCASE_FIRST. - this.to_upper_case!(p: word_start, end: idx) - } else if t_type == 11 { - // BROTLI_TRANSFORM_UPPERCASE_ALL. - i = word_start - while i < idx { - step = this.to_upper_case!(p: i, end: idx) - i ~mod+= step - } - } - // TODO: support BROTLI_TRANSFORM_SHIFT_FIRST and _SHIFT_ALL (used only by - // custom dictionaries' transforms). - - // The suffix. - affix = PREFIX_SUFFIX_MAP[TRANSFORM_SUFFIX_ID[t]] - affix_len = PREFIX_SUFFIX[affix] as base.u32 - i = 0 - while i < affix_len { - j = (affix ~mod+ 1) ~mod+ i - if (j >= 217) or (idx >= 64) { - break - } - this.transformed_word[idx] = PREFIX_SUFFIX[j] - idx ~mod+= 1 - i ~mod+= 1 - } - - return idx -} - -// Mirrors ToUpperCase in transform.c, except that it does not modify bytes at -// or after args.end. It returns the step: the number of bytes in the -// (overly simplified model of a) UTF-8 encoded character. -pri func decoder.to_upper_case!(p: base.u32, end: base.u32) base.u32 { - var c : base.u8 - - if args.p >= 64 { - return 1 - } - c = this.transformed_word[args.p] - if c < 0xC0 { - if ('a' <= c) and (c <= 'z') { - this.transformed_word[args.p] = c ^ 32 - } - return 1 - } - // An overly simplified uppercasing model for UTF-8. - if c < 0xE0 { - if ((args.p + 1) < args.end) and ((args.p + 1) < 64) { - this.transformed_word[args.p + 1] ^= 32 - } - return 2 - } - // An arbitrary transform for three byte characters. - if ((args.p + 2) < args.end) and ((args.p + 2) < 64) { - this.transformed_word[args.p + 2] ^= 5 - } - return 3 -} diff --git a/std/brotli/decode_quirks.wuffs b/std/brotli/decode_quirks.wuffs new file mode 100644 index 00000000..5f59cdb5 --- /dev/null +++ b/std/brotli/decode_quirks.wuffs @@ -0,0 +1,32 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// -------- + +// Quirks are discussed in (/doc/note/quirks.md). +// +// The base38 encoding of "brot" is 0x0A_ADE4. Left shifting by 10 gives +// 0x2AB7_9000. +pri const QUIRKS_BASE : base.u32 = 0x2AB7_9000 + +// -------- + +// When this quirk is set, a positive value enables large-window brotli +// (https://datatracker.ietf.org/doc/draft-vandevenne-shared-brotli-format/), +// with window sizes up to (1 << 30) bytes. Zero means standard (RFC 7932) +// brotli. It mirrors the C code's BROTLI_DECODER_PARAM_LARGE_WINDOW. +// +// It must be set before decoding starts. Setting it also increases the +// workbuf_len. +// +// Without this quirk, a stream that starts with the large-window WBITS escape +// is rejected with "#unsupported large window". With this quirk, a stream that +// does not start with that escape is decoded as standard brotli. +pub const QUIRK_LARGE_WINDOW : base.u32 = 0x2AB7_9000 | 0x00 diff --git a/test/c/mimiclib/brotli.c b/test/c/mimiclib/brotli.c index 2608f94c..08e17cfc 100644 --- a/test/c/mimiclib/brotli.c +++ b/test/c/mimiclib/brotli.c @@ -72,3 +72,75 @@ mimic_brotli_decode(wuffs_base__io_buffer* dst, int error_code = 0; return mimic_brotli_decode_error_code(dst, src, &error_code); } + +// ---------------- Large Window and Dictionaries + +#include "brotli/shared_dictionary.h" + +// mimic_brotli_decode_with_options is like mimic_brotli_decode_error_code but +// it also sets BROTLI_DECODER_PARAM_LARGE_WINDOW (if large_window is true) and +// attaches dictionaries (with BrotliDecoderAttachDictionary). Each +// dict_types[i] is a BrotliSharedDictionaryType value: 0 for +// BROTLI_SHARED_DICTIONARY_RAW and 1 for BROTLI_SHARED_DICTIONARY_SERIALIZED. +// +// libbrotlidec only supports serialized dictionaries when built with +// BROTLI_EXPERIMENTAL (which is not the default). When attaching a dictionary +// fails, this function returns +// mimic_brotli_attach_dictionary_failed_message. +const char mimic_brotli_attach_dictionary_failed_message[] = + "libbrotlidec: BrotliDecoderAttachDictionary failed"; + +const char* // +mimic_brotli_decode_with_options(wuffs_base__io_buffer* dst, + wuffs_base__io_buffer* src, + bool large_window, + size_t num_dicts, + const uint32_t* dict_types, + const wuffs_base__slice_u8* dicts, + int* error_code_ptr) { + *error_code_ptr = 0; + BrotliDecoderState* state = BrotliDecoderCreateInstance(NULL, NULL, NULL); + if (!state) { + return "libbrotlidec: BrotliDecoderCreateInstance failed"; + } + if (large_window && + !BrotliDecoderSetParameter(state, BROTLI_DECODER_PARAM_LARGE_WINDOW, + 1u)) { + BrotliDecoderDestroyInstance(state); + return "libbrotlidec: BrotliDecoderSetParameter failed"; + } + for (size_t i = 0; i < num_dicts; i++) { + if (!BrotliDecoderAttachDictionary( + state, (BrotliSharedDictionaryType)dict_types[i], dicts[i].len, + dicts[i].ptr)) { + BrotliDecoderDestroyInstance(state); + return mimic_brotli_attach_dictionary_failed_message; + } + } + + size_t avail_in = wuffs_base__io_buffer__reader_length(src); + const uint8_t* next_in = wuffs_base__io_buffer__reader_pointer(src); + size_t avail_out = wuffs_base__io_buffer__writer_length(dst); + uint8_t* next_out = wuffs_base__io_buffer__writer_pointer(dst); + size_t total_out = 0; + + BrotliDecoderResult result = BrotliDecoderDecompressStream( + state, &avail_in, &next_in, &avail_out, &next_out, &total_out); + *error_code_ptr = (int)BrotliDecoderGetErrorCode(state); + BrotliDecoderDestroyInstance(state); + + src->meta.ri += wuffs_base__io_buffer__reader_length(src) - avail_in; + dst->meta.wi += wuffs_base__io_buffer__writer_length(dst) - avail_out; + + switch (result) { + case BROTLI_DECODER_RESULT_SUCCESS: + return NULL; + case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: + return "libbrotlidec: needs more input"; + case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: + return "libbrotlidec: needs more output"; + default: + break; + } + return "libbrotlidec: BrotliDecoderDecompressStream failed"; +} diff --git a/test/c/std/brotli.c b/test/c/std/brotli.c index d1464aa3..5ddf9446 100644 --- a/test/c/std/brotli.c +++ b/test/c/std/brotli.c @@ -538,6 +538,7 @@ brotli_c_error_code(const char* status) { {wuffs_brotli__error__bad_final_padding, -15}, {wuffs_brotli__error__bad_distance, -16}, {wuffs_brotli__error__bad_block_switch, -17}, + {wuffs_brotli__error__bad_compound_dictionary, -18}, {wuffs_brotli__error__truncated_input, 2}, }; if (!status) { @@ -633,6 +634,855 @@ test_mimic_brotli_decode_malformed() { #endif // WUFFS_MIMIC +// ---------------- Large Window and Dictionary Tests + +// See test/data/brotli/README.md for where the large-window (lw), raw +// dictionary (raw) and serialized shared dictionary (sd) test files came from. + +typedef struct { + uint32_t type; // WUFFS_BROTLI__DICTIONARY_TYPE_ETC. + const char* filename; + // lo and hi select the [lo .. hi) bytes of the file. A zero hi means the + // whole file. + size_t lo; + size_t hi; +} brotli_dict_spec; + +#define BROTLI_DICT_SPECS_MAX 4 + +// g_brotli_dict_sets lists the sets of dictionaries (attached in order) that +// the test files use. A NULL filename terminates each set. +struct { + brotli_dict_spec specs[BROTLI_DICT_SPECS_MAX]; +} g_brotli_dict_sets[] = { +#define BROTLI_DICT_SET_NONE 0 + {{{0, NULL, 0, 0}}}, +#define BROTLI_DICT_SET_RAW_ROMEO 1 + {{{WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/romeo.txt", 0, 0}}}, +#define BROTLI_DICT_SET_RAW_ALICE_CHUNKS 2 + {{ + {WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/brotli/alice29.txt", 0, + 20000}, + {WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/brotli/alice29.txt", + 20000, 20001}, + {WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/brotli/alice29.txt", + 20001, 20001}, + {WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/brotli/alice29.txt", + 20001, 60000}, + }}, +#define BROTLI_DICT_SET_RAW_ROMEO_CHUNKS 3 + {{ + {WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/romeo.txt", 0, 300}, + {WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/romeo.txt", 300, 500}, + {WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/romeo.txt", 500, 942}, + }}, +#define BROTLI_DICT_SET_SD_FULL 4 + {{{WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, + "test/data/brotli/shared-dictionary.full.bin", 0, 0}}}, +#define BROTLI_DICT_SET_SD_WORDS 5 + {{{WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, + "test/data/brotli/shared-dictionary.words.bin", 0, 0}}}, +#define BROTLI_DICT_SET_SD_PREFIX_AND_RAW 6 + {{ + {WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, + "test/data/brotli/shared-dictionary.prefix.bin", 0, 0}, + {WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/romeo.txt", 0, 300}, + }}, +#define BROTLI_DICT_SET_SD_UTF8 7 + {{{WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, + "test/data/brotli/shared-dictionary.utf8.bin", 0, 0}}}, +}; + +// g_brotli_ext_vectors lists (compressed, decompressed) pairs of test files +// that need the large-window quirk or attached dictionaries. +struct { + const char* src_filename; + const char* want_filename; + bool large_window; + size_t dict_set; +} g_brotli_ext_vectors[] = { + {"test/data/brotli/alice29.txt.q11lw26.br", "test/data/brotli/alice29.txt", + true, BROTLI_DICT_SET_NONE}, + {"test/data/brotli/alice29.txt.q5lw10.br", "test/data/brotli/alice29.txt", + true, BROTLI_DICT_SET_NONE}, + {"test/data/brotli/romeo.txt.q11lw30.br", "test/data/romeo.txt", true, + BROTLI_DICT_SET_NONE}, + {"test/data/brotli/romeo.txt.q9lw28.br", "test/data/romeo.txt", true, + BROTLI_DICT_SET_NONE}, + {"test/data/brotli/romeo.txt.q11w22.raw-romeo.br", "test/data/romeo.txt", + false, BROTLI_DICT_SET_RAW_ROMEO}, + {"test/data/brotli/romeo.txt.q11lw30.raw-romeo.br", "test/data/romeo.txt", + true, BROTLI_DICT_SET_RAW_ROMEO}, + {"test/data/brotli/alice29.txt.q5w18.raw-chunks.br", + "test/data/brotli/alice29.txt", false, BROTLI_DICT_SET_RAW_ALICE_CHUNKS}, + {"test/data/brotli/alice29.txt.q11w18.raw-chunks.br", + "test/data/brotli/alice29.txt", false, BROTLI_DICT_SET_RAW_ALICE_CHUNKS}, + {"test/data/brotli/romeo.txt.q11w16.raw-chunks.br", "test/data/romeo.txt", + false, BROTLI_DICT_SET_RAW_ROMEO_CHUNKS}, + {"test/data/brotli/shared-dictionary.text.q5w18.sd-full.br", + "test/data/brotli/shared-dictionary.text", false, BROTLI_DICT_SET_SD_FULL}, + {"test/data/brotli/shared-dictionary.text.q11w18.sd-full.br", + "test/data/brotli/shared-dictionary.text", false, BROTLI_DICT_SET_SD_FULL}, + {"test/data/brotli/shared-dictionary.text.q11lw24.sd-full.br", + "test/data/brotli/shared-dictionary.text", true, BROTLI_DICT_SET_SD_FULL}, + {"test/data/brotli/alice29.txt.q11w22.sd-words.br", + "test/data/brotli/alice29.txt", false, BROTLI_DICT_SET_SD_WORDS}, + {"test/data/brotli/asyoulik.txt.q11w18.sd-prefix-raw.br", + "test/data/brotli/asyoulik.txt", false, BROTLI_DICT_SET_SD_PREFIX_AND_RAW}, + {"test/data/brotli/romeo-mutated.sd-utf8.br", + "test/data/brotli/romeo-mutated.sd-utf8.txt", false, + BROTLI_DICT_SET_SD_UTF8}, +}; + +// g_brotli_ext_malformed is like g_brotli_malformed but for inputs that need +// the large-window quirk or attached dictionaries. The invalid.lw-*, +// invalid.raw-* and invalid.sd-* files were found by mutating the +// g_brotli_ext_vectors files (see test/data/brotli/README.md). +struct { + const char* src_filename; + const char* data; + size_t len; + bool large_window; + size_t dict_set; + const char* want_status; +} g_brotli_ext_malformed[] = { + // BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS: the large-window escape + // followed by a non-zero reserved bit. + {NULL, "\x91", 1, true, BROTLI_DICT_SET_NONE, + wuffs_brotli__error__bad_window_bits}, + // BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS: large-window WBITS of 9 (too + // small) and 62 (too large). + {NULL, "\x11\x09", 2, true, BROTLI_DICT_SET_NONE, + wuffs_brotli__error__bad_window_bits}, + {NULL, "\x11\x3E", 2, true, BROTLI_DICT_SET_NONE, + wuffs_brotli__error__bad_window_bits}, + // BROTLI_DECODER_NEEDS_MORE_INPUT: a truncated large-window header. + {NULL, "\x11", 1, true, BROTLI_DICT_SET_NONE, + wuffs_brotli__error__truncated_input}, + // BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS: a large-window stream without + // the large-window quirk. + {"test/data/brotli/romeo.txt.q11lw30.br", NULL, 0, false, + BROTLI_DICT_SET_NONE, wuffs_brotli__error__unsupported_large_window}, + // BROTLI_DECODER_ERROR_FORMAT_DISTANCE. + {"test/data/brotli/invalid.lw-distance.br", NULL, 0, true, + BROTLI_DICT_SET_NONE, wuffs_brotli__error__bad_distance}, + // BROTLI_DECODER_ERROR_FORMAT_DICTIONARY. + {"test/data/brotli/invalid.lw-dictionary.br", NULL, 0, true, + BROTLI_DICT_SET_NONE, wuffs_brotli__error__bad_dictionary}, + // BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY. + {"test/data/brotli/invalid.raw-compound-dictionary.br", NULL, 0, false, + BROTLI_DICT_SET_RAW_ROMEO_CHUNKS, + wuffs_brotli__error__bad_compound_dictionary}, + // BROTLI_DECODER_ERROR_FORMAT_DICTIONARY. + {"test/data/brotli/invalid.raw-dictionary.br", NULL, 0, false, + BROTLI_DICT_SET_RAW_ROMEO_CHUNKS, wuffs_brotli__error__bad_dictionary}, + // BROTLI_DECODER_ERROR_FORMAT_TRANSFORM. + {"test/data/brotli/invalid.raw-transform.br", NULL, 0, false, + BROTLI_DICT_SET_RAW_ROMEO_CHUNKS, wuffs_brotli__error__bad_transform}, + // BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY: a stream compressed without + // a dictionary refers to the static dictionary, but with a compound + // dictionary attached, the same distance refers to the compound + // dictionary. + {"test/data/brotli/romeo.txt.q11w22.br", NULL, 0, false, + BROTLI_DICT_SET_SD_FULL, wuffs_brotli__error__bad_compound_dictionary}, + // BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY. + {"test/data/brotli/invalid.sd-compound-dictionary.br", NULL, 0, false, + BROTLI_DICT_SET_SD_FULL, wuffs_brotli__error__bad_compound_dictionary}, + // BROTLI_DECODER_ERROR_FORMAT_DICTIONARY. + {"test/data/brotli/invalid.sd-dictionary.br", NULL, 0, false, + BROTLI_DICT_SET_SD_FULL, wuffs_brotli__error__bad_dictionary}, + // BROTLI_DECODER_ERROR_FORMAT_TRANSFORM. + {"test/data/brotli/invalid.sd-transform.br", NULL, 0, false, + BROTLI_DICT_SET_SD_FULL, wuffs_brotli__error__bad_transform}, + {"test/data/brotli/invalid.lw-sd-transform.br", NULL, 0, true, + BROTLI_DICT_SET_SD_FULL, wuffs_brotli__error__bad_transform}, +}; + +// g_brotli_dict_arrays holds the dictionaries' bytes. +uint8_t g_brotli_dict_arrays[BROTLI_DICT_SPECS_MAX][256 * 1024]; + +// load_brotli_dict_set loads the dict_set'th g_brotli_dict_sets entry. +const char* // +load_brotli_dict_set(size_t dict_set, + size_t* num_dicts, + uint32_t* dict_types, + wuffs_base__slice_u8* dicts) { + *num_dicts = 0; + if (dict_set >= WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_dict_sets)) { + return "load_brotli_dict_set: dict_set out of range"; + } + for (size_t i = 0; i < BROTLI_DICT_SPECS_MAX; i++) { + const brotli_dict_spec* spec = &g_brotli_dict_sets[dict_set].specs[i]; + if (!spec->filename) { + break; + } + wuffs_base__io_buffer buf = ((wuffs_base__io_buffer){ + .data = wuffs_base__make_slice_u8(g_brotli_dict_arrays[i], + sizeof g_brotli_dict_arrays[i]), + }); + CHECK_STRING(read_file(&buf, spec->filename)); + size_t hi = spec->hi ? spec->hi : buf.meta.wi; + if ((spec->lo > hi) || (hi > buf.meta.wi)) { + return "load_brotli_dict_set: bad lo/hi"; + } + dict_types[i] = spec->type; + dicts[i] = wuffs_base__make_slice_u8(buf.data.ptr + spec->lo, + hi - spec->lo); + *num_dicts = i + 1; + } + return NULL; +} + +// get_brotli_big_workbuf sets *workbuf to a malloc'ed buffer that is large +// enough for any large-window stream and 1 MiB of dictionaries. It is never +// freed. Only the parts of it that the decoder uses are touched, so the +// operating system should only need to commit as much memory as the stream's +// window size. +const char* // +get_brotli_big_workbuf(wuffs_base__slice_u8* workbuf) { + static uint8_t* ptr = NULL; + const size_t n = WUFFS_BROTLI__DECODER_WORKBUF_LEN_LARGE_WINDOW + (1 << 20); + if (!ptr) { + ptr = (uint8_t*)malloc(n); + if (!ptr) { + return "get_brotli_big_workbuf: out of memory"; + } + } + *workbuf = wuffs_base__make_slice_u8(ptr, n); + return NULL; +} + +// wuffs_brotli_decode_with_options is like wuffs_brotli_decode but it can +// also set the large-window quirk and attach dictionaries. +const char* // +wuffs_brotli_decode_with_options(wuffs_base__io_buffer* dst, + wuffs_base__io_buffer* src, + uint64_t wlimit, + uint64_t rlimit, + bool large_window, + size_t dict_set) { + size_t num_dicts = 0; + uint32_t dict_types[BROTLI_DICT_SPECS_MAX] = {0}; + wuffs_base__slice_u8 dicts[BROTLI_DICT_SPECS_MAX] = {0}; + CHECK_STRING(load_brotli_dict_set(dict_set, &num_dicts, dict_types, dicts)); + + wuffs_brotli__decoder dec; + CHECK_STATUS("initialize", + wuffs_brotli__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED)); + // The large-window workbuf_len (more than 1 GiB) is larger than + // g_work_slice_u8. + wuffs_base__slice_u8 workbuf = g_work_slice_u8; + if (large_window) { + CHECK_STATUS("set_quirk", + wuffs_brotli__decoder__set_quirk( + &dec, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, 1)); + CHECK_STRING(get_brotli_big_workbuf(&workbuf)); + } + for (size_t i = 0; i < num_dicts; i++) { + CHECK_STATUS("add_dictionary", + wuffs_brotli__decoder__add_dictionary( + &dec, dict_types[i], dicts[i], workbuf)); + } + wuffs_base__range_ii_u64 workbuf_len = + wuffs_brotli__decoder__workbuf_len(&dec); + if (workbuf_len.max_incl > workbuf.len) { + return "workbuf_len is too large"; + } + workbuf.len = workbuf_len.max_incl; + + while (true) { + wuffs_base__io_buffer limited_dst = make_limited_writer(*dst, wlimit); + wuffs_base__io_buffer limited_src = make_limited_reader(*src, rlimit); + + wuffs_base__status status = wuffs_brotli__decoder__transform_io( + &dec, &limited_dst, &limited_src, workbuf); + + dst->meta.wi += limited_dst.meta.wi; + src->meta.ri += limited_src.meta.ri; + + if (((wlimit < UINT64_MAX) && + (status.repr == wuffs_base__suspension__short_write)) || + ((rlimit < UINT64_MAX) && + (status.repr == wuffs_base__suspension__short_read))) { + continue; + } + return status.repr; + } +} + +const char* // +do_test_brotli_decode_ext_all(uint64_t wlimit, uint64_t rlimit) { + size_t n = WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_ext_vectors); + for (size_t i = 0; i < n; i++) { + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = g_src_slice_u8, + }); + wuffs_base__io_buffer have = ((wuffs_base__io_buffer){ + .data = g_have_slice_u8, + }); + wuffs_base__io_buffer want = ((wuffs_base__io_buffer){ + .data = g_want_slice_u8, + }); + CHECK_STRING(read_file(&src, g_brotli_ext_vectors[i].src_filename)); + CHECK_STRING(read_file(&want, g_brotli_ext_vectors[i].want_filename)); + const char* status = wuffs_brotli_decode_with_options( + &have, &src, wlimit, rlimit, g_brotli_ext_vectors[i].large_window, + g_brotli_ext_vectors[i].dict_set); + if (status) { + RETURN_FAIL("%s: %s", g_brotli_ext_vectors[i].src_filename, status); + } + status = check_io_buffers_equal("", &have, &want); + if (status) { + RETURN_FAIL("%s: %s", g_brotli_ext_vectors[i].src_filename, status); + } + if (src.meta.ri != src.meta.wi) { + RETURN_FAIL("%s: ri: have %zu, want %zu", + g_brotli_ext_vectors[i].src_filename, src.meta.ri, + src.meta.wi); + } + } + return NULL; +} + +const char* // +test_wuffs_brotli_decode_ext_all() { + CHECK_FOCUS(__func__); + return do_test_brotli_decode_ext_all(UINT64_MAX, UINT64_MAX); +} + +const char* // +test_wuffs_brotli_decode_ext_all_with_io_limits() { + CHECK_FOCUS(__func__); + return do_test_brotli_decode_ext_all(1, 1); +} + +// make_brotli_far_data writes the decompressed form of +// test/data/brotli/far17m.q9lw25.br: 64 KiB of xorshift32 output, 17 MiB of +// zeroes and the same 64 KiB again. Its second 64 KiB copy is more than +// (1 << 24) bytes after the first. +const char* // +make_brotli_far_data(wuffs_base__io_buffer* dst) { + const size_t a = 65536; + const size_t z = 17 << 20; + if ((dst->data.len - dst->meta.wi) < (a + z + a)) { + return "make_brotli_far_data: dst is too short"; + } + uint8_t* p = dst->data.ptr + dst->meta.wi; + uint32_t x = 0x12345678; + for (size_t i = 0; i < a; i++) { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + p[i] = (uint8_t)(x >> 24); + } + memset(p + a, 0, z); + memcpy(p + a + z, p, a); + dst->meta.wi += a + z + a; + return NULL; +} + +const char* // +do_test_brotli_decode_large_window_far( + const char* (*codec_func)(wuffs_base__io_buffer*, + wuffs_base__io_buffer*, + uint64_t, + uint64_t, + bool, + size_t), + uint64_t wlimit, + uint64_t rlimit) { + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = g_src_slice_u8, + }); + wuffs_base__io_buffer have = ((wuffs_base__io_buffer){ + .data = g_have_slice_u8, + }); + wuffs_base__io_buffer want = ((wuffs_base__io_buffer){ + .data = g_want_slice_u8, + }); + CHECK_STRING(read_file(&src, "test/data/brotli/far17m.q9lw25.br")); + CHECK_STRING(make_brotli_far_data(&want)); + CHECK_STRING( + (*codec_func)(&have, &src, wlimit, rlimit, true, BROTLI_DICT_SET_NONE)); + return check_io_buffers_equal("", &have, &want); +} + +const char* // +test_wuffs_brotli_decode_large_window_far() { + CHECK_FOCUS(__func__); + CHECK_STRING(do_test_brotli_decode_large_window_far( + wuffs_brotli_decode_with_options, UINT64_MAX, UINT64_MAX)); + return do_test_brotli_decode_large_window_far( + wuffs_brotli_decode_with_options, 4099, 113); +} + +const char* // +test_wuffs_brotli_decode_large_window_quirk() { + CHECK_FOCUS(__func__); + + // With the quirk set, standard (not large-window) streams still decode. + size_t n = WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_vectors); + for (size_t i = 0; i < n; i++) { + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = g_src_slice_u8, + }); + wuffs_base__io_buffer have = ((wuffs_base__io_buffer){ + .data = g_have_slice_u8, + }); + wuffs_base__io_buffer want = ((wuffs_base__io_buffer){ + .data = g_want_slice_u8, + }); + CHECK_STRING(read_file(&src, g_brotli_vectors[i].src_filename)); + CHECK_STRING(read_file(&want, g_brotli_vectors[i].want_filename)); + const char* status = wuffs_brotli_decode_with_options( + &have, &src, UINT64_MAX, UINT64_MAX, true, BROTLI_DICT_SET_NONE); + if (status) { + RETURN_FAIL("%s: %s", g_brotli_vectors[i].src_filename, status); + } + CHECK_STRING(check_io_buffers_equal("", &have, &want)); + } + + // The quirk's value and the workbuf_len. + wuffs_brotli__decoder dec; + CHECK_STATUS("initialize", wuffs_brotli__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__DEFAULT_OPTIONS)); + if (wuffs_brotli__decoder__get_quirk(&dec, + WUFFS_BROTLI__QUIRK_LARGE_WINDOW) != 0) { + RETURN_FAIL("get_quirk #0: have non-zero, want zero"); + } + wuffs_base__range_ii_u64 r = wuffs_brotli__decoder__workbuf_len(&dec); + if ((r.min_incl != WUFFS_BROTLI__DECODER_WORKBUF_LEN_STANDARD) || + (r.max_incl != WUFFS_BROTLI__DECODER_WORKBUF_LEN_STANDARD) || + (r.max_incl != ((1 << 24) + 542))) { + RETURN_FAIL("workbuf_len #0: have %" PRIu64, r.max_incl); + } + CHECK_STATUS("set_quirk", wuffs_brotli__decoder__set_quirk( + &dec, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, 1)); + if (wuffs_brotli__decoder__get_quirk(&dec, + WUFFS_BROTLI__QUIRK_LARGE_WINDOW) != 1) { + RETURN_FAIL("get_quirk #1: have zero, want non-zero"); + } + r = wuffs_brotli__decoder__workbuf_len(&dec); + if ((r.min_incl != r.max_incl) || + (r.max_incl != WUFFS_BROTLI__DECODER_WORKBUF_LEN_LARGE_WINDOW) || + (r.max_incl != ((1 << 30) + 542))) { + RETURN_FAIL("workbuf_len #1: have %" PRIu64, r.max_incl); + } + + // Quirks cannot be set after decoding has started. + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = g_src_slice_u8, + }); + wuffs_base__io_buffer have = ((wuffs_base__io_buffer){ + .data = g_have_slice_u8, + }); + CHECK_STRING(read_file(&src, "test/data/brotli/alice29.txt.q5lw10.br")); + src.meta.wi = 3; + src.meta.closed = false; + wuffs_base__slice_u8 workbuf = g_work_slice_u8; + wuffs_base__status status = + wuffs_brotli__decoder__transform_io(&dec, &have, &src, workbuf); + if (status.repr != wuffs_base__suspension__short_read) { + RETURN_FAIL("transform_io: have \"%s\", want \"%s\"", status.repr, + wuffs_base__suspension__short_read); + } + status = wuffs_brotli__decoder__set_quirk( + &dec, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, 0); + if (status.repr != wuffs_base__error__bad_call_sequence) { + RETURN_FAIL("set_quirk: have \"%s\", want \"%s\"", status.repr, + wuffs_base__error__bad_call_sequence); + } + return NULL; +} + +const char* // +test_wuffs_brotli_decode_ext_malformed() { + CHECK_FOCUS(__func__); + size_t n = WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_ext_malformed); + for (size_t i = 0; i < n; i++) { + for (int j = 0; j < 2; j++) { + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = g_src_slice_u8, + }); + wuffs_base__io_buffer have = ((wuffs_base__io_buffer){ + .data = g_have_slice_u8, + }); + if (g_brotli_ext_malformed[i].src_filename) { + CHECK_STRING(read_file(&src, g_brotli_ext_malformed[i].src_filename)); + } else { + memcpy(src.data.ptr, g_brotli_ext_malformed[i].data, + g_brotli_ext_malformed[i].len); + src.meta.wi = g_brotli_ext_malformed[i].len; + src.meta.closed = true; + } + uint64_t limit = (j == 0) ? UINT64_MAX : 1; + const char* have_status = wuffs_brotli_decode_with_options( + &have, &src, limit, limit, g_brotli_ext_malformed[i].large_window, + g_brotli_ext_malformed[i].dict_set); + const char* want_status = g_brotli_ext_malformed[i].want_status; + if (have_status != want_status) { + RETURN_FAIL("i=%zu, j=%d: have \"%s\", want \"%s\"", i, j, have_status, + want_status); + } + } + } + return NULL; +} + +const char* // +test_wuffs_brotli_add_dictionary() { + CHECK_FOCUS(__func__); + wuffs_base__io_buffer full = ((wuffs_base__io_buffer){ + .data = wuffs_base__make_slice_u8(g_brotli_dict_arrays[0], + sizeof g_brotli_dict_arrays[0]), + }); + wuffs_base__io_buffer words = ((wuffs_base__io_buffer){ + .data = wuffs_base__make_slice_u8(g_brotli_dict_arrays[1], + sizeof g_brotli_dict_arrays[1]), + }); + wuffs_base__io_buffer prefix = ((wuffs_base__io_buffer){ + .data = wuffs_base__make_slice_u8(g_brotli_dict_arrays[2], + sizeof g_brotli_dict_arrays[2]), + }); + CHECK_STRING( + read_file(&full, "test/data/brotli/shared-dictionary.full.bin")); + CHECK_STRING( + read_file(&words, "test/data/brotli/shared-dictionary.words.bin")); + CHECK_STRING( + read_file(&prefix, "test/data/brotli/shared-dictionary.prefix.bin")); + wuffs_base__slice_u8 full_s = + wuffs_base__make_slice_u8(full.data.ptr, full.meta.wi); + wuffs_base__slice_u8 words_s = + wuffs_base__make_slice_u8(words.data.ptr, words.meta.wi); + wuffs_base__slice_u8 prefix_s = + wuffs_base__make_slice_u8(prefix.data.ptr, prefix.meta.wi); + wuffs_base__slice_u8 raw_s = + wuffs_base__make_slice_u8(prefix.data.ptr, 100); + wuffs_base__slice_u8 workbuf = g_work_slice_u8; + wuffs_brotli__decoder dec; + wuffs_base__status status; + + // An unknown dictionary type. + CHECK_STATUS("initialize", wuffs_brotli__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__DEFAULT_OPTIONS)); + status = wuffs_brotli__decoder__add_dictionary(&dec, 2, raw_s, workbuf); + if (status.repr != wuffs_base__error__bad_argument) { + RETURN_FAIL("type 2: have \"%s\", want \"%s\"", status.repr, + wuffs_base__error__bad_argument); + } + + // At most 15 compound (raw or prefix) dictionaries. The workbuf_len grows + // with each one. + for (int i = 0; i < 15; i++) { + CHECK_STATUS("add_dictionary #0", + wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, raw_s, workbuf)); + wuffs_base__range_ii_u64 r = wuffs_brotli__decoder__workbuf_len(&dec); + uint64_t want = WUFFS_BROTLI__DECODER_WORKBUF_LEN_STANDARD + + (100 * ((uint64_t)i + 1)); + if ((r.min_incl != want) || (r.max_incl != want)) { + RETURN_FAIL("workbuf_len: have %" PRIu64 ", want %" PRIu64, r.max_incl, + want); + } + } + status = wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, raw_s, workbuf); + if (status.repr != wuffs_brotli__error__too_many_compound_dictionaries) { + RETURN_FAIL("16th raw: have \"%s\", want \"%s\"", status.repr, + wuffs_brotli__error__too_many_compound_dictionaries); + } + status = wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, prefix_s, workbuf); + if (status.repr != wuffs_brotli__error__too_many_compound_dictionaries) { + RETURN_FAIL("16th prefix: have \"%s\", want \"%s\"", status.repr, + wuffs_brotli__error__too_many_compound_dictionaries); + } + // A serialized dictionary without a prefix is still OK. + CHECK_STATUS("add_dictionary #1", + wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, words_s, + workbuf)); + + // Only one serialized dictionary can have custom word or transform lists, + // but a serialized dictionary without them (only a prefix) is OK. + CHECK_STATUS("initialize", wuffs_brotli__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__DEFAULT_OPTIONS)); + CHECK_STATUS("add_dictionary #2", + wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, full_s, + workbuf)); + status = wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, words_s, workbuf); + if (status.repr != + wuffs_brotli__error__unsupported_multiple_custom_static_dictionaries) { + RETURN_FAIL( + "words after full: have \"%s\", want \"%s\"", status.repr, + wuffs_brotli__error__unsupported_multiple_custom_static_dictionaries); + } + CHECK_STATUS("add_dictionary #3", + wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, prefix_s, + workbuf)); + // Attaching a prefix-only serialized dictionary resets the static + // dictionary to the built-in one, so custom lists can be attached again. + CHECK_STATUS("add_dictionary #4", + wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, words_s, + workbuf)); + + // Invalid serialized dictionaries: every proper prefix of a valid one, and + // a bad magic number. A failed add_dictionary does not change the + // workbuf_len. + CHECK_STATUS("initialize", wuffs_brotli__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__DEFAULT_OPTIONS)); + for (size_t i = 0; i < full_s.len; i++) { + status = wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, + wuffs_base__make_slice_u8(full_s.ptr, i), workbuf); + if (status.repr != wuffs_brotli__error__bad_shared_dictionary) { + RETURN_FAIL("prefix length %zu: have \"%s\", want \"%s\"", i, + status.repr, wuffs_brotli__error__bad_shared_dictionary); + } + } + full_s.ptr[0] ^= 1; + status = wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, full_s, workbuf); + full_s.ptr[0] ^= 1; + if (status.repr != wuffs_brotli__error__bad_shared_dictionary) { + RETURN_FAIL("bad magic: have \"%s\", want \"%s\"", status.repr, + wuffs_brotli__error__bad_shared_dictionary); + } + if (wuffs_brotli__decoder__workbuf_len(&dec).max_incl != + WUFFS_BROTLI__DECODER_WORKBUF_LEN_STANDARD) { + RETURN_FAIL("workbuf_len changed"); + } + + // A too-short workbuf. + status = wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, raw_s, + wuffs_base__make_slice_u8(workbuf.ptr, 99)); + if (status.repr != wuffs_base__error__bad_workbuf_length) { + RETURN_FAIL("short workbuf: have \"%s\", want \"%s\"", status.repr, + wuffs_base__error__bad_workbuf_length); + } + CHECK_STATUS("add_dictionary #5", + wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, raw_s, + wuffs_base__make_slice_u8(workbuf.ptr, 100))); + + // A transform_io workbuf that is shorter than the dictionaries. + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = g_src_slice_u8, + }); + wuffs_base__io_buffer have = ((wuffs_base__io_buffer){ + .data = g_have_slice_u8, + }); + CHECK_STRING(read_file(&src, "test/data/brotli/romeo.txt.q11w22.br")); + status = wuffs_brotli__decoder__transform_io( + &dec, &have, &src, wuffs_base__make_slice_u8(workbuf.ptr, 99)); + if (status.repr != wuffs_base__error__bad_workbuf_length) { + RETURN_FAIL("transform_io: have \"%s\", want \"%s\"", status.repr, + wuffs_base__error__bad_workbuf_length); + } + + // Dictionaries cannot be added after decoding has started. + CHECK_STATUS("initialize", wuffs_brotli__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__DEFAULT_OPTIONS)); + src.meta.ri = 0; + src.meta.wi = 0; + src.meta.closed = false; + // With no input, decoding has not started (the C code is still in its + // BROTLI_STATE_UNINITED state). + status = wuffs_brotli__decoder__transform_io(&dec, &have, &src, workbuf); + if (status.repr != wuffs_base__suspension__short_read) { + RETURN_FAIL("transform_io #0: have \"%s\", want \"%s\"", status.repr, + wuffs_base__suspension__short_read); + } + CHECK_STATUS("add_dictionary #6", + wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, raw_s, workbuf)); + CHECK_STRING(read_file(&src, "test/data/brotli/romeo.txt.q11w22.br")); + src.meta.wi = 1; + src.meta.closed = false; + status = wuffs_brotli__decoder__transform_io(&dec, &have, &src, workbuf); + if (status.repr != wuffs_base__suspension__short_read) { + RETURN_FAIL("transform_io #1: have \"%s\", want \"%s\"", status.repr, + wuffs_base__suspension__short_read); + } + status = wuffs_brotli__decoder__add_dictionary( + &dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, raw_s, workbuf); + if (status.repr != wuffs_base__error__bad_call_sequence) { + RETURN_FAIL("after start: have \"%s\", want \"%s\"", status.repr, + wuffs_base__error__bad_call_sequence); + } + return NULL; +} + +// ---------------- Large Window and Dictionary Mimic Tests + +#ifdef WUFFS_MIMIC + +// do_test_mimic_brotli_differential_with_options is like +// do_test_mimic_brotli_differential but with the large-window quirk and +// attached dictionaries. It sets *skipped (and returns NULL) if libbrotlidec +// cannot attach the dictionaries: by default, it does not support serialized +// dictionaries. +const char* // +do_test_mimic_brotli_differential_with_options(const char* name, + wuffs_base__io_buffer* src, + const char* want_filename, + bool want_error, + bool large_window, + size_t dict_set, + bool* skipped) { + *skipped = false; + wuffs_base__io_buffer have = ((wuffs_base__io_buffer){ + .data = g_have_slice_u8, + }); + wuffs_base__io_buffer want = ((wuffs_base__io_buffer){ + .data = g_want_slice_u8, + }); + wuffs_base__io_buffer src0 = *src; + const char* have_status = wuffs_brotli_decode_with_options( + &have, &src0, UINT64_MAX, UINT64_MAX, large_window, dict_set); + + size_t num_dicts = 0; + uint32_t dict_types[BROTLI_DICT_SPECS_MAX] = {0}; + wuffs_base__slice_u8 dicts[BROTLI_DICT_SPECS_MAX] = {0}; + CHECK_STRING(load_brotli_dict_set(dict_set, &num_dicts, dict_types, dicts)); + wuffs_base__io_buffer src1 = *src; + int error_code = 0; + const char* want_status = mimic_brotli_decode_with_options( + &want, &src1, large_window, num_dicts, dict_types, dicts, &error_code); + if (want_status == mimic_brotli_attach_dictionary_failed_message) { + *skipped = true; + return NULL; + } + + if ((have_status == NULL) != (want_status == NULL)) { + RETURN_FAIL("%s: have \"%s\", want \"%s\" (error code %d)", name, + have_status, want_status, error_code); + } else if ((have_status != NULL) && + (brotli_c_error_code(have_status) != error_code)) { + RETURN_FAIL("%s: have \"%s\", want error code %d", name, have_status, + error_code); + } else if ((have_status != NULL) != want_error) { + RETURN_FAIL("%s: have \"%s\", want_error %d", name, have_status, + (int)want_error); + } else if (!want_error && (src0.meta.ri != src1.meta.ri)) { + RETURN_FAIL("%s: ri: have %zu, want %zu", name, src0.meta.ri, src1.meta.ri); + } + if (want_error) { + if (have.meta.wi > want.meta.wi) { + have.meta.wi = want.meta.wi; + } else { + want.meta.wi = have.meta.wi; + } + } + const char* status = check_io_buffers_equal("", &have, &want); + if (status) { + RETURN_FAIL("%s: %s", name, status); + } + + // Also check the golden file. + if (want_filename) { + wuffs_base__io_buffer golden = ((wuffs_base__io_buffer){ + .data = g_want_slice_u8, + }); + CHECK_STRING(read_file(&golden, want_filename)); + status = check_io_buffers_equal("", &have, &golden); + if (status) { + RETURN_FAIL("%s: golden: %s", name, status); + } + } + return NULL; +} + +const char* // +test_mimic_brotli_decode_ext_differential() { + CHECK_FOCUS(__func__); + size_t n = WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_ext_vectors); + size_t num_skipped = 0; + for (size_t i = 0; i < n; i++) { + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = g_src_slice_u8, + }); + CHECK_STRING(read_file(&src, g_brotli_ext_vectors[i].src_filename)); + bool skipped = false; + CHECK_STRING(do_test_mimic_brotli_differential_with_options( + g_brotli_ext_vectors[i].src_filename, &src, + g_brotli_ext_vectors[i].want_filename, false, + g_brotli_ext_vectors[i].large_window, + g_brotli_ext_vectors[i].dict_set, &skipped)); + num_skipped += skipped ? 1 : 0; + } + // Only the serialized dictionary files (6 of them) can be skipped. + if (num_skipped > 6) { + RETURN_FAIL("too many skipped: %zu", num_skipped); + } + return NULL; +} + +const char* // +test_mimic_brotli_decode_ext_malformed() { + CHECK_FOCUS(__func__); + size_t n = WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_ext_malformed); + for (size_t i = 0; i < n; i++) { + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = g_src_slice_u8, + }); + if (g_brotli_ext_malformed[i].src_filename) { + CHECK_STRING(read_file(&src, g_brotli_ext_malformed[i].src_filename)); + } else { + memcpy(src.data.ptr, g_brotli_ext_malformed[i].data, + g_brotli_ext_malformed[i].len); + src.meta.wi = g_brotli_ext_malformed[i].len; + src.meta.closed = true; + } + char name[64]; + snprintf(name, sizeof name, "g_brotli_ext_malformed[%zu]", i); + bool skipped = false; + CHECK_STRING(do_test_mimic_brotli_differential_with_options( + name, &src, NULL, true, g_brotli_ext_malformed[i].large_window, + g_brotli_ext_malformed[i].dict_set, &skipped)); + } + return NULL; +} + +// mimic_brotli_decode_large_window adapts mimic_brotli_decode_with_options +// to do_test_brotli_decode_large_window_far's codec_func type. +const char* // +mimic_brotli_decode_large_window(wuffs_base__io_buffer* dst, + wuffs_base__io_buffer* src, + uint64_t wlimit, + uint64_t rlimit, + bool large_window, + size_t dict_set) { + if ((wlimit < UINT64_MAX) || (rlimit < UINT64_MAX) || + (dict_set != BROTLI_DICT_SET_NONE)) { + return "unsupported option"; + } + int error_code = 0; + return mimic_brotli_decode_with_options(dst, src, large_window, 0, NULL, + NULL, &error_code); +} + +const char* // +test_mimic_brotli_decode_large_window_far() { + CHECK_FOCUS(__func__); + return do_test_brotli_decode_large_window_far( + mimic_brotli_decode_large_window, UINT64_MAX, UINT64_MAX); +} + +#endif // WUFFS_MIMIC + // ---------------- Brotli Benches const char* // @@ -674,6 +1524,14 @@ proc g_tests[] = { test_wuffs_brotli_decode_truncated, test_wuffs_brotli_decode_workbuf_length, + // Large window and dictionaries. + test_wuffs_brotli_add_dictionary, + test_wuffs_brotli_decode_ext_all, + test_wuffs_brotli_decode_ext_all_with_io_limits, + test_wuffs_brotli_decode_ext_malformed, + test_wuffs_brotli_decode_large_window_far, + test_wuffs_brotli_decode_large_window_quirk, + #ifdef WUFFS_MIMIC test_mimic_brotli_decode_alice29, @@ -682,6 +1540,11 @@ proc g_tests[] = { test_mimic_brotli_decode_malformed, test_mimic_brotli_decode_romeo, + // Large window and dictionaries. + test_mimic_brotli_decode_ext_differential, + test_mimic_brotli_decode_ext_malformed, + test_mimic_brotli_decode_large_window_far, + #endif // WUFFS_MIMIC NULL, diff --git a/test/data/brotli/README.md b/test/data/brotli/README.md index ae5add90..e24dace4 100644 --- a/test/data/brotli/README.md +++ b/test/data/brotli/README.md @@ -27,3 +27,57 @@ the other (valid) files and keeping, for each libbrotlidec v1.2.1 error code, the smallest mutated file that produced that error code. The file name mentions that error code, e.g. `invalid.cl-space.br` produces `BROTLI_DECODER_ERROR_FORMAT_CL_SPACE`. + + +## Large Window and Dictionary Test Data + +These files exercise the `QUIRK_LARGE_WINDOW` quirk and `add_dictionary`. They +were generated with brotli v1.2.1: the command line tool where it suffices +and otherwise a small C program calling `BrotliEncoderSetParameter` (with +`BROTLI_PARAM_LARGE_WINDOW`), `BrotliEncoderPrepareDictionary` and +`BrotliEncoderAttachPreparedDictionary`, linked against a libbrotlienc built +with `BROTLI_EXPERIMENTAL` (needed for serialized shared dictionaries). Every +file's decompressed form was checked with libbrotlidec (also built with +`BROTLI_EXPERIMENTAL`). + +- `*.qNlwM.br` are large-window streams (quality N, window bits M), e.g. + `brotli -q 11 --large_window=26 -o alice29.txt.q11lw26.br alice29.txt`. +- `far17m.q9lw25.br` is a large-window (25 window bits) quality 9 stream whose + decompressed form (17 MiB plus 128 KiB, generated by the test program + instead of being checked in) is 64 KiB of xorshift32 output, 17 MiB of + zeroes and the same 64 KiB again. The second copy refers back more than (1 + << 24) bytes. A standard (24 window bits) encoding is twice as large. +- `*.raw-romeo.br` use `../romeo.txt` as a raw (LZ77 prefix) dictionary, e.g. + `brotli -q 11 -w 22 -D ../romeo.txt -o romeo.txt.q11w22.raw-romeo.br + ../romeo.txt`. +- `*.raw-chunks.br` use several raw dictionaries (compound dictionary chunks): + `alice29.txt`'s bytes `[0 .. 20000)`, `[20000 .. 20001)`, `[20001 .. 20001)` + (empty) and `[20001 .. 60000)`, or `../romeo.txt`'s bytes `[0 .. 300)`, `[300 + .. 500)` and `[500 .. 942)`. +- `shared-dictionary.*.bin` are serialized shared dictionaries (the format + parsed by `BrotliSharedDictionaryAttach` in brotli's + `c/common/shared_dictionary.c`), built by a script: + - `full` has a 500 byte LZ77 prefix (the start of `../romeo.txt`), two + custom word lists (common words from `alice29.txt` and `asyoulik.txt`), + two custom transform lists (including `BROTLI_TRANSFORM_SHIFT_FIRST` and + `BROTLI_TRANSFORM_SHIFT_ALL` transforms), three (word list, transform + list) combinations (the third is the built-in pair) and a context map. + - `words` has one custom word list, the built-in transforms and no prefix. + - `prefix` has only an LZ77 prefix (the start of `asyoulik.txt`). + - `utf8` has a word list of UTF-8 (and invalid UTF-8) words and shift + transforms with various parameters. +- `shared-dictionary.text` is made of `shared-dictionary.full.bin` words, + transformed by its transforms. `*.sd-full.br` (and `*.sd-words.br`) were + compressed with `shared-dictionary.full.bin` (and `.words.bin`) attached. + `asyoulik.txt.q11w18.sd-prefix-raw.br` was compressed with + `shared-dictionary.prefix.bin` and then `../romeo.txt`'s first 300 bytes + (raw) attached. +- `romeo-mutated.sd-utf8.br` is a mutated `romeo.txt.q5w10.br` that decodes + (to `romeo-mutated.sd-utf8.txt`) without error when + `shared-dictionary.utf8.bin` is attached, and whose dictionary references + apply shift transforms to 1, 2, 3 and 4 byte UTF-8 sequences. +- `invalid.lw-*.br`, `invalid.raw-*.br` and `invalid.sd-*.br` are malformed. + They were found by mutating the files above, like the other `invalid.*.br` + files, with the large-window quirk set (`lw`), or the raw (`raw`, the + `romeo.txt` chunks) or serialized (`sd`, `shared-dictionary.full.bin`) + dictionaries attached. diff --git a/test/data/brotli/alice29.txt.q11lw26.br b/test/data/brotli/alice29.txt.q11lw26.br new file mode 100644 index 00000000..3d6e8ff2 Binary files /dev/null and b/test/data/brotli/alice29.txt.q11lw26.br differ diff --git a/test/data/brotli/alice29.txt.q11w18.raw-chunks.br b/test/data/brotli/alice29.txt.q11w18.raw-chunks.br new file mode 100644 index 00000000..3ae68070 Binary files /dev/null and b/test/data/brotli/alice29.txt.q11w18.raw-chunks.br differ diff --git a/test/data/brotli/alice29.txt.q11w22.sd-words.br b/test/data/brotli/alice29.txt.q11w22.sd-words.br new file mode 100644 index 00000000..580045d6 Binary files /dev/null and b/test/data/brotli/alice29.txt.q11w22.sd-words.br differ diff --git a/test/data/brotli/alice29.txt.q5lw10.br b/test/data/brotli/alice29.txt.q5lw10.br new file mode 100644 index 00000000..5b1a80d0 Binary files /dev/null and b/test/data/brotli/alice29.txt.q5lw10.br differ diff --git a/test/data/brotli/alice29.txt.q5w18.raw-chunks.br b/test/data/brotli/alice29.txt.q5w18.raw-chunks.br new file mode 100644 index 00000000..7c7503cf Binary files /dev/null and b/test/data/brotli/alice29.txt.q5w18.raw-chunks.br differ diff --git a/test/data/brotli/asyoulik.txt.q11w18.sd-prefix-raw.br b/test/data/brotli/asyoulik.txt.q11w18.sd-prefix-raw.br new file mode 100644 index 00000000..122144b5 Binary files /dev/null and b/test/data/brotli/asyoulik.txt.q11w18.sd-prefix-raw.br differ diff --git a/test/data/brotli/far17m.q9lw25.br b/test/data/brotli/far17m.q9lw25.br new file mode 100644 index 00000000..f50f26c8 Binary files /dev/null and b/test/data/brotli/far17m.q9lw25.br differ diff --git a/test/data/brotli/invalid.lw-dictionary.br b/test/data/brotli/invalid.lw-dictionary.br new file mode 100644 index 00000000..8f42f78b Binary files /dev/null and b/test/data/brotli/invalid.lw-dictionary.br differ diff --git a/test/data/brotli/invalid.lw-distance.br b/test/data/brotli/invalid.lw-distance.br new file mode 100644 index 00000000..3a0dbc49 Binary files /dev/null and b/test/data/brotli/invalid.lw-distance.br differ diff --git a/test/data/brotli/invalid.lw-sd-transform.br b/test/data/brotli/invalid.lw-sd-transform.br new file mode 100644 index 00000000..649c6e0f Binary files /dev/null and b/test/data/brotli/invalid.lw-sd-transform.br differ diff --git a/test/data/brotli/invalid.raw-compound-dictionary.br b/test/data/brotli/invalid.raw-compound-dictionary.br new file mode 100644 index 00000000..9e4a396c Binary files /dev/null and b/test/data/brotli/invalid.raw-compound-dictionary.br differ diff --git a/test/data/brotli/invalid.raw-dictionary.br b/test/data/brotli/invalid.raw-dictionary.br new file mode 100644 index 00000000..8d05a472 Binary files /dev/null and b/test/data/brotli/invalid.raw-dictionary.br differ diff --git a/test/data/brotli/invalid.raw-transform.br b/test/data/brotli/invalid.raw-transform.br new file mode 100644 index 00000000..05585ddc Binary files /dev/null and b/test/data/brotli/invalid.raw-transform.br differ diff --git a/test/data/brotli/invalid.sd-compound-dictionary.br b/test/data/brotli/invalid.sd-compound-dictionary.br new file mode 100644 index 00000000..c635954f --- /dev/null +++ b/test/data/brotli/invalid.sd-compound-dictionary.br @@ -0,0 +1 @@ +âvŒ¸)MëÍ{> ài©Ã`n gήy¼€ÈfÏò0¬aQIÿËS)”߇„ \ No newline at end of file diff --git a/test/data/brotli/invalid.sd-dictionary.br b/test/data/brotli/invalid.sd-dictionary.br new file mode 100644 index 00000000..eefad2e7 --- /dev/null +++ b/test/data/brotli/invalid.sd-dictionary.br @@ -0,0 +1 @@ +âvŒ¸)MëÍ{> ài©Ã`n gήy¼€ÈfÏò‰Aióûì×baQIÿË])”Þ‡„rÁ| ài©Ã`n gήy¼€ÈfÏò0­�¯>n€‰Áióû \ No newline at end of file diff --git a/test/data/brotli/romeo-mutated.sd-utf8.br b/test/data/brotli/romeo-mutated.sd-utf8.br new file mode 100644 index 00000000..dde030d3 Binary files /dev/null and b/test/data/brotli/romeo-mutated.sd-utf8.br differ diff --git a/test/data/brotli/romeo-mutated.sd-utf8.txt b/test/data/brotli/romeo-mutated.sd-utf8.txt new file mode 100644 index 00000000..01c167a8 --- /dev/null +++ b/test/data/brotli/romeo-mutated.sd-utf8.txt @@ -0,0 +1,45 @@ +Romeo and Juliet +Excerpt Ÿ˜€ Act 2, Scene 2 + +JULIET +O €xy,ULIET +! linksfore art thouULIET +? +Deny thy signedo andtake p thy name; +Or, Contaou wilt not, be but sworn my `₫é, +And I'll the worlde bua Capulet. + +ROMEO +[Aside] Sif(! I usin more, or sif(! I Perso athouis?JULIET +O 'Tisut swy name; + Italiany `âenemy; +Tu wit thouyhold,aou wghot, a Montague. +What's Montague? remaininr hand,ninr foot, +Nnr arm,ninr face,ninr They were part +Be woring to a man. Obe busomey were me; +! +What's in a me; +? Italilinks we cf(! a four +By They were me; + situatsme(! as onkey; +SoIET +? + situa, were heot, ET +? + cf(!'d, +Retaoutbrat dsin perf + +Whililinks he owes +Wiou wttbrat نك.€xy,ULd + +Many name;©, +Andf; widthe me; + linksmainie par =falsee +Takeo ll muyholt. + +ROMEOI takeolseeo athoy das: +Co ll e but `â‚«Ão andd I'lbe new missized; +Hencsforth I sound investor€xy.s?JULIET +! +Wh a m art thouU? Itahide be🗿èé'd in €¿øÿ; +Sostumblest o my `counsel? diff --git a/test/data/brotli/romeo.txt.q11lw30.br b/test/data/brotli/romeo.txt.q11lw30.br new file mode 100644 index 00000000..5b653d37 Binary files /dev/null and b/test/data/brotli/romeo.txt.q11lw30.br differ diff --git a/test/data/brotli/romeo.txt.q11lw30.raw-romeo.br b/test/data/brotli/romeo.txt.q11lw30.raw-romeo.br new file mode 100644 index 00000000..3f0ead0c Binary files /dev/null and b/test/data/brotli/romeo.txt.q11lw30.raw-romeo.br differ diff --git a/test/data/brotli/romeo.txt.q11w16.raw-chunks.br b/test/data/brotli/romeo.txt.q11w16.raw-chunks.br new file mode 100644 index 00000000..82e80283 Binary files /dev/null and b/test/data/brotli/romeo.txt.q11w16.raw-chunks.br differ diff --git a/test/data/brotli/romeo.txt.q11w22.raw-romeo.br b/test/data/brotli/romeo.txt.q11w22.raw-romeo.br new file mode 100644 index 00000000..55ea79e6 Binary files /dev/null and b/test/data/brotli/romeo.txt.q11w22.raw-romeo.br differ diff --git a/test/data/brotli/romeo.txt.q9lw28.br b/test/data/brotli/romeo.txt.q9lw28.br new file mode 100644 index 00000000..64554f69 Binary files /dev/null and b/test/data/brotli/romeo.txt.q9lw28.br differ diff --git a/test/data/brotli/shared-dictionary.full.bin b/test/data/brotli/shared-dictionary.full.bin new file mode 100644 index 00000000..05462139 Binary files /dev/null and b/test/data/brotli/shared-dictionary.full.bin differ diff --git a/test/data/brotli/shared-dictionary.prefix.bin b/test/data/brotli/shared-dictionary.prefix.bin new file mode 100644 index 00000000..a4742ae7 Binary files /dev/null and b/test/data/brotli/shared-dictionary.prefix.bin differ diff --git a/test/data/brotli/shared-dictionary.text b/test/data/brotli/shared-dictionary.text new file mode 100644 index 00000000..018aafae --- /dev/null +++ b/test/data/brotli/shared-dictionary.text @@ -0,0 +1,41 @@ +causeHAVE  +uncoTURTLE, uhoughtfullycharles -audrey-fatherdaughter Somethingsheph HASTILY, phebeSHOULDSHEPHERDInterruptedlike fortune things. hoingextraordinary. erhaps said aterpillar Wonderrabbi well everything's Everything charlesthere corin anything's -silvius-WILLIAMDisappointment better RepliedDAUGHTER will thoughtfully  QuestionsilviusAnythingturtle + looking mucquite +charlesthe disappointment INTERRUPTED, that FORTUNEuncomfortable, daughter Childrenph the uncomfortable right + -love-bettermistresstraordinaryreplied's -brother-derstand there t yourorlando-should-senior bnmudqr`shnm ph this -should-  +haveSENIOR the children extraordinary. wouldcon Dormousenversation-audrey- h silvius knnjdc sen the executioner disappointmentcorin oli y okingexecutioner children g`ssdq phebe -daughter-w adventures's -jaques-frightened's opportunity's the interrupted w mistress little. thoughtfully + anything -should- THATContemptuouslyeconthings. contemptuouslrself t corin CORIN-mistress-will noonqstmhsx beginning Extraordinaryextraordinary conversatioOpportunity -father-getting. enter conversatioPRITHEEcontesheph mockanother. HAVE inute l thoughtfully's withperhaps-orlando-seemed + Wonderforestthe frightened which willremember. have co THAT, daughter -william-Disappointmenttuchdaughter  -audrey-LOOKED, the turtle orlandofather sh Interrupted beginning could's   rosalindhave xtraordinary Looking jaquesnversationuncomfortablable disappointment phebe -orlando-dormouse mistresshoughtfully THERE nterrupted op thoughtfullextraordinary, -there-contemptuouslySHALLshall jaq creatures -senior-phebe t CREATURES, sheph ed Thoughtfullythe nothing LookingINTO, everything con Alice-prithee-charlesursewhichlooked rather uWhile White DAUGHTERDisappointment love your Opportunitydifferent. rosalind -would-EVERYTHING, -father- father -oliver-ANOTHER, en enough celia-forest- the thought ROSALINDseniorFORTUNEFATHERdhildren uncomfortable's ce -that-love mistr ONLY, should phebe-brother-l eyeifferent orlandoInterrupted-senior-pheberueen ExtraordinaryBEFORE, shepherdCHARLESbetter should brotherfortune t -orlando-audrey mistr daugh will sen disappointmentextraordinarythe things withieardokd`rd HAVE disappointment, sh ph noonqstmhsx very-oliver-ce daughterbdqs`hmkx w BROTHERTHISl ORLANDO the other oli will nversationconversatiowhich ously mistr  ce rosalind because's thoughtfull disappointment -this-AUDREYdisappointment mistr the understand frightened +y gdqrdke Creatures onversation AUDREY conversation went's extraordinary's WITHextrautifulanotheren interrupted -rosalind-orlando-orlando-Extraordinaryeirrosalind suddenly, -charles-charlesvoice +bnythingonversation t CHARLES-another--silvius-contemptuously +UNCOMFORTABLE, caterpillashepherd uncomfortable -father-father would xtraordinary SHOULDvgdm JAQUES-there--shepherd-creatures. Little ROSALIND-will- DAUGHTER-another-daugh rosal without's nary curious. ckaud would MISTRESScharles thou shallCertainly-orlando-extraordinary +seniordaughter thoughtfullshepherdWITHonly. -senior-Continued this EXTRAORDINARY, fortune  +THAT-audrey-ufat more -charles-things. BROTHERFRIGHTENED, sghmf THERElovethe certainly aud this -will- brotherthe getting  + with's shepherdoli THIS -another-the seemed xiouslyoliver-shepherd-caterpillar +which wh daugh mistress Opportunity getting -rosalind-dmntfg ANOTHERthoughtfully SHOULDprithee oliverbet eisappointment thou certainly With methingsho replied.  + this shepherdcharlesrosalindbetter en sheph -which--orlando--celia-rightened ke fat the beautiful will HAVE-have--love-WOULD thisfor MUCH, well's thouprit contemptuously -celia-vncomfortablelly-oliver- finishelovethe conversation thoughtfullyOLIVER-enter-ar LOOKED, onversation -love-phebe Conversationuncomfortable, extraordinary +mistressbefore. onder adventures have-thou-SILVIUS-silvius-conversation's t more's daughtersho -shepherd-white, tmcdqrs`mc rosal thingtalking,   alice. ormouse Thinking remember Round  Perhapsthere for MISTRESS beginning. your-this-anxiouslmistress UNCOMFORTABLE, Uncomfortable -phebe-wh etting CONTEMPTUOUSLY, silvius isappointment thoadventures's extraordinar caterpillar mistressthe interrupted   +   rosal seniorreat SILVIUSnough daughter able contecorin mistress-charles-going +Contemptuously brot nderstand HAVEAUDREY your celia -should- willCONTEMPTUOUSLY, -audrey-sho thiseginning char willtmentmarch's sheph fverythingconversation. the mouse -audrey-there silv othing willonly. the understand LOVEsheph daugh BeforekingecutionerConversation AUDREYPRITHEE-which-the should -jaques- should. withginishedbet tmbnlenqs`akd like. beginning -should-rosalind NOTHING, bnmsdlostntrkx conversation   +large. something-rosalind--mistress--another--there--william-Contemptuously That executioner's thoughtfully, thinking However fat uhen shouldperhapDAUGHTERopSHOULD disappointment the extraordinary thereqerhapscorinThoughtfullywould contemptuously orlando terpillarmistress about -rosalind--fortune-abbit thereanot interrupted-your-this  such -daughter-the queen ph Conversationsgntfgsetkkx orlando silvius -father- Anything extraordinary Before Begant there uhemthe could THATperhaps ruestionsgntfgsetkkx aud rosal getting +mistr mistr extraordinary + ierecharles thinkingYOURsilv t interrupted +which audrey daugh thothis Disappointment extraordinary, your voice edEXTRAORDINARY, comfortable conversation,  -prithee-h HEARD, which everythin xtraordinary understand children white. -father-WhatThoughtfully co silviusmust +ightenedrosal would somethin -father--shall--would-extraordinary's SHEPHERDextrjaq there InterruptedInterrupted THATsen SENIORyour conversation +celia -celia--better--love-   SHOULDphebe OLIVERmistr remember Uncomfortable -jaques-rosal aud conversation THOUGHTFULLY, the before WOULD-forest-ere -mistress-williambrother -will--that-THAT-prithee-for fatherWILL rosalindxecutioner understand's much, brother come -mistress-dwdbtshnmdq SILVIUSaudrey nomentonversation -brother- ROSALINDyourppportunitywithshepherden corin jaques withthis EVERYTHING, the were -fortune- vncomfortablemistress  celiarosalindthosghmfr remember WILLIAMaud Disappointmentcorin fxecutionerntemptuously daughterell Extraordinaryad`tshetk thouopportunity venturesthe come -shepherd-uchess caterpillar. mistressSHOULD   -rosalind-celia ROSALINDntemptuouslyEXTRAORDINARY, know fortune the again THEREthere disappointmenthoughtfullygryphon. SHOULDmistr CORIN-shall-thinkingEVERYTHING, OLIVERSomething DAUGHTERyourOLIVERshepherd thoughtfully. -orlando-FORESTBROTHERrselffortune -which-y lovephebe-celia-dontemptuouslyund Executionerlike Looking-prithee-WOULDlove-forest- -thou--father-ConversationrosalindCreatures fortune uncomfortable phebeinterrupted WILLIAMSHEPHERDDisappointment wh xecutioner should rosalind jaquesDAUGHTERremembe SILVIUSfverythingaterpillar HAVEthe remember Executionerw theirwent charlesbetterhead DAUGHTERcontemptuousl FORTUNEbrot CATERPILLAR, eard gryphon +WOULDshepherd enterloveuncomfortable, Understand-oliver-ENTERthe quite -rosalind-fortunedaugh bettercontemptuouslsomCELIA the voice betterthey, -should-audreywould co understandTHOUnversation  + lice -enter- Caterpillar THINKING, shepherd Disappointmentoliver-mistress-senior co WILLIAMBROTHERrosal jaquesORLANDOuncomfortable + executioney your, FORTUNE-there-mistr mistr sheph with +en mistress the without Contemptuouslyphebe -daughter-eautiful nterrupted orlandoopportunity's contemptuouslybeginning + brot Going anot usecurious, w shepherd dormouse + uncomfortable Beautifulgryphon's MISTRESSdaughter Everything erythingEXTRAORDINARY, Thoughtfully -charles-extraordinary, ion  while +forest daugh SILVIUSyour Caterpillar en suddenly vnmcdq PHEBE-shepherd-shoulddventures fortune understand +EXTRAORDINARY, Adventuresdontemptuouslyable hastily, thoughtfully. Frightened which h mistr mistress  -shall-enterthe understand -better-uestion Contemptuously ph willinterrupted Gettingknow, shall into. hmsn Mock Continued seemed have extraordinary's march, celiaWHICHb`sdqohkk`q SILVIUSunderstanDAUGHTER the caterpillar dudqxsghmf better -another-THERE brother understand. -there-phebe ableYOUR-celia- t FORTUNE AUDREY-shepherd-enter extraordinar mistress WHICHchildren's disappointment's BROTHERddenlyGetting tmbnlenqs`akd DAUGHTERTHERE-silvius-vncomfortablerosal remember, BETTERWITHdaugh Creaturesdwithout, contemptuously's PHEBEdaughterWILLthincelia opportunit ightenedextraordinary -oliver-Extraordinarycorinicethoughtfully + Thoughtfullythis-that- LookedConversation vhsg uch large littlorla -corin-mistress there-this- conversation. extraordinary's thoughtfullExtraordinary must, portunityDAUGHTER conversation's william Everything-daughter-frightened's sho Finished another, mistr CONTEMPTUOUSLY, -love-the creatures sheph the suddenly orla seemed. vncomfortablefor ith w senior rabbitmistress ANOTHERtho-daughter--shepherd-remember's fxtraordinary-shall-opportunity-audrey-our Thoughtorlando -shall--shall- BeginningInterrupted FINISHED, tmentorlando -charles-uhemwhat's -orlando-contemptuously's tuddenlyCOURSE, CHILDREN, phebe -celia- something betterunderstandCHARLES thoughtfully, different's white -there-Frightened y silvius orla jnterruptedFORTUNEph -thou-hoing WILLIAMturtle's rabbithe beautiful Thoughtfully disappointment. Beginning VERY, seemed said ph char HOWEVER, rosalind BeautifulCURIOUS, RABBIT, extraordinary +Anything SILVIUSextraordinar thoughtfully. -better-your shall dontemptuouslyfort SHEPHERD daterpillarsh daughter oli disappointmententer Children -jaques-haves`kjhmf orlando SENIORwhich daughter down Uncomfortable rosalindrosal co mistr might. -should-phebe wo loveq`aahs opportunity. Uncomfortable oprosalind ooking -which-thou the thinking marchbefore, -with--there-bet t oliverfortune-celia-jaquesprithee th -your- -fortune--senior-disappointment lly-with-CONTEMPTUOUSLY, come. charles WHEN, sh prit onversation the again -shepherd-jaques silvius RoundBROTHERbrot enter corin orlandosabbitWOULD-this-getting. CONVERSATION, which Questioncorin thatORLANDOthe such hings thoughtfully en aud Themmistr rious Contemptuously daughterround, the caterpillar Disappointment art shall INTERRUPTED, goingSHEPHERDsuddenly. -better- THISWILL-orlando-THAT Anxiously contemptuously. FORTUNEagain +-have-HAVETHOUGHTFULLY, nother Uncomfortablesho -forest-corin DAUGHTERHavefrightened everything +would onversation sh -your--with-  shepherd ChildrenorlandoCHARLESbrother Went something -which-rosalind-silvius-anotherdaughter CELIAHerself extraordinary + Enough Extraordinary-prithee-wo corin HAVE-prithee-daughterhoweverce adventures w must phebe extraordinary's SHOULD-brother-traordinaryrosalind  will ouldrosalind the something -that--prithee- king +father  the uncomfortable FOREST First  brotherquestion +WENT, THEREmistr  -prithee- Uncomfortablefort with question outwould WOULDsheph BETTERright's narybrother william tting-will-WHICHSILVIUS-another-onder bnmudqr`shnm JAQUES -better-they's THISphebeWOULD -mistress-Anxiously thouContemptuously With ittle sheph Caterpillardifferent.  + THISThoughtfullybrother -charles- sen betterTHISwould -senior-that that bnmudqr`shnm CURIOUS, \ No newline at end of file diff --git a/test/data/brotli/shared-dictionary.text.q11lw24.sd-full.br b/test/data/brotli/shared-dictionary.text.q11lw24.sd-full.br new file mode 100644 index 00000000..9b8ea4ad Binary files /dev/null and b/test/data/brotli/shared-dictionary.text.q11lw24.sd-full.br differ diff --git a/test/data/brotli/shared-dictionary.text.q11w18.sd-full.br b/test/data/brotli/shared-dictionary.text.q11w18.sd-full.br new file mode 100644 index 00000000..2d09bfef Binary files /dev/null and b/test/data/brotli/shared-dictionary.text.q11w18.sd-full.br differ diff --git a/test/data/brotli/shared-dictionary.text.q5w18.sd-full.br b/test/data/brotli/shared-dictionary.text.q5w18.sd-full.br new file mode 100644 index 00000000..d862e85e Binary files /dev/null and b/test/data/brotli/shared-dictionary.text.q5w18.sd-full.br differ diff --git a/test/data/brotli/shared-dictionary.utf8.bin b/test/data/brotli/shared-dictionary.utf8.bin new file mode 100644 index 00000000..4b2da301 Binary files /dev/null and b/test/data/brotli/shared-dictionary.utf8.bin differ diff --git a/test/data/brotli/shared-dictionary.words.bin b/test/data/brotli/shared-dictionary.words.bin new file mode 100644 index 00000000..7f3a8814 Binary files /dev/null and b/test/data/brotli/shared-dictionary.words.bin differ