From 88c32bc66852455bdac35b68b97dd8c06f9692fb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 00:16:37 +0000 Subject: [PATCH] std/brotli: port bit_cost.c and histogram.c for the encoder Add the bit_cost struct (encode_bit_cost.wuffs, encode_histogram.wuffs): BrotliBitsEntropy, BrotliPopulationCost{Literal,Command,Distance}, FastLog2, the histogram helpers and BrotliBuildHistogramsWithContext, with doubles computed by std/softfloat in the C code's operation order. Histograms and commands are little-endian byte slices with the C structs' memory layout. Add a test-only metablock_tester struct and the script/manual-test-brotli-metablock-parity.c harness, which compares BitsEntropy and PopulationCost* bit for bit with libbrotlienc v1.2.1 (200000 random histograms: 0 mismatches). --- release/c/wuffs-unsupported-snapshot.c | 1030 ++++++++++++++++++ script/manual-test-brotli-metablock-parity.c | 274 +++++ std/brotli/README.md | 9 + std/brotli/encode_bit_cost.wuffs | 320 ++++++ std/brotli/encode_histogram.wuffs | 268 +++++ std/brotli/encode_tester.wuffs | 47 + 6 files changed, 1948 insertions(+) create mode 100644 script/manual-test-brotli-metablock-parity.c create mode 100644 std/brotli/encode_bit_cost.wuffs create mode 100644 std/brotli/encode_histogram.wuffs create mode 100644 std/brotli/encode_tester.wuffs diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 1c27c25e..e234280a 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -9277,8 +9277,12 @@ extern const char wuffs_brotli__error__unsupported_encoder_quality[]; typedef struct wuffs_brotli__decoder__struct wuffs_brotli__decoder; +typedef struct wuffs_brotli__bit_cost__struct wuffs_brotli__bit_cost; + typedef struct wuffs_brotli__encoder__struct wuffs_brotli__encoder; +typedef struct wuffs_brotli__metablock_tester__struct wuffs_brotli__metablock_tester; + #ifdef __cplusplus extern "C" { #endif @@ -9311,6 +9315,16 @@ wuffs_brotli__encoder__initialize( size_t sizeof__wuffs_brotli__encoder(void); +wuffs_base__status WUFFS_BASE__WARN_UNUSED_RESULT +wuffs_brotli__metablock_tester__initialize( + wuffs_brotli__metablock_tester* self, + size_t sizeof_star_self, + uint64_t wuffs_version, + uint32_t options); + +size_t +sizeof__wuffs_brotli__metablock_tester(void); + // ---------------- Allocs // These functions allocate and initialize Wuffs structs. They return NULL if @@ -9335,6 +9349,9 @@ wuffs_brotli__encoder__alloc_as__wuffs_base__io_transformer(void) { return (wuffs_base__io_transformer*)(wuffs_brotli__encoder__alloc()); } +wuffs_brotli__metablock_tester* +wuffs_brotli__metablock_tester__alloc(void); + // ---------------- Upcasts static inline wuffs_base__io_transformer* @@ -9438,6 +9455,20 @@ wuffs_brotli__encoder__transform_io( wuffs_base__io_buffer* a_src, wuffs_base__slice_u8 a_workbuf); +WUFFS_BASE__GENERATED_C_CODE +WUFFS_BASE__MAYBE_STATIC uint64_t +wuffs_brotli__metablock_tester__bits_entropy( + const wuffs_brotli__metablock_tester* self, + wuffs_base__slice_u8 a_population, + uint64_t a_size); + +WUFFS_BASE__GENERATED_C_CODE +WUFFS_BASE__MAYBE_STATIC uint64_t +wuffs_brotli__metablock_tester__population_cost( + const wuffs_brotli__metablock_tester* self, + wuffs_base__slice_u8 a_histogram, + uint32_t a_data_size); + #ifdef __cplusplus } // extern "C" #endif @@ -9771,6 +9802,27 @@ struct wuffs_brotli__decoder__struct { #endif // __cplusplus }; // struct wuffs_brotli__decoder__struct +struct wuffs_brotli__bit_cost__struct { + // Do not access the private_impl's or private_data's fields directly. There + // is no API/ABI compatibility or safety guarantee if you do so. Instead, use + // the wuffs_foo__bar__baz functions. + // + // It is a struct, not a struct*, so that the outermost wuffs_foo__bar struct + // can be stack allocated when WUFFS_IMPLEMENTATION is defined. + + struct { + uint32_t magic; + uint32_t active_coroutine; + wuffs_base__vtable null_vtable; + + } private_impl; + + struct { + wuffs_softfloat__arith f_sf; + } private_data; + +}; // struct wuffs_brotli__bit_cost__struct + struct wuffs_brotli__encoder__struct { // Do not access the private_impl's or private_data's fields directly. There // is no API/ABI compatibility or safety guarantee if you do so. Instead, use @@ -9976,6 +10028,94 @@ struct wuffs_brotli__encoder__struct { #endif // __cplusplus }; // struct wuffs_brotli__encoder__struct +struct wuffs_brotli__metablock_tester__struct { + // Do not access the private_impl's or private_data's fields directly. There + // is no API/ABI compatibility or safety guarantee if you do so. Instead, use + // the wuffs_foo__bar__baz functions. + // + // It is a struct, not a struct*, so that the outermost wuffs_foo__bar struct + // can be stack allocated when WUFFS_IMPLEMENTATION is defined. + + struct { + uint32_t magic; + uint32_t active_coroutine; + wuffs_base__vtable null_vtable; + + uint32_t f_dummy; + } private_impl; + + struct { + wuffs_brotli__bit_cost f_bc; + } private_data; + +#ifdef __cplusplus +#if defined(WUFFS_BASE__HAVE_UNIQUE_PTR) + using unique_ptr = std::unique_ptr; + + // On failure, the alloc_etc functions return nullptr. They don't throw. + + static inline unique_ptr + alloc() { + return unique_ptr(wuffs_brotli__metablock_tester__alloc()); + } +#endif // defined(WUFFS_BASE__HAVE_UNIQUE_PTR) + +#if defined(WUFFS_BASE__HAVE_EQ_DELETE) && !defined(WUFFS_IMPLEMENTATION) + // Disallow constructing or copying an object via standard C++ mechanisms, + // e.g. the "new" operator, as this struct is intentionally opaque. Its total + // size and field layout is not part of the public, stable, memory-safe API. + // Use malloc or memcpy and the sizeof__wuffs_foo__bar function instead, and + // call wuffs_foo__bar__baz methods (which all take a "this"-like pointer as + // their first argument) rather than tweaking bar.private_impl.qux fields. + // + // In C, we can just leave wuffs_foo__bar as an incomplete type (unless + // WUFFS_IMPLEMENTATION is #define'd). In C++, we define a complete type in + // order to provide convenience methods. These forward on "this", so that you + // can write "bar->baz(etc)" instead of "wuffs_foo__bar__baz(bar, etc)". + wuffs_brotli__metablock_tester__struct() = delete; + wuffs_brotli__metablock_tester__struct(const wuffs_brotli__metablock_tester__struct&) = delete; + wuffs_brotli__metablock_tester__struct& operator=( + const wuffs_brotli__metablock_tester__struct&) = delete; +#endif // defined(WUFFS_BASE__HAVE_EQ_DELETE) && !defined(WUFFS_IMPLEMENTATION) + +#if !defined(WUFFS_IMPLEMENTATION) + // As above, the size of the struct is not part of the public API, and unless + // WUFFS_IMPLEMENTATION is #define'd, this struct type T should be heap + // allocated, not stack allocated. Its size is not intended to be known at + // compile time, but it is unfortunately divulged as a side effect of + // defining C++ convenience methods. Use "sizeof__T()", calling the function, + // instead of "sizeof T", invoking the operator. To make the two values + // different, so that passing the latter will be rejected by the initialize + // function, we add an arbitrary amount of dead weight. + uint8_t dead_weight[123000000]; // 123 MB. +#endif // !defined(WUFFS_IMPLEMENTATION) + + inline wuffs_base__status WUFFS_BASE__WARN_UNUSED_RESULT + initialize( + size_t sizeof_star_self, + uint64_t wuffs_version, + uint32_t options) { + return wuffs_brotli__metablock_tester__initialize( + this, sizeof_star_self, wuffs_version, options); + } + + inline uint64_t + bits_entropy( + wuffs_base__slice_u8 a_population, + uint64_t a_size) const { + return wuffs_brotli__metablock_tester__bits_entropy(this, a_population, a_size); + } + + inline uint64_t + population_cost( + wuffs_base__slice_u8 a_histogram, + uint32_t a_data_size) const { + return wuffs_brotli__metablock_tester__population_cost(this, a_histogram, a_data_size); + } + +#endif // __cplusplus +}; // struct wuffs_brotli__metablock_tester__struct + #endif // defined(__cplusplus) || defined(WUFFS_IMPLEMENTATION) #endif // !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__BROTLI) || defined(WUFFS_NONMONOLITHIC) @@ -59528,6 +59668,22 @@ WUFFS_BROTLI__DICTIONARY[122784] WUFFS_BASE__POTENTIALLY_UNUSED = { #define WUFFS_BROTLI__QUIRKS_BASE 716673024u +#define WUFFS_BROTLI__HISTOGRAM_LITERAL_DATA_SIZE 256u + +#define WUFFS_BROTLI__HISTOGRAM_COMMAND_DATA_SIZE 704u + +#define WUFFS_BROTLI__HISTOGRAM_DISTANCE_DATA_SIZE 544u + +#define WUFFS_BROTLI__HISTOGRAM_LITERAL_SIZE 1040u + +#define WUFFS_BROTLI__HISTOGRAM_COMMAND_SIZE 2832u + +#define WUFFS_BROTLI__HISTOGRAM_DISTANCE_SIZE 2192u + +#define WUFFS_BROTLI__F64_HUGE_VAL 9218868437227405312u + +#define WUFFS_BROTLI__F64_1E99 6088095589093318446u + #define WUFFS_BROTLI__STREAM_PROCESSING 0u #define WUFFS_BROTLI__STREAM_FLUSH_REQUESTED 1u @@ -59562,6 +59718,13 @@ WUFFS_BROTLI__DICTIONARY[122784] WUFFS_BASE__POTENTIALLY_UNUSED = { // ---------------- Private Initializer Prototypes +wuffs_base__status WUFFS_BASE__WARN_UNUSED_RESULT +wuffs_brotli__bit_cost__initialize( + wuffs_brotli__bit_cost* self, + size_t sizeof_star_self, + uint64_t wuffs_version, + uint32_t options); + // ---------------- Private Function Prototypes WUFFS_BASE__GENERATED_C_CODE @@ -59911,6 +60074,56 @@ wuffs_brotli__decoder__take_distance_from_ring_buffer( wuffs_brotli__decoder* self, uint32_t a_code); +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__ld32( + const wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_s, + uint64_t a_i); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__ld64( + const wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_s, + uint64_t a_i); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__st32( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_s, + uint64_t a_i, + uint32_t a_v); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__st64( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_s, + uint64_t a_i, + uint64_t a_v); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__fast_log2( + const wuffs_brotli__bit_cost* self, + uint64_t a_v); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__bits_entropy( + const wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_population, + uint64_t a_size); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__population_cost( + const wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_histogram, + uint32_t a_data_size); + WUFFS_BASE__GENERATED_C_CODE static wuffs_base__empty_struct wuffs_brotli__encoder__write_bits( @@ -60463,6 +60676,90 @@ wuffs_brotli__encoder__compress_fragment_two_pass( uint32_t a_table_bits, wuffs_base__slice_u8 a_s); +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__histogram_clear( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_h, + uint32_t a_data_size); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__histogram_add( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_h, + uint32_t a_data_size, + uint64_t a_val); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__histogram_add_histogram( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_h, + wuffs_base__slice_u8 a_v, + uint32_t a_data_size); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__histogram_copy( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_h, + wuffs_base__slice_u8 a_v, + uint32_t a_data_size); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__histogram_total_count( + const wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_h, + uint32_t a_data_size); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__histogram_bit_cost( + const wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_h, + uint32_t a_data_size); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__histogram_set_bit_cost( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_h, + uint32_t a_data_size, + uint64_t a_bit_cost); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__build_histograms_with_context( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_cmds, + uint64_t a_num_commands, + wuffs_base__slice_u8 a_lit_types, + wuffs_base__slice_u8 a_lit_lengths, + uint64_t a_lit_num_blocks, + wuffs_base__slice_u8 a_cmd_types, + wuffs_base__slice_u8 a_cmd_lengths, + uint64_t a_cmd_num_blocks, + wuffs_base__slice_u8 a_dist_types, + wuffs_base__slice_u8 a_dist_lengths, + uint64_t a_dist_num_blocks, + wuffs_base__slice_u8 a_ringbuffer, + uint64_t a_start_pos, + uint64_t a_mask, + uint8_t a_prev_byte, + uint8_t a_prev_byte2, + uint32_t a_context_mode, + wuffs_base__slice_u8 a_lit_histos, + wuffs_base__slice_u8 a_cmd_histos, + wuffs_base__slice_u8 a_dist_histos); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__command_distance_context( + const wuffs_brotli__bit_cost* self, + uint32_t a_cmd_prefix); + // ---------------- VTables const wuffs_base__io_transformer__func_ptrs @@ -60564,6 +60861,56 @@ sizeof__wuffs_brotli__decoder(void) { return sizeof(wuffs_brotli__decoder); } +wuffs_base__status WUFFS_BASE__WARN_UNUSED_RESULT +wuffs_brotli__bit_cost__initialize( + wuffs_brotli__bit_cost* self, + size_t sizeof_star_self, + uint64_t wuffs_version, + uint32_t options){ + if (!self) { + return wuffs_base__make_status(wuffs_base__error__bad_receiver); + } + if (sizeof(*self) != sizeof_star_self) { + return wuffs_base__make_status(wuffs_base__error__bad_sizeof_receiver); + } + if (((wuffs_version >> 32) != WUFFS_VERSION_MAJOR) || + (((wuffs_version >> 16) & 0xFFFF) > WUFFS_VERSION_MINOR)) { + return wuffs_base__make_status(wuffs_base__error__bad_wuffs_version); + } + + if ((options & WUFFS_INITIALIZE__ALREADY_ZEROED) != 0) { + // The whole point of this if-check is to detect an uninitialized *self. + // We disable the warning on GCC. Clang-5.0 does not have this warning. +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + if (self->private_impl.magic != 0) { + return wuffs_base__make_status(wuffs_base__error__initialize_falsely_claimed_already_zeroed); + } +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + } else { + if ((options & WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED) == 0) { + memset(self, 0, sizeof(*self)); + options |= WUFFS_INITIALIZE__ALREADY_ZEROED; + } else { + memset(&(self->private_impl), 0, sizeof(self->private_impl)); + } + } + + { + wuffs_base__status z = wuffs_softfloat__arith__initialize( + &self->private_data.f_sf, sizeof(self->private_data.f_sf), WUFFS_VERSION, options); + if (z.repr) { + return z; + } + } + self->private_impl.magic = WUFFS_BASE__MAGIC; + return wuffs_base__make_status(NULL); +} + wuffs_base__status WUFFS_BASE__WARN_UNUSED_RESULT wuffs_brotli__encoder__initialize( wuffs_brotli__encoder* self, @@ -60638,6 +60985,76 @@ sizeof__wuffs_brotli__encoder(void) { return sizeof(wuffs_brotli__encoder); } +wuffs_base__status WUFFS_BASE__WARN_UNUSED_RESULT +wuffs_brotli__metablock_tester__initialize( + wuffs_brotli__metablock_tester* self, + size_t sizeof_star_self, + uint64_t wuffs_version, + uint32_t options){ + if (!self) { + return wuffs_base__make_status(wuffs_base__error__bad_receiver); + } + if (sizeof(*self) != sizeof_star_self) { + return wuffs_base__make_status(wuffs_base__error__bad_sizeof_receiver); + } + if (((wuffs_version >> 32) != WUFFS_VERSION_MAJOR) || + (((wuffs_version >> 16) & 0xFFFF) > WUFFS_VERSION_MINOR)) { + return wuffs_base__make_status(wuffs_base__error__bad_wuffs_version); + } + + if ((options & WUFFS_INITIALIZE__ALREADY_ZEROED) != 0) { + // The whole point of this if-check is to detect an uninitialized *self. + // We disable the warning on GCC. Clang-5.0 does not have this warning. +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + if (self->private_impl.magic != 0) { + return wuffs_base__make_status(wuffs_base__error__initialize_falsely_claimed_already_zeroed); + } +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + } else { + if ((options & WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED) == 0) { + memset(self, 0, sizeof(*self)); + options |= WUFFS_INITIALIZE__ALREADY_ZEROED; + } else { + memset(&(self->private_impl), 0, sizeof(self->private_impl)); + } + } + + { + wuffs_base__status z = wuffs_brotli__bit_cost__initialize( + &self->private_data.f_bc, sizeof(self->private_data.f_bc), WUFFS_VERSION, options); + if (z.repr) { + return z; + } + } + self->private_impl.magic = WUFFS_BASE__MAGIC; + return wuffs_base__make_status(NULL); +} + +wuffs_brotli__metablock_tester* +wuffs_brotli__metablock_tester__alloc(void) { + wuffs_brotli__metablock_tester* x = + (wuffs_brotli__metablock_tester*)(calloc(1, sizeof(wuffs_brotli__metablock_tester))); + if (!x) { + return NULL; + } + if (wuffs_brotli__metablock_tester__initialize( + x, sizeof(wuffs_brotli__metablock_tester), WUFFS_VERSION, WUFFS_INITIALIZE__ALREADY_ZEROED).repr) { + free(x); + return NULL; + } + return x; +} + +size_t +sizeof__wuffs_brotli__metablock_tester(void) { + return sizeof(wuffs_brotli__metablock_tester); +} + // ---------------- Function Implementations // -------- func brotli.decoder.get_quirk @@ -64485,6 +64902,277 @@ wuffs_brotli__decoder__take_distance_from_ring_buffer( return wuffs_base__make_empty_struct(); } +// -------- func brotli.bit_cost.ld32 + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__ld32( + const wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_s, + uint64_t a_i) { + wuffs_base__slice_u8 v_q = {0}; + + if (a_i < ((uint64_t)(a_s.len))) { + v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i); + if (((uint64_t)(v_q.len)) >= 4u) { + return wuffs_base__peek_u32le__no_bounds_check(v_q.ptr); + } + } + return 0u; +} + +// -------- func brotli.bit_cost.ld64 + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__ld64( + const wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_s, + uint64_t a_i) { + wuffs_base__slice_u8 v_q = {0}; + + if (a_i < ((uint64_t)(a_s.len))) { + v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i); + if (((uint64_t)(v_q.len)) >= 8u) { + return wuffs_base__peek_u64le__no_bounds_check(v_q.ptr); + } + } + return 0u; +} + +// -------- func brotli.bit_cost.st32 + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__st32( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_s, + uint64_t a_i, + uint32_t a_v) { + wuffs_base__slice_u8 v_q = {0}; + + if (a_i < ((uint64_t)(a_s.len))) { + v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i); + if (((uint64_t)(v_q.len)) >= 4u) { + wuffs_base__poke_u32le__no_bounds_check(v_q.ptr, a_v); + } + } + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.bit_cost.st64 + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__st64( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_s, + uint64_t a_i, + uint64_t a_v) { + wuffs_base__slice_u8 v_q = {0}; + + if (a_i < ((uint64_t)(a_s.len))) { + v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i); + if (((uint64_t)(v_q.len)) >= 8u) { + wuffs_base__poke_u64le__no_bounds_check(v_q.ptr, a_v); + } + } + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.bit_cost.fast_log2 + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__fast_log2( + const wuffs_brotli__bit_cost* self, + uint64_t a_v) { + if (a_v < 256u) { + return WUFFS_BROTLI__LOG2_TABLE[a_v]; + } + return wuffs_softfloat__arith__log2_u64(&self->private_data.f_sf, a_v); +} + +// -------- func brotli.bit_cost.bits_entropy + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__bits_entropy( + const wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_population, + uint64_t a_size) { + uint64_t v_sum = 0; + uint64_t v_retval = 0; + uint64_t v_p = 0; + wuffs_base__slice_u8 v_q = {0}; + uint64_t v_n = 0; + + v_q = a_population; + v_n = a_size; + while (v_n > 0u) { + v_n -= 1u; + v_p = 0u; + if (((uint64_t)(v_q.len)) >= 4u) { + v_p = ((uint64_t)(wuffs_base__peek_u32le__no_bounds_check(v_q.ptr))); + v_q = wuffs_base__slice_u8__subslice_i(v_q, 4u); + } + v_sum += v_p; + v_retval = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_p), wuffs_brotli__bit_cost__fast_log2(self, v_p))); + } + if (v_sum != 0u) { + v_retval = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum), wuffs_brotli__bit_cost__fast_log2(self, v_sum))); + } + if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum)) != 0u) { + v_retval = wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum); + } + return v_retval; +} + +// -------- func brotli.bit_cost.population_cost + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__population_cost( + const wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_histogram, + uint32_t a_data_size) { + uint64_t v_total = 0; + uint32_t v_count = 0; + uint32_t v_s[4] = {0}; + uint64_t v_bits = 0; + uint32_t v_i = 0; + uint32_t v_j = 0; + uint32_t v_k = 0; + uint32_t v_c = 0; + uint32_t v_histo[4] = {0}; + uint32_t v_h0 = 0; + uint32_t v_h1 = 0; + uint32_t v_h2 = 0; + uint32_t v_h23 = 0; + uint32_t v_histomax = 0; + uint64_t v_max_depth = 0; + uint32_t v_depth_histo[18] = {0}; + uint64_t v_log2total = 0; + uint64_t v_log2p = 0; + uint64_t v_depth = 0; + uint32_t v_reps = 0; + uint64_t v_sum = 0; + uint64_t v_retval = 0; + uint64_t v_p = 0; + + v_total = wuffs_brotli__bit_cost__ld64(self, a_histogram, (((uint64_t)(a_data_size)) * 4u)); + if (v_total == 0u) { + return 4622945017495814144u; + } + v_i = 0u; + while (v_i < a_data_size) { + if (wuffs_brotli__bit_cost__ld32(self, a_histogram, (((uint64_t)(v_i)) * 4u)) > 0u) { + if (v_count < 4u) { + v_s[v_count] = v_i; + } + v_count += 1u; + if (v_count > 4u) { + break; + } + } + v_i += 1u; + } + if (v_count == 1u) { + return 4622945017495814144u; + } + if (v_count == 2u) { + return wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, 4626322717216342016u, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_total)); + } + if (v_count == 3u) { + v_h0 = wuffs_brotli__bit_cost__ld32(self, a_histogram, (((uint64_t)(v_s[0u])) * 4u)); + v_h1 = wuffs_brotli__bit_cost__ld32(self, a_histogram, (((uint64_t)(v_s[1u])) * 4u)); + v_h2 = wuffs_brotli__bit_cost__ld32(self, a_histogram, (((uint64_t)(v_s[2u])) * 4u)); + v_histomax = wuffs_base__u32__max(v_h0, wuffs_base__u32__max(v_h1, v_h2)); + return wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, 4628574517030027264u, wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, ((uint32_t)(2u * ((uint32_t)(((uint32_t)(v_h0 + v_h1)) + v_h2)))))), wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, v_histomax)); + } + if (v_count == 4u) { + v_i = 0u; + while (v_i < 4u) { + v_histo[v_i] = wuffs_brotli__bit_cost__ld32(self, a_histogram, (((uint64_t)(v_s[v_i])) * 4u)); + v_i += 1u; + } + v_i = 0u; + while (v_i < 4u) { + v_j = ((uint32_t)(v_i + 1u)); + while (v_j < 4u) { + if (v_histo[v_j] > v_histo[(v_i & 3u)]) { + v_k = v_histo[v_j]; + v_histo[v_j] = v_histo[(v_i & 3u)]; + v_histo[(v_i & 3u)] = v_k; + } + v_j += 1u; + } + v_i += 1u; + } + v_h23 = ((uint32_t)(v_histo[2u] + v_histo[3u])); + v_histomax = wuffs_base__u32__max(v_h23, v_histo[0u]); + return wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, 4630404104378646528u, wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, ((uint32_t)(3u * v_h23)))), wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, ((uint32_t)(2u * ((uint32_t)(v_histo[0u] + v_histo[1u])))))), wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, v_histomax)); + } + v_max_depth = 1u; + v_log2total = wuffs_brotli__bit_cost__fast_log2(self, v_total); + v_i = 0u; + while (v_i < a_data_size) { + v_c = wuffs_brotli__bit_cost__ld32(self, a_histogram, (((uint64_t)(v_i)) * 4u)); + if (v_c > 0u) { + v_log2p = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_log2total, wuffs_brotli__bit_cost__fast_log2(self, ((uint64_t)(v_c)))); + v_depth = wuffs_softfloat__arith__f64_to_u64_trunc(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_log2p, 4602678819172646912u)); + v_bits = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_bits, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, v_c), v_log2p)); + if (v_depth > 15u) { + v_depth = 15u; + } + if (v_depth > ((uint64_t)(v_max_depth))) { + v_max_depth = (v_depth & 15u); + } + v_depth_histo[(v_depth & 15u)] += 1u; + v_i += 1u; + } else { + v_reps = 1u; + v_k = ((uint32_t)(v_i + 1u)); + while (v_k < a_data_size) { + if (wuffs_brotli__bit_cost__ld32(self, a_histogram, (((uint64_t)(v_k)) * 4u)) != 0u) { + break; + } + v_reps += 1u; + v_k += 1u; + } + wuffs_private_impl__u32__sat_add_indirect(&v_i, v_reps); + if (v_i >= a_data_size) { + break; + } + if (v_reps < 3u) { + v_depth_histo[0u] += v_reps; + } else { + v_reps -= 2u; + while (v_reps > 0u) { + v_depth_histo[17u] += 1u; + v_bits = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_bits, 4613937818241073152u); + v_reps >>= 3u; + } + } + } + } + v_bits = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_bits, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, (18u + (2u * v_max_depth)))); + v_i = 0u; + while (v_i < 18u) { + v_p = ((uint64_t)(v_depth_histo[v_i])); + v_sum += v_p; + v_retval = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_p), wuffs_brotli__bit_cost__fast_log2(self, v_p))); + v_i += 1u; + } + if (v_sum != 0u) { + v_retval = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum), wuffs_brotli__bit_cost__fast_log2(self, v_sum))); + } + if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum)) != 0u) { + v_retval = wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum); + } + return wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_bits, v_retval); +} + // -------- func brotli.encoder.write_bits WUFFS_BASE__GENERATED_C_CODE @@ -68017,6 +68705,348 @@ wuffs_brotli__encoder__compress_fragment_two_pass( return wuffs_base__make_empty_struct(); } +// -------- func brotli.bit_cost.histogram_clear + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__histogram_clear( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_h, + uint32_t a_data_size) { + uint64_t v_n = 0; + + v_n = (((uint64_t)(a_data_size)) * 4u); + if (v_n <= ((uint64_t)(a_h.len))) { + wuffs_private_impl__bulk_memset(a_h.ptr, v_n, 0u); + } + wuffs_brotli__bit_cost__st64(self, a_h, v_n, 0u); + wuffs_brotli__bit_cost__st64(self, a_h, (v_n + 8u), 9218868437227405312u); + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.bit_cost.histogram_add + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__histogram_add( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_h, + uint32_t a_data_size, + uint64_t a_val) { + uint64_t v_n = 0; + + v_n = (((uint64_t)(a_data_size)) * 4u); + if (a_val < ((uint64_t)(a_data_size))) { + wuffs_brotli__bit_cost__st32(self, a_h, ((uint64_t)(a_val * 4u)), ((uint32_t)(wuffs_brotli__bit_cost__ld32(self, a_h, ((uint64_t)(a_val * 4u))) + 1u))); + } + wuffs_brotli__bit_cost__st64(self, a_h, v_n, ((uint64_t)(wuffs_brotli__bit_cost__ld64(self, a_h, v_n) + 1u))); + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.bit_cost.histogram_add_histogram + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__histogram_add_histogram( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_h, + wuffs_base__slice_u8 a_v, + uint32_t a_data_size) { + uint64_t v_n = 0; + wuffs_base__slice_u8 v_p = {0}; + wuffs_base__slice_u8 v_q = {0}; + + v_n = (((uint64_t)(a_data_size)) * 4u); + wuffs_brotli__bit_cost__st64(self, a_h, v_n, ((uint64_t)(wuffs_brotli__bit_cost__ld64(self, a_h, v_n) + wuffs_brotli__bit_cost__ld64(self, a_v, v_n)))); + if ((v_n <= ((uint64_t)(a_h.len))) && (v_n <= ((uint64_t)(a_v.len)))) { + { + wuffs_base__slice_u8 i_slice_p = wuffs_base__slice_u8__subslice_j(a_h, v_n); + v_p.ptr = i_slice_p.ptr; + wuffs_base__slice_u8 i_slice_q = wuffs_base__slice_u8__subslice_j(a_v, v_n); + v_q.ptr = i_slice_q.ptr; + i_slice_p.len = ((size_t)(wuffs_base__u64__min(i_slice_p.len, i_slice_q.len))); + v_p.len = 4; + v_q.len = 4; + const uint8_t* i_end0_p = wuffs_private_impl__ptr_u8_plus_len(v_p.ptr, (((i_slice_p.len - (size_t)(v_p.ptr - i_slice_p.ptr)) / 4) * 4)); + while (v_p.ptr < i_end0_p) { + wuffs_base__poke_u32le__no_bounds_check(v_p.ptr, ((uint32_t)(wuffs_base__peek_u32le__no_bounds_check(v_p.ptr) + wuffs_base__peek_u32le__no_bounds_check(v_q.ptr)))); + v_p.ptr += 4; + v_q.ptr += 4; + } + v_p.len = 0; + v_q.len = 0; + } + } + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.bit_cost.histogram_copy + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__histogram_copy( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_h, + wuffs_base__slice_u8 a_v, + uint32_t a_data_size) { + uint64_t v_n = 0; + + v_n = ((((uint64_t)(a_data_size)) * 4u) + 16u); + if ((v_n <= ((uint64_t)(a_h.len))) && (v_n <= ((uint64_t)(a_v.len)))) { + wuffs_private_impl__slice_u8__copy_from_slice(wuffs_base__slice_u8__subslice_j(a_h, v_n), wuffs_base__slice_u8__subslice_j(a_v, v_n)); + } + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.bit_cost.histogram_total_count + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__histogram_total_count( + const wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_h, + uint32_t a_data_size) { + return wuffs_brotli__bit_cost__ld64(self, a_h, (((uint64_t)(a_data_size)) * 4u)); +} + +// -------- func brotli.bit_cost.histogram_bit_cost + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__histogram_bit_cost( + const wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_h, + uint32_t a_data_size) { + return wuffs_brotli__bit_cost__ld64(self, a_h, ((((uint64_t)(a_data_size)) * 4u) + 8u)); +} + +// -------- func brotli.bit_cost.histogram_set_bit_cost + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__histogram_set_bit_cost( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_h, + uint32_t a_data_size, + uint64_t a_bit_cost) { + wuffs_brotli__bit_cost__st64(self, a_h, ((((uint64_t)(a_data_size)) * 4u) + 8u), a_bit_cost); + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.bit_cost.build_histograms_with_context + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__bit_cost__build_histograms_with_context( + wuffs_brotli__bit_cost* self, + wuffs_base__slice_u8 a_cmds, + uint64_t a_num_commands, + wuffs_base__slice_u8 a_lit_types, + wuffs_base__slice_u8 a_lit_lengths, + uint64_t a_lit_num_blocks, + wuffs_base__slice_u8 a_cmd_types, + wuffs_base__slice_u8 a_cmd_lengths, + uint64_t a_cmd_num_blocks, + wuffs_base__slice_u8 a_dist_types, + wuffs_base__slice_u8 a_dist_lengths, + uint64_t a_dist_num_blocks, + wuffs_base__slice_u8 a_ringbuffer, + uint64_t a_start_pos, + uint64_t a_mask, + uint8_t a_prev_byte, + uint8_t a_prev_byte2, + uint32_t a_context_mode, + wuffs_base__slice_u8 a_lit_histos, + wuffs_base__slice_u8 a_cmd_histos, + wuffs_base__slice_u8 a_dist_histos) { + uint64_t v_pos = 0; + uint8_t v_prev_byte = 0; + uint8_t v_prev_byte2 = 0; + uint64_t v_lit_idx = 0; + uint64_t v_lit_type = 0; + uint64_t v_lit_length = 0; + uint64_t v_cmd_idx = 0; + uint64_t v_cmd_type = 0; + uint64_t v_cmd_length = 0; + uint64_t v_dist_idx = 0; + uint64_t v_dist_type = 0; + uint64_t v_dist_length = 0; + uint64_t v_i = 0; + uint64_t v_j = 0; + uint64_t v_c = 0; + uint32_t v_insert_len = 0; + uint32_t v_copy_len = 0; + uint32_t v_cmd_prefix = 0; + uint32_t v_dist_prefix = 0; + uint64_t v_context = 0; + uint32_t v_lut_offset = 0; + uint8_t v_b = 0; + uint64_t v_off = 0; + uint32_t v_r = 0; + + v_pos = a_start_pos; + v_prev_byte = a_prev_byte; + v_prev_byte2 = a_prev_byte2; + if (a_lit_num_blocks > 0u) { + v_lit_length = ((uint64_t)(wuffs_brotli__bit_cost__ld32(self, a_lit_lengths, 0u))); + } + if (a_cmd_num_blocks > 0u) { + v_cmd_length = ((uint64_t)(wuffs_brotli__bit_cost__ld32(self, a_cmd_lengths, 0u))); + } + if (a_dist_num_blocks > 0u) { + v_dist_length = ((uint64_t)(wuffs_brotli__bit_cost__ld32(self, a_dist_lengths, 0u))); + } + v_lut_offset = ((a_context_mode & 3u) << 9u); + v_i = 0u; + while (v_i < a_num_commands) { + v_c = ((uint64_t)(v_i * 16u)); + v_insert_len = wuffs_brotli__bit_cost__ld32(self, a_cmds, v_c); + v_copy_len = (wuffs_brotli__bit_cost__ld32(self, a_cmds, ((uint64_t)(v_c + 4u))) & 33554431u); + v_r = wuffs_brotli__bit_cost__ld32(self, a_cmds, ((uint64_t)(v_c + 12u))); + v_cmd_prefix = (v_r & 65535u); + v_dist_prefix = (v_r >> 16u); + if (v_cmd_length == 0u) { + v_cmd_idx += 1u; + if (v_cmd_idx < ((uint64_t)(a_cmd_types.len))) { + v_cmd_type = ((uint64_t)(a_cmd_types.ptr[v_cmd_idx])); + } else { + v_cmd_type = 0u; + } + v_cmd_length = ((uint64_t)(wuffs_brotli__bit_cost__ld32(self, a_cmd_lengths, ((uint64_t)(v_cmd_idx * 4u))))); + } + v_cmd_length -= 1u; + v_off = ((uint64_t)(v_cmd_type * 2832u)); + if (v_off <= ((uint64_t)(a_cmd_histos.len))) { + wuffs_brotli__bit_cost__histogram_add(self, wuffs_base__slice_u8__subslice_i(a_cmd_histos, v_off), 704u, ((uint64_t)(v_cmd_prefix))); + } + v_j = ((uint64_t)(v_insert_len)); + while (v_j != 0u) { + v_j -= 1u; + if (v_lit_length == 0u) { + v_lit_idx += 1u; + if (v_lit_idx < ((uint64_t)(a_lit_types.len))) { + v_lit_type = ((uint64_t)(a_lit_types.ptr[v_lit_idx])); + } else { + v_lit_type = 0u; + } + v_lit_length = ((uint64_t)(wuffs_brotli__bit_cost__ld32(self, a_lit_lengths, ((uint64_t)(v_lit_idx * 4u))))); + } + v_lit_length -= 1u; + v_context = v_lit_type; + if (a_context_mode != 4294967295u) { + v_context = ((uint64_t)(((uint64_t)(v_context << 6u)) + ((uint64_t)(((uint8_t)(WUFFS_BROTLI__CONTEXT_LOOKUP[((v_lut_offset + ((uint32_t)(v_prev_byte))) & 2047u)] | WUFFS_BROTLI__CONTEXT_LOOKUP[((v_lut_offset + 256u + ((uint32_t)(v_prev_byte2))) & 2047u)])))))); + } + v_b = 0u; + if ((v_pos & a_mask) < ((uint64_t)(a_ringbuffer.len))) { + v_b = a_ringbuffer.ptr[(v_pos & a_mask)]; + } + v_off = ((uint64_t)(v_context * 1040u)); + if (v_off <= ((uint64_t)(a_lit_histos.len))) { + wuffs_brotli__bit_cost__histogram_add(self, wuffs_base__slice_u8__subslice_i(a_lit_histos, v_off), 256u, ((uint64_t)(v_b))); + } + v_prev_byte2 = v_prev_byte; + v_prev_byte = v_b; + v_pos += 1u; + } + v_pos += ((uint64_t)(v_copy_len)); + if (v_copy_len != 0u) { + v_prev_byte2 = 0u; + if ((((uint64_t)(v_pos - 2u)) & a_mask) < ((uint64_t)(a_ringbuffer.len))) { + v_prev_byte2 = a_ringbuffer.ptr[(((uint64_t)(v_pos - 2u)) & a_mask)]; + } + v_prev_byte = 0u; + if ((((uint64_t)(v_pos - 1u)) & a_mask) < ((uint64_t)(a_ringbuffer.len))) { + v_prev_byte = a_ringbuffer.ptr[(((uint64_t)(v_pos - 1u)) & a_mask)]; + } + if (v_cmd_prefix >= 128u) { + if (v_dist_length == 0u) { + v_dist_idx += 1u; + if (v_dist_idx < ((uint64_t)(a_dist_types.len))) { + v_dist_type = ((uint64_t)(a_dist_types.ptr[v_dist_idx])); + } else { + v_dist_type = 0u; + } + v_dist_length = ((uint64_t)(wuffs_brotli__bit_cost__ld32(self, a_dist_lengths, ((uint64_t)(v_dist_idx * 4u))))); + } + v_dist_length -= 1u; + v_context = ((uint64_t)(((uint64_t)(v_dist_type << 2u)) + ((uint64_t)(wuffs_brotli__bit_cost__command_distance_context(self, v_cmd_prefix))))); + v_off = ((uint64_t)(v_context * 2192u)); + if (v_off <= ((uint64_t)(a_dist_histos.len))) { + wuffs_brotli__bit_cost__histogram_add(self, wuffs_base__slice_u8__subslice_i(a_dist_histos, v_off), 544u, ((uint64_t)((v_dist_prefix & 1023u)))); + } + } + } + v_i += 1u; + } + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.bit_cost.command_distance_context + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__command_distance_context( + const wuffs_brotli__bit_cost* self, + uint32_t a_cmd_prefix) { + uint32_t v_r = 0; + uint32_t v_c = 0; + + v_r = (a_cmd_prefix >> 6u); + v_c = (a_cmd_prefix & 7u); + if (((v_r == 0u) || + (v_r == 2u) || + (v_r == 4u) || + (v_r == 7u)) && (v_c <= 2u)) { + return v_c; + } + return 3u; +} + +// -------- func brotli.metablock_tester.bits_entropy + +WUFFS_BASE__GENERATED_C_CODE +WUFFS_BASE__MAYBE_STATIC uint64_t +wuffs_brotli__metablock_tester__bits_entropy( + const wuffs_brotli__metablock_tester* self, + wuffs_base__slice_u8 a_population, + uint64_t a_size) { + if (!self) { + return 0; + } + if ((self->private_impl.magic != WUFFS_BASE__MAGIC) && + (self->private_impl.magic != WUFFS_BASE__DISABLED)) { + return 0; + } + + return wuffs_brotli__bit_cost__bits_entropy(&self->private_data.f_bc, a_population, a_size); +} + +// -------- func brotli.metablock_tester.population_cost + +WUFFS_BASE__GENERATED_C_CODE +WUFFS_BASE__MAYBE_STATIC uint64_t +wuffs_brotli__metablock_tester__population_cost( + const wuffs_brotli__metablock_tester* self, + wuffs_base__slice_u8 a_histogram, + uint32_t a_data_size) { + if (!self) { + return 0; + } + if ((self->private_impl.magic != WUFFS_BASE__MAGIC) && + (self->private_impl.magic != WUFFS_BASE__DISABLED)) { + return 0; + } + + if (a_data_size == 256u) { + return wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, a_histogram, 256u); + } else if (a_data_size == 704u) { + return wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, a_histogram, 704u); + } else if (a_data_size == 544u) { + return wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, a_histogram, 544u); + } + return 0u; +} + #endif // !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__BROTLI) #if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__BZIP2) diff --git a/script/manual-test-brotli-metablock-parity.c b/script/manual-test-brotli-metablock-parity.c new file mode 100644 index 00000000..be226a60 --- /dev/null +++ b/script/manual-test-brotli-metablock-parity.c @@ -0,0 +1,274 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// ---------------- + +// manual-test-brotli-metablock-parity.c checks that std/brotli's ports of +// libbrotlienc v1.2.1's bit cost, histogram, block splitter, clustering and +// metablock functions (exposed for testing by the brotli.metablock_tester +// struct) produce bit-for-bit identical results to the C functions +// (BrotliBitsEntropy, BrotliPopulationCost*, BrotliSplitBlock, +// BrotliClusterHistograms*, BrotliBuildMetaBlock, BrotliOptimizeHistograms, +// etc.), which are BROTLI_INTERNAL but linkable from the static libraries. +// +// Run it from the Wuffs root directory with (as one command line): +// +// gcc -O2 -std=c99 -I/path/to/brotli/c -I/path/to/brotli/c/include +// script/manual-test-brotli-metablock-parity.c +// /path/to/build/libbrotlienc.a /path/to/build/libbrotlicommon.a +// -lm -o /tmp/brotli-metablock-parity +// /tmp/brotli-metablock-parity [-quick] +// +// where /path/to/brotli is a checkout of the brotli repository at tag v1.2.1 +// (the internal headers under c/enc are needed) and /path/to/build is a CMake +// build of it with BUILD_SHARED_LIBS=OFF. The C code's doubles are compared +// as their bit patterns, so the reference must be built for x86-64 with the +// default flags (SSE2, no FMA contraction) and linked against glibc's log2. +// +// It prints each mismatch and a summary, and exits with a non-zero status if +// there were any mismatches. + +#define _POSIX_C_SOURCE 200809L + +#include +#include +#include +#include +#include +#include +#include + +#include "enc/bit_cost.h" +#include "enc/block_splitter.h" +#include "enc/cluster.h" +#include "enc/command.h" +#include "enc/entropy_encode.h" +#include "enc/histogram.h" +#include "enc/memory.h" +#include "enc/metablock.h" +#include "enc/params.h" + +#define WUFFS_IMPLEMENTATION +#define WUFFS_CONFIG__MODULES +#define WUFFS_CONFIG__MODULE__BASE +#define WUFFS_CONFIG__MODULE__BROTLI +#define WUFFS_CONFIG__MODULE__SOFTFLOAT +#include "../release/c/wuffs-unsupported-snapshot.c" + +// ---------------- Utilities + +static uint64_t g_rng_state = 0x123456789ABCDEF1ull; + +static uint64_t // +rng64(void) { + // xorshift64*. + g_rng_state ^= g_rng_state >> 12; + g_rng_state ^= g_rng_state << 25; + g_rng_state ^= g_rng_state >> 27; + return g_rng_state * 0x2545F4914F6CDD1Dull; +} + +static uint32_t // +rng_below(uint32_t n) { + return n ? (uint32_t)((rng64() >> 32) % n) : 0; +} + +static uint64_t // +f64_bits(double d) { + uint64_t u; + memcpy(&u, &d, 8); + return u; +} + +static uint64_t g_num_checks = 0; +static uint64_t g_num_mismatches = 0; +static uint64_t g_counts[64]; +static const char* g_count_names[64]; + +static void // +count(int i, const char* name) { + g_counts[i]++; + g_count_names[i] = name; +} + +#define CHECK_EQ(what, c_value, w_value) \ + do { \ + uint64_t c_v_ = (uint64_t)(c_value); \ + uint64_t w_v_ = (uint64_t)(w_value); \ + g_num_checks++; \ + if (c_v_ != w_v_) { \ + g_num_mismatches++; \ + if (g_num_mismatches < 50) { \ + printf("MISMATCH %s (%s): C 0x%016" PRIX64 " vs Wuffs 0x%016" PRIX64 \ + "\n", \ + what, g_config_name, c_v_, w_v_); \ + } \ + } \ + } while (0) + +static char g_config_name[256]; + +static wuffs_brotli__metablock_tester* g_tester = NULL; + +// ---------------- BitsEntropy and PopulationCost + +// fill_counts fills n u32 counts with a random distribution of the given +// style. +static void // +fill_counts(uint32_t* counts, size_t n, int style) { + size_t i; + memset(counts, 0, n * sizeof(uint32_t)); + switch (style) { + case 0: // Uniform small. + for (i = 0; i < n; i++) { + counts[i] = rng_below(10); + } + break; + case 1: // Sparse. + for (i = 0; i < n; i++) { + counts[i] = (rng_below(8) == 0) ? rng_below(1000) : 0; + } + break; + case 2: { // 1 to 5 non-zero symbols. + size_t k = 1 + rng_below(5); + for (i = 0; i < k; i++) { + counts[rng_below((uint32_t)n)] += 1 + rng_below(100000); + } + break; + } + case 3: // Large counts. + for (i = 0; i < n; i++) { + counts[i] = (rng_below(3) == 0) ? (uint32_t)(rng64() >> 40) : 0; + } + break; + case 4: // Zipf-ish. + for (i = 0; i < n; i++) { + counts[i] = (uint32_t)(100000 / (1 + i + rng_below(3))); + } + break; + case 5: // Runs of zeros of various lengths. + for (i = 0; i < n;) { + size_t run = 1 + rng_below(40); + uint32_t v = rng_below(2) ? 0 : 1 + rng_below(50); + for (; run && (i < n); run--, i++) { + counts[i] = v; + } + } + break; + case 6: // All zeros. + break; + default: { // Huge counts (u32 overflow in the 2 * (...) expressions). + size_t k = 2 + rng_below(4); + for (i = 0; i < k; i++) { + counts[rng_below((uint32_t)n)] = 0x7FFFFFFFu + rng_below(0x7FFFFFFF); + } + break; + } + } +} + +static void // +test_bit_cost(int iters) { + static uint32_t counts[1024]; + static uint8_t hbuf[2832]; + int it; + for (it = 0; it < iters; it++) { + int style = (int)rng_below(8); + size_t size = 1 + rng_below(704); + snprintf(g_config_name, sizeof(g_config_name), + "bit_cost iter=%d style=%d size=%zu", it, style, size); + fill_counts(counts, size, style); + { + double c = BrotliBitsEntropy(counts, size); + uint64_t w = wuffs_brotli__metablock_tester__bits_entropy( + g_tester, wuffs_base__make_slice_u8((uint8_t*)counts, size * 4), + size); + CHECK_EQ("BitsEntropy", f64_bits(c), w); + count(0, "BitsEntropy"); + } + if (style == 7) { + // Huge counts make the C code's total_count_ inconsistent (and its + // (size_t)(log2p + 0.5) undefined), so skip PopulationCost. + continue; + } + { + HistogramLiteral h; + size_t i; + fill_counts(h.data_, 256, style); + h.total_count_ = 0; + for (i = 0; i < 256; i++) { + h.total_count_ += h.data_[i]; + } + h.bit_cost_ = 0; + memcpy(hbuf, &h, sizeof(h)); + CHECK_EQ("PopulationCostLiteral", f64_bits(BrotliPopulationCostLiteral(&h)), + wuffs_brotli__metablock_tester__population_cost( + g_tester, wuffs_base__make_slice_u8(hbuf, sizeof(h)), 256)); + count(1, "PopulationCostLiteral"); + } + { + HistogramCommand h; + size_t i; + fill_counts(h.data_, 704, style); + h.total_count_ = 0; + for (i = 0; i < 704; i++) { + h.total_count_ += h.data_[i]; + } + h.bit_cost_ = 0; + memcpy(hbuf, &h, sizeof(h)); + CHECK_EQ("PopulationCostCommand", f64_bits(BrotliPopulationCostCommand(&h)), + wuffs_brotli__metablock_tester__population_cost( + g_tester, wuffs_base__make_slice_u8(hbuf, sizeof(h)), 704)); + count(2, "PopulationCostCommand"); + } + { + HistogramDistance h; + size_t i; + fill_counts(h.data_, 544, style); + h.total_count_ = 0; + for (i = 0; i < 544; i++) { + h.total_count_ += h.data_[i]; + } + h.bit_cost_ = 0; + memcpy(hbuf, &h, sizeof(h)); + CHECK_EQ("PopulationCostDistance", + f64_bits(BrotliPopulationCostDistance(&h)), + wuffs_brotli__metablock_tester__population_cost( + g_tester, wuffs_base__make_slice_u8(hbuf, sizeof(h)), 544)); + count(3, "PopulationCostDistance"); + } + } +} + +// ---------------- Main + +int // +main(int argc, char** argv) { + bool quick = (argc > 1) && !strcmp(argv[1], "-quick"); + int i; + g_tester = wuffs_brotli__metablock_tester__alloc(); + if (!g_tester) { + printf("could not allocate the tester\n"); + return 1; + } + + test_bit_cost(quick ? 20000 : 200000); + + for (i = 0; i < 64; i++) { + if (g_count_names[i]) { + printf("%-32s %10" PRIu64 " configurations\n", g_count_names[i], + g_counts[i]); + } + } + printf("%" PRIu64 " checks, %" PRIu64 " mismatches\n", g_num_checks, + g_num_mismatches); + free(g_tester); + return g_num_mismatches ? 1 : 0; +} diff --git a/std/brotli/README.md b/std/brotli/README.md index 25cd99d2..db513a40 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -189,6 +189,15 @@ The source files are: - `encode_fragment_fast.wuffs`: `compress_fragment.c` (quality 0). - `encode_fragment_two_pass.wuffs`: `compress_fragment_two_pass.c` (quality 1). +- `encode_bit_cost.wuffs`: `bit_cost.c`, `bit_cost_inc.h` and `fast_log.h` + (`BrotliBitsEntropy`, `BrotliPopulationCost{Literal,Command,Distance}`), as + methods of the `bit_cost` struct. +- `encode_histogram.wuffs`: `histogram.h`, `histogram_inc.h` and + `histogram.c` (histogram helpers and `BrotliBuildHistogramsWithContext`), + also as `bit_cost` methods. +- `encode_tester.wuffs`: TEST ONLY. The `metablock_tester` struct exposes + the internal functions above to C test programs. It is not part of the + encoder API. - `common_encoder_tables.wuffs`: tables generated by `go run script/print-brotli-tables.go -encoder` from the C code. diff --git a/std/brotli/encode_bit_cost.wuffs b/std/brotli/encode_bit_cost.wuffs new file mode 100644 index 00000000..0fe71b84 --- /dev/null +++ b/std/brotli/encode_bit_cost.wuffs @@ -0,0 +1,320 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// -------- + +// This file mirrors bit_cost.c, bit_cost_inc.h and fast_log.h. The functions +// are methods of the bit_cost struct (whose only state is the stateless +// softfloat arith), so that other structs (the encoder, the +// metablock_builder) can embed one with a single field. +// +// Doubles are IEEE 754 binary64 bit patterns (base.u64). A uint32_t array +// (e.g. a histogram's data_) is a slice of little-endian bytes, because +// Wuffs' C code generator does not support "slice base.u32" arguments. +// +// A histogram (the C code's HistogramLiteral, HistogramCommand and +// HistogramDistance structs) is (4 * data_size) + 16 bytes, exactly the C +// struct's memory layout on a 64-bit little-endian machine: +// +// [0 .. 4 * data_size) data_, as u32le values +// [4 * data_size .. (4 * data_size) + 8) total_count_, as a u64le +// [(4 * data_size) + 8 .. (4 * data_size) + 16) bit_cost_, as the u64le +// bits of a binary64 +// +// data_size is 256 (HISTOGRAM_LITERAL_DATA_SIZE), 704 +// (HISTOGRAM_COMMAND_DATA_SIZE) or 544 (HISTOGRAM_DISTANCE_DATA_SIZE). The C +// code's templated functions (the FN(X) macros, instantiated three times) are +// one Wuffs function each, with a data_size argument. + +pri const HISTOGRAM_LITERAL_DATA_SIZE : base.u32 = 256 +pri const HISTOGRAM_COMMAND_DATA_SIZE : base.u32 = 704 +pri const HISTOGRAM_DISTANCE_DATA_SIZE : base.u32 = 544 + +// The byte sizes of the three histogram types. +pri const HISTOGRAM_LITERAL_SIZE : base.u64 = 1040 +pri const HISTOGRAM_COMMAND_SIZE : base.u64 = 2832 +pri const HISTOGRAM_DISTANCE_SIZE : base.u64 = 2192 + +// F64_HUGE_VAL is +infinity (the C code's HUGE_VAL). +pri const F64_HUGE_VAL : base.u64 = 0x7FF0_0000_0000_0000 + +// F64_1E99 is the C code's 1e99. +pri const F64_1E99 : base.u64 = 0x547D_42AE_A287_9F2E + +pri struct bit_cost?( + sf : softfloat.arith, +) + +// ld32 returns the u32le at s[i ..], or 0 if that is out of bounds (which +// the callers never do). +pri func bit_cost.ld32(s: roslice base.u8, i: base.u64) base.u32 { + var q : roslice base.u8 + + if args.i < args.s.length() { + q = args.s[args.i ..] + if q.length() >= 4 { + return q.peek_u32le() + } + } + return 0 +} + +// ld64 is like ld32 but for a u64le. +pri func bit_cost.ld64(s: roslice base.u8, i: base.u64) base.u64 { + var q : roslice base.u8 + + if args.i < args.s.length() { + q = args.s[args.i ..] + if q.length() >= 8 { + return q.peek_u64le() + } + } + return 0 +} + +// st32 stores v as a u32le at s[i ..], or does nothing if that is out of +// bounds (which the callers never do). +pri func bit_cost.st32!(s: slice base.u8, i: base.u64, v: base.u32) { + var q : slice base.u8 + + if args.i < args.s.length() { + q = args.s[args.i ..] + if q.length() >= 4 { + q.poke_u32le!(a: args.v) + } + } +} + +// st64 is like st32 but for a u64le. +pri func bit_cost.st64!(s: slice base.u8, i: base.u64, v: base.u64) { + var q : slice base.u8 + + if args.i < args.s.length() { + q = args.s[args.i ..] + if q.length() >= 8 { + q.poke_u64le!(a: args.v) + } + } +} + +// Mirrors FastLog2 in fast_log.h. The result is the bit pattern of an IEEE +// 754 binary64 value. +pri func bit_cost.fast_log2(v: base.u64) base.u64 { + if args.v < 256 { + return LOG2_TABLE[args.v] + } + return this.sf.log2_u64(v: args.v) +} + +// Mirrors BrotliBitsEntropy in bit_cost.c. population holds (at least) size +// u32le values. The C code's "goto odd_number_of_elements_left" only changes +// the loop's unrolling, not the order of the operations. +pri func bit_cost.bits_entropy(population: roslice base.u8, size: base.u64) base.u64 { + var sum : base.u64 + var retval : base.u64 + var p : base.u64 + var q : roslice base.u8 + var n : base.u64 + + q = args.population + n = args.size + while n > 0 { + n -= 1 + p = 0 + if q.length() >= 4 { + p = q.peek_u32le() as base.u64 + q = q[4 ..] + } + sum ~mod+= p + retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul( + a: this.sf.f64_from_u64(a: p), b: this.fast_log2(v: p))) + } + if sum <> 0 { + retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul( + a: this.sf.f64_from_u64(a: sum), b: this.fast_log2(v: sum))) + } + if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 { + retval = this.sf.f64_from_u64(a: sum) + } + return retval +} + +// Mirrors BrotliPopulationCostLiteral, BrotliPopulationCostCommand and +// BrotliPopulationCostDistance (FN(BrotliPopulationCost) in bit_cost_inc.h). +// histogram is a histogram of data_size symbols (see the layout above). +pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u32[..= 704]) base.u64 { + var total : base.u64 + var count : base.u32 + var s : array[4] base.u32 + var bits : base.u64 + var i : base.u32 + var j : base.u32 + var k : base.u32 + var c : base.u32 + var histo : array[4] base.u32 + var h0 : base.u32 + var h1 : base.u32 + var h2 : base.u32 + var h23 : base.u32 + var histomax : base.u32 + var max_depth : base.u64[..= 15] + var depth_histo : array[18] base.u32 + var log2total : base.u64 + var log2p : base.u64 + var depth : base.u64 + var reps : base.u32 + var sum : base.u64 + var retval : base.u64 + var p : base.u64 + + total = this.ld64(s: args.histogram, i: (args.data_size as base.u64) * 4) + if total == 0 { + // kOneSymbolHistogramCost. + return 0x4028_0000_0000_0000 + } + i = 0 + while i < args.data_size { + if this.ld32(s: args.histogram, i: (i as base.u64) * 4) > 0 { + if count < 4 { + s[count] = i + } + count ~mod+= 1 + if count > 4 { + break + } + } + i ~mod+= 1 + } + if count == 1 { + // kOneSymbolHistogramCost. + return 0x4028_0000_0000_0000 + } + if count == 2 { + // kTwoSymbolHistogramCost + (double)total_count_. + return this.sf.f64_add(a: 0x4034_0000_0000_0000, b: this.sf.f64_from_u64(a: total)) + } + if count == 3 { + h0 = this.ld32(s: args.histogram, i: (s[0] as base.u64) * 4) + h1 = this.ld32(s: args.histogram, i: (s[1] as base.u64) * 4) + h2 = this.ld32(s: args.histogram, i: (s[2] as base.u64) * 4) + histomax = h0.max(no_less_than: h1.max(no_less_than: h2)) + // kThreeSymbolHistogramCost + 2 * (histo0 + histo1 + histo2) - + // histomax, where the integer expression is uint32_t arithmetic. + return this.sf.f64_sub( + a: this.sf.f64_add( + a: 0x403C_0000_0000_0000, + b: this.sf.f64_from_u32(a: 2 ~mod* ((h0 ~mod+ h1) ~mod+ h2))), + b: this.sf.f64_from_u32(a: histomax)) + } + if count == 4 { + i = 0 + while i < 4 { + histo[i] = this.ld32(s: args.histogram, i: (s[i] as base.u64) * 4) + i ~mod+= 1 + } + // Sort. + i = 0 + while i < 4 { + j = i ~mod+ 1 + while j < 4 { + if histo[j] > histo[i & 3] { + k = histo[j] + histo[j] = histo[i & 3] + histo[i & 3] = k + } + j ~mod+= 1 + } + i ~mod+= 1 + } + h23 = histo[2] ~mod+ histo[3] + histomax = h23.max(no_less_than: histo[0]) + // kFourSymbolHistogramCost + 3 * h23 + 2 * (histo[0] + histo[1]) - + // histomax, evaluated left to right. + return this.sf.f64_sub( + a: this.sf.f64_add( + a: this.sf.f64_add( + a: 0x4042_8000_0000_0000, + b: this.sf.f64_from_u32(a: 3 ~mod* h23)), + b: this.sf.f64_from_u32(a: 2 ~mod* (histo[0] ~mod+ histo[1]))), + b: this.sf.f64_from_u32(a: histomax)) + } + + // Compute the entropy of the histogram and simultaneously build a + // simplified histogram of the code length codes. + max_depth = 1 + log2total = this.fast_log2(v: total) + i = 0 + while i < args.data_size { + c = this.ld32(s: args.histogram, i: (i as base.u64) * 4) + if c > 0 { + log2p = this.sf.f64_sub(a: log2total, b: this.fast_log2(v: c as base.u64)) + depth = this.sf.f64_to_u64_trunc(a: this.sf.f64_add(a: log2p, b: 0x3FE0_0000_0000_0000)) + bits = this.sf.f64_add(a: bits, b: this.sf.f64_mul( + a: this.sf.f64_from_u32(a: c), b: log2p)) + if depth > 15 { + depth = 15 + } + if depth > (max_depth as base.u64) { + max_depth = depth & 15 + } + depth_histo[depth & 15] ~mod+= 1 + i ~mod+= 1 + } else { + // Compute the run length of zeros. + reps = 1 + k = i ~mod+ 1 + while k < args.data_size { + if this.ld32(s: args.histogram, i: (k as base.u64) * 4) <> 0 { + break + } + reps ~mod+= 1 + k ~mod+= 1 + } + i ~sat+= reps + if i >= args.data_size { + // Don't add any cost for the last zero run. + break + } + if reps < 3 { + depth_histo[0] ~mod+= reps + } else { + reps -= 2 + while reps > 0 { + depth_histo[17] ~mod+= 1 + // Add the 3 extra bits for the 17 code length code. + bits = this.sf.f64_add(a: bits, b: 0x4008_0000_0000_0000) + reps >>= 3 + } + } + } + } + // Add the estimated encoding cost of the code length code histogram. + bits = this.sf.f64_add(a: bits, b: this.sf.f64_from_u64(a: 18 + (2 * max_depth))) + + // Add the entropy of the code length code histogram. This is + // BrotliBitsEntropy(depth_histo, BROTLI_CODE_LENGTH_CODES), inlined + // because depth_histo is a local array. + i = 0 + while i < 18 { + p = depth_histo[i] as base.u64 + sum ~mod+= p + retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul( + a: this.sf.f64_from_u64(a: p), b: this.fast_log2(v: p))) + i ~mod+= 1 + } + if sum <> 0 { + retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul( + a: this.sf.f64_from_u64(a: sum), b: this.fast_log2(v: sum))) + } + if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 { + retval = this.sf.f64_from_u64(a: sum) + } + return this.sf.f64_add(a: bits, b: retval) +} diff --git a/std/brotli/encode_histogram.wuffs b/std/brotli/encode_histogram.wuffs new file mode 100644 index 00000000..fbb8c12f --- /dev/null +++ b/std/brotli/encode_histogram.wuffs @@ -0,0 +1,268 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// -------- + +// This file mirrors histogram.h, histogram_inc.h and histogram.c. See +// encode_bit_cost.wuffs for the histograms' byte layout. The functions are +// methods of the bit_cost struct. +// +// A command (the C code's Command struct) is 16 bytes, exactly the C struct's +// memory layout on a little-endian machine: +// +// [ 0 .. 4) insert_len_, as a u32le +// [ 4 .. 8) copy_len_, as a u32le (the copy length in the low 25 bits, +// copy_code - copy_len in the high 7 bits) +// [ 8 .. 12) dist_extra_, as a u32le +// [12 .. 14) cmd_prefix_, as a u16le +// [14 .. 16) dist_prefix_, as a u16le +// +// A BlockSplit's types are a slice of u8 (one per block) and its lengths are +// a slice of u32le values (one per block). + +// Mirrors FN(HistogramClear) in histogram_inc.h. +pri func bit_cost.histogram_clear!(h: slice base.u8, data_size: base.u32[..= 704]) { + var n : base.u64 + + n = (args.data_size as base.u64) * 4 + if n <= args.h.length() { + args.h[.. n].bulk_memset!(byte_value: 0) + } + this.st64!(s: args.h, i: n, v: 0) + this.st64!(s: args.h, i: n + 8, v: F64_HUGE_VAL) +} + +// Mirrors FN(HistogramAdd) in histogram_inc.h. +pri func bit_cost.histogram_add!(h: slice base.u8, data_size: base.u32[..= 704], val: base.u64) { + var n : base.u64 + + n = (args.data_size as base.u64) * 4 + if args.val < (args.data_size as base.u64) { + this.st32!(s: args.h, i: args.val ~mod* 4, v: this.ld32(s: args.h, i: args.val ~mod* 4) ~mod+ 1) + } + this.st64!(s: args.h, i: n, v: this.ld64(s: args.h, i: n) ~mod+ 1) +} + +// Mirrors FN(HistogramAddHistogram) in histogram_inc.h: h += v. +pri func bit_cost.histogram_add_histogram!(h: slice base.u8, v: roslice base.u8, data_size: base.u32[..= 704]) { + var n : base.u64 + var p : slice base.u8 + var q : roslice base.u8 + + n = (args.data_size as base.u64) * 4 + this.st64!(s: args.h, i: n, v: this.ld64(s: args.h, i: n) ~mod+ this.ld64(s: args.v, i: n)) + if (n <= args.h.length()) and (n <= args.v.length()) { + iterate (p = args.h[.. n], q = args.v[.. n])(length: 4, advance: 4, unroll: 1) { + p.poke_u32le!(a: p.peek_u32le() ~mod+ q.peek_u32le()) + } + } +} + +// histogram_copy is the C code's struct assignment "*h = *v". +pri func bit_cost.histogram_copy!(h: slice base.u8, v: roslice base.u8, data_size: base.u32[..= 704]) { + var n : base.u64 + + n = ((args.data_size as base.u64) * 4) + 16 + if (n <= args.h.length()) and (n <= args.v.length()) { + args.h[.. n].copy_from_slice!(s: args.v[.. n]) + } +} + +// histogram_total_count returns the histogram's total_count_. +pri func bit_cost.histogram_total_count(h: roslice base.u8, data_size: base.u32[..= 704]) base.u64 { + return this.ld64(s: args.h, i: (args.data_size as base.u64) * 4) +} + +// histogram_bit_cost returns the histogram's bit_cost_. +pri func bit_cost.histogram_bit_cost(h: roslice base.u8, data_size: base.u32[..= 704]) base.u64 { + return this.ld64(s: args.h, i: ((args.data_size as base.u64) * 4) + 8) +} + +// histogram_set_bit_cost sets the histogram's bit_cost_. +pri func bit_cost.histogram_set_bit_cost!(h: slice base.u8, data_size: base.u32[..= 704], bit_cost: base.u64) { + this.st64!(s: args.h, i: ((args.data_size as base.u64) * 4) + 8, v: args.bit_cost) +} + +// Mirrors BrotliBuildHistogramsWithContext in histogram.c (with the +// BlockSplitIterator functions inlined). +// +// The literal, command and distance block splits are given as their types +// (u8s), lengths (u32les) and num_blocks. A block split's lengths are the C +// code's NULL pointer when its num_blocks is zero. +// +// The C code's context_modes argument is either NULL or an array whose +// elements are all the same (BrotliBuildMetaBlock's literal_context_mode). +// Here, context_mode is that common element, or 0xFFFF_FFFF for NULL. +// +// lit_histos, cmd_histos and dist_histos are arrays of literal (1040 byte), +// command (2832 byte) and distance (2192 byte) histograms. +pri func bit_cost.build_histograms_with_context!( + cmds: roslice base.u8, + num_commands: base.u64, + lit_types: roslice base.u8, + lit_lengths: roslice base.u8, + lit_num_blocks: base.u64, + cmd_types: roslice base.u8, + cmd_lengths: roslice base.u8, + cmd_num_blocks: base.u64, + dist_types: roslice base.u8, + dist_lengths: roslice base.u8, + dist_num_blocks: base.u64, + ringbuffer: roslice base.u8, + start_pos: base.u64, + mask: base.u64, + prev_byte: base.u8, + prev_byte2: base.u8, + context_mode: base.u32, + lit_histos: slice base.u8, + cmd_histos: slice base.u8, + dist_histos: slice base.u8) { + var pos : base.u64 + var prev_byte : base.u8 + var prev_byte2 : base.u8 + var lit_idx : base.u64 + var lit_type : base.u64 + var lit_length : base.u64 + var cmd_idx : base.u64 + var cmd_type : base.u64 + var cmd_length : base.u64 + var dist_idx : base.u64 + var dist_type : base.u64 + var dist_length : base.u64 + var i : base.u64 + var j : base.u64 + var c : base.u64 + var insert_len : base.u32 + var copy_len : base.u32 + var cmd_prefix : base.u32 + var dist_prefix : base.u32 + var context : base.u64 + var lut_offset : base.u32[..= 1536] + var b : base.u8 + var off : base.u64 + var r : base.u32 + + pos = args.start_pos + prev_byte = args.prev_byte + prev_byte2 = args.prev_byte2 + if args.lit_num_blocks > 0 { + lit_length = this.ld32(s: args.lit_lengths, i: 0) as base.u64 + } + if args.cmd_num_blocks > 0 { + cmd_length = this.ld32(s: args.cmd_lengths, i: 0) as base.u64 + } + if args.dist_num_blocks > 0 { + dist_length = this.ld32(s: args.dist_lengths, i: 0) as base.u64 + } + lut_offset = (args.context_mode & 3) << 9 + + i = 0 + while i < args.num_commands { + c = i ~mod* 16 + insert_len = this.ld32(s: args.cmds, i: c) + copy_len = this.ld32(s: args.cmds, i: c ~mod+ 4) & 0x1FF_FFFF + r = this.ld32(s: args.cmds, i: c ~mod+ 12) + cmd_prefix = r & 0xFFFF + dist_prefix = r >> 16 + + // BlockSplitIteratorNext(&insert_and_copy_it). + if cmd_length == 0 { + cmd_idx ~mod+= 1 + if cmd_idx < args.cmd_types.length() { + cmd_type = args.cmd_types[cmd_idx] as base.u64 + } else { + cmd_type = 0 + } + cmd_length = this.ld32(s: args.cmd_lengths, i: cmd_idx ~mod* 4) as base.u64 + } + cmd_length ~mod-= 1 + off = cmd_type ~mod* HISTOGRAM_COMMAND_SIZE + if off <= args.cmd_histos.length() { + this.histogram_add!(h: args.cmd_histos[off ..], data_size: HISTOGRAM_COMMAND_DATA_SIZE, val: cmd_prefix as base.u64) + } + + j = insert_len as base.u64 + while j <> 0 { + j -= 1 + // BlockSplitIteratorNext(&literal_it). + if lit_length == 0 { + lit_idx ~mod+= 1 + if lit_idx < args.lit_types.length() { + lit_type = args.lit_types[lit_idx] as base.u64 + } else { + lit_type = 0 + } + lit_length = this.ld32(s: args.lit_lengths, i: lit_idx ~mod* 4) as base.u64 + } + lit_length ~mod-= 1 + context = lit_type + if args.context_mode <> 0xFFFF_FFFF { + context = (context ~mod<< 6) ~mod+ (( + CONTEXT_LOOKUP[(lut_offset + (prev_byte as base.u32)) & 2047] | + CONTEXT_LOOKUP[(lut_offset + 256 + (prev_byte2 as base.u32)) & 2047]) as base.u64) + } + b = 0 + if (pos & args.mask) < args.ringbuffer.length() { + b = args.ringbuffer[pos & args.mask] + } + off = context ~mod* HISTOGRAM_LITERAL_SIZE + if off <= args.lit_histos.length() { + this.histogram_add!(h: args.lit_histos[off ..], data_size: HISTOGRAM_LITERAL_DATA_SIZE, val: b as base.u64) + } + prev_byte2 = prev_byte + prev_byte = b + pos ~mod+= 1 + } + + pos ~mod+= copy_len as base.u64 + if copy_len <> 0 { + prev_byte2 = 0 + if ((pos ~mod- 2) & args.mask) < args.ringbuffer.length() { + prev_byte2 = args.ringbuffer[(pos ~mod- 2) & args.mask] + } + prev_byte = 0 + if ((pos ~mod- 1) & args.mask) < args.ringbuffer.length() { + prev_byte = args.ringbuffer[(pos ~mod- 1) & args.mask] + } + if cmd_prefix >= 128 { + // BlockSplitIteratorNext(&dist_it). + if dist_length == 0 { + dist_idx ~mod+= 1 + if dist_idx < args.dist_types.length() { + dist_type = args.dist_types[dist_idx] as base.u64 + } else { + dist_type = 0 + } + dist_length = this.ld32(s: args.dist_lengths, i: dist_idx ~mod* 4) as base.u64 + } + dist_length ~mod-= 1 + context = (dist_type ~mod<< 2) ~mod+ (this.command_distance_context(cmd_prefix: cmd_prefix) as base.u64) + off = context ~mod* HISTOGRAM_DISTANCE_SIZE + if off <= args.dist_histos.length() { + this.histogram_add!(h: args.dist_histos[off ..], data_size: HISTOGRAM_DISTANCE_DATA_SIZE, val: (dist_prefix & 0x3FF) as base.u64) + } + } + } + i ~mod+= 1 + } +} + +// Mirrors CommandDistanceContext in command.h. +pri func bit_cost.command_distance_context(cmd_prefix: base.u32) base.u32 { + var r : base.u32 + var c : base.u32 + + r = args.cmd_prefix >> 6 + c = args.cmd_prefix & 7 + if ((r == 0) or (r == 2) or (r == 4) or (r == 7)) and (c <= 2) { + return c + } + return 3 +} diff --git a/std/brotli/encode_tester.wuffs b/std/brotli/encode_tester.wuffs new file mode 100644 index 00000000..1c5cc099 --- /dev/null +++ b/std/brotli/encode_tester.wuffs @@ -0,0 +1,47 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// -------- + +// TEST ONLY. The metablock_tester struct is not part of the encoder API. It +// exposes the encoder's internal (pri) bit cost, histogram, block splitter, +// clustering and metablock functions as pub methods, so that C test programs +// (test/c/std/brotli.c and script/manual-test-brotli-metablock-parity.c) can +// compare them with the C code's BROTLI_INTERNAL functions. Its methods and +// their semantics may change at any time. + +pub struct metablock_tester?( + dummy : base.u32, +) + ( + // Wuffs' C code generator expects same-package struct fields to be in + // the private_data section (after the "+"). + bc : bit_cost, +) + +// bits_entropy mirrors BrotliBitsEntropy. population holds size u32le values. +pub func metablock_tester.bits_entropy(population: roslice base.u8, size: base.u64) base.u64 { + return this.bc.bits_entropy(population: args.population, size: args.size) +} + +// population_cost mirrors BrotliPopulationCostLiteral (data_size 256), +// BrotliPopulationCostCommand (704) and BrotliPopulationCostDistance (544). +// histogram has the C struct's layout: data_size u32le counts, then the +// total_count_ u64le (and then the bit_cost_, which is ignored). It returns 0 +// for any other data_size. +pub func metablock_tester.population_cost(histogram: roslice base.u8, data_size: base.u32) base.u64 { + if args.data_size == 256 { + return this.bc.population_cost(histogram: args.histogram, data_size: 256) + } else if args.data_size == 704 { + return this.bc.population_cost(histogram: args.histogram, data_size: 704) + } else if args.data_size == 544 { + return this.bc.population_cost(histogram: args.histogram, data_size: 544) + } + return 0 +}