mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-17 16:32:11 +01:00
Each bit_cost (one in the encoder, one in its metablock_builder) has a 64Ki-entry FastLog2 cache. The metablock_builder filled all of it before building each quality 10 or 11 metablock (about 20 ms per encoder, since each entry is an exact software log2), and the encoder's own cache was never filled, so the quality 10 and 11 literal cost estimates computed one software log2 per input byte. Instead, a bitmap (zeroed when the struct is initialized) records which cache entries are valid, and the new fast_log2_memo fills an entry the first time it is needed. Typically only a few thousand entries are used. bits_entropy, population_cost and their callers become impure (!) to call it. The values are computed by the same functions, so the results are unchanged. On alice29 at quality 10, the encoder executes about 20% fewer instructions, and a 1000-byte input takes 6 ms instead of 21 ms.
220 lines
9.5 KiB
Plaintext
220 lines
9.5 KiB
Plaintext
// Copyright 2026 The Wuffs Authors.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
|
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
|
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
|
|
// option. This file may not be copied, modified, or distributed
|
|
// except according to those terms.
|
|
//
|
|
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
|
|
|
// --------
|
|
|
|
// TEST ONLY. The metablock_tester struct is not part of the encoder API. It
|
|
// exposes the encoder's internal (pri) bit cost, histogram, block splitter,
|
|
// clustering and metablock functions as pub methods, so that C test programs
|
|
// (test/c/std/brotli.c and script/manual-test-brotli-metablock-parity.c) can
|
|
// compare them with the C code's BROTLI_INTERNAL functions. Its methods and
|
|
// their semantics may change at any time.
|
|
|
|
pub struct metablock_tester?(
|
|
// worst_case is the workbuf_len_worst_case for the last
|
|
// build_meta_block call.
|
|
worst_case : base.u64,
|
|
) + (
|
|
// Wuffs' C code generator expects same-package struct fields to be in
|
|
// the private_data section (after the "+").
|
|
bc : bit_cost,
|
|
mb : metablock_builder,
|
|
)
|
|
|
|
// bits_entropy mirrors BrotliBitsEntropy. population holds size u32le values.
|
|
pub func metablock_tester.bits_entropy!(population: roslice base.u8, size: base.u64) base.u64 {
|
|
var c : base.u64
|
|
|
|
c = this.bc.bits_entropy!(population: args.population, size: args.size)
|
|
return c
|
|
}
|
|
|
|
// population_cost mirrors BrotliPopulationCostLiteral (data_size 256),
|
|
// BrotliPopulationCostCommand (704) and BrotliPopulationCostDistance (544).
|
|
// histogram has the C struct's layout: data_size u32le counts, then the
|
|
// total_count_ u64le (and then the bit_cost_, which is ignored). It returns 0
|
|
// for any other data_size.
|
|
pub func metablock_tester.population_cost!(histogram: roslice base.u8, data_size: base.u32) base.u64 {
|
|
var c : base.u64
|
|
|
|
if args.data_size == 256 {
|
|
c = this.bc.population_cost!(histogram: args.histogram, data_size: 256)
|
|
return c
|
|
} else if args.data_size == 704 {
|
|
c = this.bc.population_cost!(histogram: args.histogram, data_size: 704)
|
|
return c
|
|
} else if args.data_size == 544 {
|
|
c = this.bc.population_cost!(histogram: args.histogram, data_size: 544)
|
|
return c
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// init_distance_params mirrors BrotliInitDistanceParams. Its results are
|
|
// returned by get (keys 24 to 28).
|
|
pub func metablock_tester.init_distance_params!(npostfix: base.u32, ndirect: base.u32, large_window: base.u32) {
|
|
this.mb.init_distance_params!(npostfix: args.npostfix, ndirect: args.ndirect, large_window: args.large_window <> 0)
|
|
}
|
|
|
|
// split_block mirrors BrotliSplitBlock (on a fresh MetaBlockSplit), with the
|
|
// given quality (the only BrotliEncoderParams field it uses). Its results
|
|
// are returned by get (keys 0 to 11) and are in workbuf.
|
|
pub func metablock_tester.split_block!(cmds: roslice base.u8, num_commands: base.u64,
|
|
data: roslice base.u8, pos: base.u64, mask: base.u64, quality: base.u32, workbuf: slice base.u8) base.status {
|
|
this.mb.set_params!(quality: args.quality, large_window: false, disable_literal_context_modeling: false)
|
|
this.mb.reset_arena!(wb_len: args.workbuf.length())
|
|
this.mb.alloc_block_splits!(cmds: args.cmds, num_commands: args.num_commands)
|
|
if this.mb.oom {
|
|
return base."#bad workbuf length"
|
|
}
|
|
this.mb.split_block!(wb: args.workbuf, cmds: args.cmds, num_commands: args.num_commands,
|
|
data: args.data, pos: args.pos, mask: args.mask)
|
|
if this.mb.oom {
|
|
return base."#bad workbuf length"
|
|
}
|
|
return ok
|
|
}
|
|
|
|
// cluster_histograms mirrors BrotliClusterHistogramsLiteral (data_size 256),
|
|
// BrotliClusterHistogramsCommand (704) and BrotliClusterHistogramsDistance
|
|
// (544). The in_size input histograms are copied to the start of workbuf.
|
|
// The output histograms (with room for in_size histograms) and
|
|
// histogram_symbols (in_size u32les) are at the workbuf offsets returned by
|
|
// get (keys 18 and 12). It returns the C code's *out_size, or 0xFFFF_FFFF_FFFF_FFFF
|
|
// if workbuf is too short or data_size is invalid.
|
|
pub func metablock_tester.cluster_histograms!(data_size: base.u32, in_histos: roslice base.u8, in_size: base.u64,
|
|
max_histograms: base.u64, workbuf: slice base.u8) base.u64 {
|
|
var ds : base.u32[..= 704]
|
|
var hs : base.u64
|
|
var n : base.u64
|
|
var in_off : base.u64
|
|
var out_off : base.u64
|
|
var sym_off : base.u64
|
|
var out_size : base.u64
|
|
|
|
if args.data_size == 256 {
|
|
ds = 256
|
|
} else if args.data_size == 704 {
|
|
ds = 704
|
|
} else if args.data_size == 544 {
|
|
ds = 544
|
|
} else {
|
|
return 0xFFFF_FFFF_FFFF_FFFF
|
|
}
|
|
hs = this.mb.hsize(ds: ds)
|
|
this.mb.reset_arena!(wb_len: args.workbuf.length())
|
|
n = args.in_size ~mod* hs
|
|
in_off = this.mb.alloc!(n: n)
|
|
out_off = this.mb.alloc!(n: n)
|
|
sym_off = this.mb.alloc!(n: args.in_size ~mod* 4)
|
|
if this.mb.oom or (n > args.in_histos.length()) or (n > args.workbuf.length()) {
|
|
return 0xFFFF_FFFF_FFFF_FFFF
|
|
}
|
|
args.workbuf[.. n].copy_from_slice!(s: args.in_histos[.. n])
|
|
out_size = this.mb.cluster_histograms!(wb: args.workbuf, ds: ds, in_histos: in_off, in_size: args.in_size,
|
|
max_histograms: args.max_histograms, out_histos: out_off, histogram_symbols: sym_off)
|
|
if this.mb.oom {
|
|
return 0xFFFF_FFFF_FFFF_FFFF
|
|
}
|
|
this.mb.histograms_off[0] = out_off
|
|
this.mb.context_map_off[0] = sym_off
|
|
return out_size
|
|
}
|
|
|
|
// build_meta_block mirrors BrotliBuildMetaBlock. The params are the given
|
|
// quality, large_window, disable_literal_context_modeling and distance params
|
|
// (as set by BrotliInitDistanceParams(npostfix, ndirect, large_window)). Like
|
|
// the C code, it rewrites the commands' distance prefixes and extra bits. Its
|
|
// results are returned by get and are in workbuf.
|
|
pub func metablock_tester.build_meta_block!(workbuf: slice base.u8, ringbuffer: roslice base.u8, pos: base.u64, mask: base.u64,
|
|
prev_byte: base.u8, prev_byte2: base.u8, cmds: slice base.u8, num_commands: base.u64,
|
|
literal_context_mode: base.u32, quality: base.u32, large_window: base.u32,
|
|
disable_literal_context_modeling: base.u32, npostfix: base.u32, ndirect: base.u32) base.status {
|
|
var status : base.status
|
|
var i : base.u64
|
|
var literals : base.u64
|
|
|
|
i = 0
|
|
while i < args.num_commands {
|
|
literals ~mod+= this.mb.ld32(wb: args.cmds, i: i ~mod* 16) as base.u64
|
|
i ~mod+= 1
|
|
}
|
|
this.worst_case = this.mb.workbuf_len_worst_case(num_literals: literals, num_commands: args.num_commands)
|
|
this.mb.set_params!(quality: args.quality, large_window: args.large_window <> 0,
|
|
disable_literal_context_modeling: args.disable_literal_context_modeling <> 0)
|
|
this.mb.init_distance_params!(npostfix: args.npostfix, ndirect: args.ndirect, large_window: args.large_window <> 0)
|
|
status = this.mb.build_meta_block!(wb: args.workbuf, ringbuffer: args.ringbuffer, pos: args.pos, mask: args.mask,
|
|
prev_byte: args.prev_byte, prev_byte2: args.prev_byte2, cmds: args.cmds, num_commands: args.num_commands,
|
|
literal_context_mode: args.literal_context_mode)
|
|
return status
|
|
}
|
|
|
|
// optimize_histograms mirrors BrotliOptimizeHistograms, on the
|
|
// MetaBlockSplit from the last build_meta_block call (in workbuf).
|
|
pub func metablock_tester.optimize_histograms!(workbuf: slice base.u8, num_distance_codes: base.u32) {
|
|
this.mb.optimize_histograms!(wb: args.workbuf, num_distance_codes: args.num_distance_codes)
|
|
}
|
|
|
|
// get returns one of the metablock_builder's fields:
|
|
// - 4 * c + 0: split_num_types[c], for c (the category) in 0, 1, 2
|
|
// (literal, command, distance).
|
|
// - 4 * c + 1: split_num_blocks[c].
|
|
// - 4 * c + 2: split_types_off[c].
|
|
// - 4 * c + 3: split_lengths_off[c].
|
|
// - 12 + c: context_map_off[c].
|
|
// - 15 + c: context_map_size[c].
|
|
// - 18 + c: histograms_off[c].
|
|
// - 21 + c: histograms_size[c].
|
|
// - 24 to 28: the distance params' distance_postfix_bits,
|
|
// num_direct_distance_codes, alphabet_size_max, alphabet_size_limit and
|
|
// max_distance.
|
|
// - 29: the highest workbuf offset used (the workbuf length needed).
|
|
// - 30: the workbuf_len_worst_case for the last build_meta_block call.
|
|
pub func metablock_tester.get(key: base.u32) base.u64 {
|
|
var c : base.u32[..= 2]
|
|
|
|
if args.key < 12 {
|
|
c = ((args.key / 4) % 3) as base.u32
|
|
if (args.key % 4) == 0 {
|
|
return this.mb.split_num_types[c]
|
|
} else if (args.key % 4) == 1 {
|
|
return this.mb.split_num_blocks[c]
|
|
} else if (args.key % 4) == 2 {
|
|
return this.mb.split_types_off[c]
|
|
}
|
|
return this.mb.split_lengths_off[c]
|
|
} else if args.key < 24 {
|
|
c = ((args.key % 3) as base.u32)
|
|
if args.key < 15 {
|
|
return this.mb.context_map_off[c]
|
|
} else if args.key < 18 {
|
|
return this.mb.context_map_size[c]
|
|
} else if args.key < 21 {
|
|
return this.mb.histograms_off[c]
|
|
}
|
|
return this.mb.histograms_size[c]
|
|
} else if args.key == 24 {
|
|
return this.mb.dist_postfix_bits as base.u64
|
|
} else if args.key == 25 {
|
|
return this.mb.dist_num_direct_codes as base.u64
|
|
} else if args.key == 26 {
|
|
return this.mb.dist_alphabet_size_max as base.u64
|
|
} else if args.key == 27 {
|
|
return this.mb.dist_alphabet_size_limit as base.u64
|
|
} else if args.key == 28 {
|
|
return this.mb.dist_max_distance
|
|
} else if args.key == 29 {
|
|
return this.mb.wb_high
|
|
} else if args.key == 30 {
|
|
return this.worst_case
|
|
}
|
|
return 0
|
|
}
|