std/brotli: use metablock_builder's distance and RLE helpers in the encoder

This removes the encoder's copies of PrefixEncodeCopyDistance,
CommandRestoreDistanceCode, BrotliInitDistanceParams and
BrotliOptimizeHuffmanCountsForRle. The shared BrotliInitDistanceParams
also handles large-window distance params (used at quality 2).
This commit is contained in:
Claude committed 2026-10-27 01:15:03 +00:00
1 parent abecc768e5
commit 314b20e832
5 files changed
+179 -674

No files matched your search

+7 -57
View File
@@ -153,37 +153,6 @@ pri func encoder.get_length_code(insertlen: base.u64, copylen: base.u64, use_las
use_last_distance: args.use_last_distance)
}
// Mirrors PrefixEncodeCopyDistance in prefix.h. It returns the code (the C
// code's *code, in the low 16 bits) and the extra bits (the C code's
// *extra_bits, in the high 32 bits).
pri func encoder.prefix_encode_copy_distance(distance_code: base.u64, num_direct_codes: base.u64, postfix_bits: base.u32[..= 3]) base.u64 {
var dist : base.u64
var bucket : base.u64
var postfix_mask : base.u64
var postfix : base.u64
var prefix : base.u64
var offset : base.u64
var nbits : base.u64
var code : base.u64
var extra : base.u64
if args.distance_code < (16 ~mod+ args.num_direct_codes) {
return args.distance_code & 0xFFFF
}
dist = ((1 as base.u64) << ((args.postfix_bits + 2) & 63)) ~mod+
((args.distance_code ~mod- 16) ~mod- args.num_direct_codes)
bucket = (this.log2_floor_non_zero(n: dist) as base.u64) ~mod- 1
postfix_mask = ((1 as base.u64) << (args.postfix_bits & 63)) ~mod- 1
postfix = dist & postfix_mask
prefix = (dist >> (bucket & 63)) & 1
offset = (2 ~mod+ prefix) ~mod<< (bucket & 63)
nbits = bucket ~mod- (args.postfix_bits as base.u64)
code = (nbits ~mod<< 10) | (((16 ~mod+ args.num_direct_codes) ~mod+
(((2 ~mod* (nbits ~mod- 1)) ~mod+ prefix) ~mod<< (args.postfix_bits & 63))) ~mod+ postfix)
extra = ((dist ~mod- offset) >> (args.postfix_bits & 63)) & 0xFFFF_FFFF
return (code & 0xFFFF) | (extra << 32)
}
// Mirrors InitCommand in command.h, storing the n'th command.
pri func encoder.init_command!(wb: slice base.u8, n: base.u64, insertlen: base.u64, copylen: base.u64, copylen_code_delta: base.u64, distance_code: base.u64) {
var delta : base.u32
@@ -192,10 +161,10 @@ pri func encoder.init_command!(wb: slice base.u8, n: base.u64, insertlen: base.u
var cmd_prefix : base.u16
delta = (args.copylen_code_delta & 0xFF) as base.u32
pe = this.prefix_encode_copy_distance(
pe = this.mb.prefix_encode_copy_distance(
distance_code: args.distance_code,
num_direct_codes: this.num_direct_distance_codes as base.u64,
postfix_bits: this.distance_postfix_bits & 3)
postfix_bits: this.distance_postfix_bits)
dist_prefix = (pe & 0xFFFF) as base.u16
cmd_prefix = this.get_length_code(
insertlen: args.insertlen,
@@ -219,31 +188,12 @@ pri func encoder.init_insert_command!(wb: slice base.u8, n: base.u64, insertlen:
dist_prefix: 16)
}
// Mirrors CommandRestoreDistanceCode in command.h.
// Mirrors CommandRestoreDistanceCode in command.h, for the i'th command.
pri func encoder.cmd_restore_distance_code(wb: roslice base.u8, i: base.u64) base.u32 {
var dist_prefix : base.u32
var dcode : base.u32
var nbits : base.u32
var extra : base.u32
var postfix_mask : base.u32
var hcode : base.u32
var lcode : base.u32
var offset : base.u32
var npostfix : base.u32
dist_prefix = this.cmd_dist_prefix(wb: args.wb, i: args.i) as base.u32
npostfix = this.distance_postfix_bits & 31
if (dist_prefix & 0x3FF) < (16 ~mod+ this.num_direct_distance_codes) {
return dist_prefix & 0x3FF
}
dcode = dist_prefix & 0x3FF
nbits = dist_prefix >> 10
extra = this.cmd_dist_extra(wb: args.wb, i: args.i)
postfix_mask = ((1 as base.u32) << npostfix) ~mod- 1
hcode = ((dcode ~mod- this.num_direct_distance_codes) ~mod- 16) >> npostfix
lcode = ((dcode ~mod- this.num_direct_distance_codes) ~mod- 16) & postfix_mask
offset = ((2 ~mod+ (hcode & 1)) ~mod<< (nbits & 31)) ~mod- 4
return ((((offset ~mod+ extra) ~mod<< npostfix) ~mod+ lcode) ~mod+ this.num_direct_distance_codes) ~mod+ 16
return this.mb.command_restore_distance_code(wb: args.wb,
c: this.cmds_off ~mod+ (args.i ~mod* 16),
npostfix: this.distance_postfix_bits,
ndirect: this.num_direct_distance_codes)
}
// Mirrors ComputeDistanceCode in backward_references.c.
+4 -3
View File
@@ -200,6 +200,10 @@ pub struct encoder? implements base.io_transformer(
) + (
bc : bit_cost,
// mb provides some metablock.c, command.h and prefix.h functions
// (see encode_metablock.wuffs).
mb : metablock_builder,
// tiny_buf is the C code's tiny_buf_ (plus slack for the 8-byte
// stores of write_bits).
tiny_buf : array[32] base.u8,
@@ -269,9 +273,6 @@ pub struct encoder? implements base.io_transformer(
gbs_entropy_tmp : array[16] base.u64,
gbs_comb_entropy : array[32] base.u64,
// BrotliOptimizeHistograms' good_for_rle.
good_for_rle : array[1024] base.u8,
// The BrotliStoreMetaBlock block encoders' state (the C code's
// BlockEncoder and BlockSplitCode), indexed by block category (with
// index 3 for a local BlockTypeCodeCalculator).
+10 -10
View File
@@ -276,17 +276,17 @@ pri func encoder.compute_layout!() {
this.rb_total_size = this.rb_size ~mod+ this.rb_tail_size
}
// Mirrors BrotliInitDistanceParams in metablock.c (without large window).
// Mirrors BrotliInitDistanceParams in metablock.c (see
// metablock_builder.init_distance_params), for this.distance_postfix_bits and
// this.num_direct_distance_codes.
pri func encoder.init_distance_params!() {
var npostfix : base.u32
var ndirect : base.u32
npostfix = this.distance_postfix_bits & 3
ndirect = this.num_direct_distance_codes
this.dist_alphabet_size_max = (16 ~mod+ ndirect) ~mod+ ((24 as base.u32) << (npostfix + 1))
this.dist_alphabet_size_limit = this.dist_alphabet_size_max
this.dist_max_distance = ((ndirect ~mod+ ((1 as base.u32) << (24 + npostfix + 2))) ~mod-
((1 as base.u32) << (npostfix + 2))) as base.u64
this.mb.init_distance_params!(
npostfix: this.distance_postfix_bits,
ndirect: this.num_direct_distance_codes,
large_window: this.large_window)
this.dist_alphabet_size_max = this.mb.dist_alphabet_size_max
this.dist_alphabet_size_limit = this.mb.dist_alphabet_size_limit
this.dist_max_distance = this.mb.dist_max_distance
}
// Mirrors WrapPosition in encode.c.
+6 -189
View File
@@ -12,8 +12,9 @@
// This file mirrors BrotliBuildMetaBlockGreedy (metablock.c and
// metablock_inc.h), the metablock builder for qualities 4 to 9, and
// BrotliOptimizeHistograms (metablock.c) with
// BrotliOptimizeHuffmanCountsForRle (entropy_encode.c).
// BrotliOptimizeHistograms (metablock.c) for its output. The latter uses
// metablock_builder.optimize_huffman_counts_for_rle (see
// encode_metablock.wuffs) for BrotliOptimizeHuffmanCountsForRle.
//
// The C code's three FN(BlockSplitter)s and its ContextBlockSplitter are one
// greedy block splitter here (the gbs_etc functions and fields), indexed by
@@ -424,202 +425,18 @@ pri func encoder.optimize_histograms!(wb: slice base.u8, num_distance_codes: bas
i = 0
while i < this.gbs_histograms_size[0] {
this.optimize_huffman_counts_for_rle!(wb: args.wb, off: this.gbs_histo_off(c: 0, ix: i), length: 256)
this.mb.optimize_huffman_counts_for_rle!(wb: args.wb, counts: this.gbs_histo_off(c: 0, ix: i), length: 256)
i ~mod+= 1
}
i = 0
while i < this.gbs_histograms_size[1] {
this.optimize_huffman_counts_for_rle!(wb: args.wb, off: this.gbs_histo_off(c: 1, ix: i), length: 704)
this.mb.optimize_huffman_counts_for_rle!(wb: args.wb, counts: this.gbs_histo_off(c: 1, ix: i), length: 704)
i ~mod+= 1
}
i = 0
while i < this.gbs_histograms_size[2] {
this.optimize_huffman_counts_for_rle!(wb: args.wb, off: this.gbs_histo_off(c: 2, ix: i),
this.mb.optimize_huffman_counts_for_rle!(wb: args.wb, counts: this.gbs_histo_off(c: 2, ix: i),
length: args.num_distance_codes as base.u64)
i ~mod+= 1
}
}
// Mirrors BrotliOptimizeHuffmanCountsForRle in entropy_encode.c, for the
// length u32le counts at args.wb[off ..]. The good_for_rle array is
// this.good_for_rle.
pri func encoder.optimize_huffman_counts_for_rle!(wb: slice base.u8, off: base.u64, length: base.u64) {
var length : base.u64
var nonzero_count : base.u64
var stride : base.u64
var limit : base.u64
var sum : base.u64
var i : base.u64
var k : base.u64
var nonzeros : base.u64
var smallest_nonzero : base.u32
var zeros : base.u64
var symbol : base.u32
var step : base.u64
var count : base.u64
var c : base.u32
var cond : base.bool
length = args.length.min(no_more_than: 704)
i = 0
while i < length {
if this.ctr(wb: args.wb, off: args.off, i: i) <> 0 {
nonzero_count ~mod+= 1
}
i ~mod+= 1
}
if nonzero_count < 16 {
return nothing
}
while (length <> 0) and (this.ctr(wb: args.wb, off: args.off, i: length ~mod- 1) == 0) {
length ~mod-= 1
}
if length == 0 {
return nothing // All zeros.
}
// Now counts[0..length - 1] does not have trailing zeros.
smallest_nonzero = 0x4000_0000
i = 0
while i < length {
c = this.ctr(wb: args.wb, off: args.off, i: i)
if c <> 0 {
nonzeros ~mod+= 1
if smallest_nonzero > c {
smallest_nonzero = c
}
}
i ~mod+= 1
}
if nonzeros < 5 {
// Small histogram will model it well.
return nothing
}
if smallest_nonzero < 4 {
zeros = length ~mod- nonzeros
if zeros < 6 {
i = 1
while (i ~mod+ 1) < length {
if (this.ctr(wb: args.wb, off: args.off, i: i ~mod- 1) <> 0) and
(this.ctr(wb: args.wb, off: args.off, i: i) == 0) and
(this.ctr(wb: args.wb, off: args.off, i: i ~mod+ 1) <> 0) {
this.poke_u32!(s: args.wb, i: args.off ~mod+ (i ~mod* 4), v: 1)
}
i ~mod+= 1
}
}
}
if nonzeros < 28 {
return nothing
}
// 2) Let's mark all population counts that already can be encoded with
// an RLE code.
i = 0
while i < length {
this.good_for_rle[i & 1023] = 0
i ~mod+= 1
}
// Let's not spoil any of the existing good RLE codes. Mark any seq of 0's
// that is longer as 5 as a good_for_rle. Mark any seq of non-0's that is
// longer as 7 as a good_for_rle.
symbol = this.ctr(wb: args.wb, off: args.off, i: 0)
step = 0
i = 0
while i <= length {
if (i == length) or (this.ctr(wb: args.wb, off: args.off, i: i) <> symbol) {
if ((symbol == 0) and (step >= 5)) or ((symbol <> 0) and (step >= 7)) {
k = 0
while k < step {
this.good_for_rle[((i ~mod- k) ~mod- 1) & 1023] = 1
k ~mod+= 1
}
}
step = 1
if i <> length {
symbol = this.ctr(wb: args.wb, off: args.off, i: i)
}
} else {
step ~mod+= 1
}
i ~mod+= 1
}
// 3) Let's replace those population counts that lead to more RLE codes.
// Math here is in 24.8 fixed point representation.
stride = 0
limit = this.rle_limit3(wb: args.wb, off: args.off, i: 0)
sum = 0
i = 0
while i <= length {
cond = (i == length)
if not cond {
cond = (this.good_for_rle[i & 1023] <> 0)
}
if (not cond) and (i <> 0) {
cond = (this.good_for_rle[(i - 1) & 1023] <> 0)
}
if not cond {
cond = ((((this.ctr(wb: args.wb, off: args.off, i: i) ~mod* 256) as base.u64) ~mod- limit) ~mod+ 1240) >= 2480
}
if cond {
if (stride >= 4) or ((stride >= 3) and (sum == 0)) {
// The stride must end, collapse what we have, if we have
// enough (4).
count = (sum ~mod+ (stride / 2)) / stride.max(no_less_than: 1)
if count == 0 {
count = 1
}
if sum == 0 {
// Don't make an all zeros stride to be upgraded to ones.
count = 0
}
k = 0
while k < stride {
// We don't want to change value at counts[i], that is
// already belonging to the next stride. Thus - 1.
this.poke_u32!(s: args.wb, i: args.off ~mod+ (((i ~mod- k) ~mod- 1) ~mod* 4),
v: (count & 0xFFFF_FFFF) as base.u32)
k ~mod+= 1
}
}
stride = 0
sum = 0
if (i ~mod+ 2) < length {
// All interesting strides have a count of at least 4, at
// least when non-zeros.
limit = this.rle_limit3(wb: args.wb, off: args.off, i: i)
} else if i < length {
limit = (this.ctr(wb: args.wb, off: args.off, i: i) ~mod* 256) as base.u64
} else {
limit = 0
}
}
stride ~mod+= 1
if i <> length {
sum ~mod+= this.ctr(wb: args.wb, off: args.off, i: i) as base.u64
if stride >= 4 {
limit = ((sum ~mod* 256) ~mod+ (stride / 2)) / stride.max(no_less_than: 1)
}
if stride == 4 {
limit ~mod+= 120
}
}
i ~mod+= 1
}
}
// ctr returns counts[i], the u32le at args.wb[off + (4 * i) ..].
pri func encoder.ctr(wb: roslice base.u8, off: base.u64, i: base.u64) base.u32 {
return this.load32le(s: args.wb, i: args.off ~mod+ (args.i ~mod* 4))
}
// rle_limit3 returns the C code's (size_t)(256 * (counts[i] + counts[i + 1]
// + counts[i + 2]) / 3 + 420), computed in uint32_t arithmetic.
pri func encoder.rle_limit3(wb: roslice base.u8, off: base.u64, i: base.u64) base.u64 {
var s : base.u32
s = this.ctr(wb: args.wb, off: args.off, i: args.i)
s ~mod+= this.ctr(wb: args.wb, off: args.off, i: args.i ~mod+ 1)
s ~mod+= this.ctr(wb: args.wb, off: args.off, i: args.i ~mod+ 2)
return (((s ~mod* 256) / 3) ~mod+ 420) as base.u64
}