std/brotli: add encoder raw (prefix) dictionaries

Mirror BrotliEncoderPrepareDictionary (BROTLI_SHARED_DICTIONARY_RAW),
CreatePreparedDictionary and BrotliEncoderAttachPreparedDictionary as
encoder.add_dictionary, with the prepared hash tables (slot offsets,
heads and items) in the workbuf after the dictionary bytes, and the
compound dictionary search (FindCompoundDictionaryMatch and
LookupCompoundDictionaryMatch) for the hashers whose
CreateBackwardReferences variant enables it, the compound dictionary
gap for static dictionary distances and the compound branch of
ExtendLastCommand.

Extend the encoder parity script with a -dict grid that also checks
round trips through libbrotlidec and the Wuffs decoder.
This commit is contained in:
Claude committed 2026-10-27 04:54:39 +00:00
1 parent b52e608b7c
commit aabdeaf4b0
6 files changed
+2057 -77

No files matched your search

+48 -2
View File
@@ -194,6 +194,19 @@ pub struct encoder? implements base.io_transformer(
hr_next_ix : base.u64,
hr_off : base.u64,
// The attached raw dictionaries (the C code's CompoundDictionary, see
// encode_dictionary.wuffs). They use the first dict_end bytes of the
// workbuf.
dict_num_chunks : base.u32[..= 15],
dict_total_size : base.u64,
dict_end : base.u64,
dict_chunk_offsets : array[16] base.u64,
dc_src_off : array[15] base.u64,
dc_size : array[15] base.u64,
dc_bucket_bits : array[15] base.u32,
dc_slot_bits : array[15] base.u32,
dc_num_items : array[15] base.u64,
// The C code's HasherSearchResult.
sr_len : base.u64,
sr_distance : base.u64,
@@ -328,6 +341,12 @@ pub struct encoder? implements base.io_transformer(
// GetHashTable result), with up to (1 << 17) elements.
hash_table : array[131072] base.u32,
// CreatePreparedDictionaryWithParams' slot_size and slot_limit
// arrays, and its slot_offsets (see encode_dictionary.wuffs).
dp_slot_size : array[4096] base.u32,
dp_slot_limit : array[4096] base.u32,
dp_slot_off : array[4096] base.u32,
// command_buf and literal_buf are the C code's command_buf_ and
// literal_buf_ (quality 1 only).
command_buf : array[131072] base.u32,
@@ -437,7 +456,10 @@ pub func encoder.workbuf_len() base.range_ii_u64 {
} else if q <= 9 {
n = this.compute_layout_offset(quality: q, lgwin: lgwin,
lgblock: this.computed_lgblock(quality: q, lgwin: lgwin), idx: LAYOUT_END)
} else {
return this.util.make_range_ii_u64(min_incl: 0, max_incl: 0)
}
n ~sat+= this.dict_end
return this.util.make_range_ii_u64(min_incl: n, max_incl: n)
}
@@ -799,7 +821,31 @@ pri func encoder.write_metadata_header!(block_size: base.u32) base.u64 {
// qualities 0 and 1, BrotliEncoderCompressStreamFast). Each (initial or
// resumed) transform_io call corresponds to one or more C calls. See
// README.md for the details.
//
// The attached dictionaries (see add_dictionary) are the first dict_end
// bytes of the workbuf. do_transform_io gets the rest of the workbuf (whose
// layout is described in README.md and encode_data.wuffs) and, as args.dwb,
// the dictionaries.
pub func encoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf: slice base.u8) {
var status : base.status
while true {
if this.dict_end > args.workbuf.length() {
return base."#bad workbuf length"
}
status =? this.do_transform_io?(dst: args.dst, src: args.src,
workbuf: args.workbuf[this.dict_end ..],
dwb: args.workbuf[.. this.dict_end])
if status.is_ok() {
return ok
} else if not status.is_suspension() {
return status
}
yield? status
}
}
pri func encoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, workbuf: slice base.u8, dwb: roslice base.u8) {
var op : base.u32
var avail : base.u64
var block_size : base.u64
@@ -863,7 +909,7 @@ pub func encoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf
continue
}
if (this.quality >= 2) and (this.input_pos <> this.last_flush_pos) {
status = this.encode_data!(wb: args.workbuf, is_last: false, force_flush: true)
status = this.encode_data!(wb: args.workbuf, dwb: args.dwb, is_last: false, force_flush: true)
if not status.is_ok() {
return status
}
@@ -979,7 +1025,7 @@ pub func encoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf
force_flush = true
}
this.update_size_hint!(available_in: args.src.length())
status = this.encode_data!(wb: args.workbuf, is_last: is_last, force_flush: force_flush)
status = this.encode_data!(wb: args.workbuf, dwb: args.dwb, is_last: is_last, force_flush: force_flush)
if not status.is_ok() {
return status
}