mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 00:32:11 +01:00
std/brotli: let a shorter encoder workbuf work for short inputs
Move the ring buffer to the end of the workbuf layout (at qualities 2 to 9). A workbuf shorter than workbuf_len now works if it holds the input so far (plus 16 bytes of slack) in the ring buffer region and the input stays short enough for the ring buffer never to wrap. This matters most for large-window brotli, where workbuf_len at lgwin 30 is more than 2 GiB, mostly for a ring buffer that short inputs never touch. The parity harness now also checks its large-window configurations with the shortest such workbuf (and that one byte less fails).
This commit is contained in:
5 files changed
+262
-61
No files matched your search
@@ -157,6 +157,12 @@ pub struct encoder? implements base.io_transformer(
|
||||
rb_cur_size : base.u32,
|
||||
rb_pos : base.u32,
|
||||
|
||||
// rb_short_workbuf is whether any transform_io call (at qualities 2
|
||||
// to 9) had a workbuf shorter than workbuf_len. If so, the ring
|
||||
// buffer's tail may be incomplete, so the input must stay short
|
||||
// enough for the ring buffer never to wrap.
|
||||
rb_short_workbuf : base.bool,
|
||||
|
||||
// The C code's BrotliDistanceParams fields (other than
|
||||
// distance_postfix_bits and num_direct_distance_codes).
|
||||
dist_alphabet_size_max : base.u32,
|
||||
@@ -809,13 +815,20 @@ pub func encoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf
|
||||
var injected : base.bool
|
||||
|
||||
var copy_input_size : base.u64
|
||||
var total : base.u64
|
||||
|
||||
this.ensure_initialized!()
|
||||
if this.quality > 9 {
|
||||
// TODO: implement Zopfli (qualities 10 and 11).
|
||||
return "#unsupported encoder quality"
|
||||
} else if (this.quality >= 2) and (args.workbuf.length() < this.workbuf_end) {
|
||||
return base."#bad workbuf length"
|
||||
// A workbuf shorter than workbuf_len works if it is long enough for
|
||||
// the input so far (see the rb_short_workbuf check below), as long
|
||||
// as the ring buffer never wraps.
|
||||
this.rb_short_workbuf = true
|
||||
if args.workbuf.length() < (this.rb_buf_off ~mod+ RB_SHORT_SLACK) {
|
||||
return base."#bad workbuf length"
|
||||
}
|
||||
}
|
||||
|
||||
while true {
|
||||
@@ -915,6 +928,16 @@ pub func encoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf
|
||||
|
||||
if (remaining <> 0) and (args.src.length() > 0) {
|
||||
copy_input_size = remaining.min(no_more_than: args.src.length())
|
||||
if this.rb_short_workbuf {
|
||||
// The ring buffer must not wrap (or reach its tail) and
|
||||
// the workbuf must hold the input so far plus some
|
||||
// slack.
|
||||
total = (this.input_pos ~mod+ copy_input_size) ~mod+ RB_SHORT_SLACK
|
||||
if (total > (this.rb_size as base.u64)) or
|
||||
((this.rb_buf_off ~mod+ total) > args.workbuf.length()) {
|
||||
return base."#bad workbuf length"
|
||||
}
|
||||
}
|
||||
mark = args.src.mark()
|
||||
args.src.skip?(n: copy_input_size)
|
||||
this.copy_input_to_ring_buffer!(wb: args.workbuf, bytes: args.src.since(mark: mark))
|
||||
|
||||
@@ -17,29 +17,36 @@
|
||||
// The workbuf holds, in this order (see README.md's "Encoder Workbuf Layout"
|
||||
// section, which compute_layout_offset mirrors):
|
||||
// - the storage (the C code's GetBrotliStorage) for one metablock,
|
||||
// - the ring buffer (the C code's RingBuffer.data_),
|
||||
// - the hasher's memory (the C code's common->extra),
|
||||
// - the commands (16 bytes each), and
|
||||
// - the commands (16 bytes each),
|
||||
// - at qualities 4 and above, the metablock split (see
|
||||
// encode_metablock_greedy.wuffs), the literal context map, the
|
||||
// EncodeContextMap symbols and the block encoders' depths and bits.
|
||||
// EncodeContextMap symbols and the block encoders' depths and bits, and
|
||||
// - the ring buffer (the C code's RingBuffer.data_).
|
||||
|
||||
// The workbuf layout's region indexes. LAYOUT_END's offset is the total
|
||||
// length.
|
||||
// The ring buffer is last, so that a workbuf shorter than workbuf_len works
|
||||
// for short inputs (see transform_io's rb_short_workbuf check).
|
||||
pri const LAYOUT_STORAGE : base.u32 = 0
|
||||
pri const LAYOUT_RB : base.u32 = 1
|
||||
pri const LAYOUT_HASH : base.u32 = 2
|
||||
pri const LAYOUT_CMDS : base.u32 = 3
|
||||
pri const LAYOUT_TYPES_0 : base.u32 = 4
|
||||
pri const LAYOUT_LENGTHS_0 : base.u32 = 7
|
||||
pri const LAYOUT_HISTO_0 : base.u32 = 10
|
||||
pri const LAYOUT_COMB : base.u32 = 13
|
||||
pri const LAYOUT_CMAP : base.u32 = 14
|
||||
pri const LAYOUT_RLE : base.u32 = 15
|
||||
pri const LAYOUT_BE_DEPTH_0 : base.u32 = 16
|
||||
pri const LAYOUT_BE_BITS_0 : base.u32 = 19
|
||||
pri const LAYOUT_HASH : base.u32 = 1
|
||||
pri const LAYOUT_CMDS : base.u32 = 2
|
||||
pri const LAYOUT_TYPES_0 : base.u32 = 3
|
||||
pri const LAYOUT_LENGTHS_0 : base.u32 = 6
|
||||
pri const LAYOUT_HISTO_0 : base.u32 = 9
|
||||
pri const LAYOUT_COMB : base.u32 = 12
|
||||
pri const LAYOUT_CMAP : base.u32 = 13
|
||||
pri const LAYOUT_RLE : base.u32 = 14
|
||||
pri const LAYOUT_BE_DEPTH_0 : base.u32 = 15
|
||||
pri const LAYOUT_BE_BITS_0 : base.u32 = 18
|
||||
pri const LAYOUT_RB : base.u32 = 21
|
||||
pri const LAYOUT_END : base.u32 = 22
|
||||
|
||||
// RB_SHORT_SLACK is how many bytes past the input's end (in the ring buffer)
|
||||
// a short workbuf must have: the C code reads at most 7 bytes past the
|
||||
// input's end (its kSlackForEightByteHashingEverywhere), rounded up.
|
||||
pri const RB_SHORT_SLACK : base.u64 = 16
|
||||
|
||||
// The C code's MAX_NUM_DELAYED_SYMBOLS (in quality.h).
|
||||
pri const MAX_NUM_DELAYED_SYMBOLS : base.u64 = 0x2FFF
|
||||
|
||||
|
||||
Reference in new issue
Block a user