mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-20 16:47:28 +01:00
std/lzma: stash LZMA state across multiple chunks
This commit is contained in:
1 parent
6c71bb229b
commit
6017c37232
5 files changed
+459
-198
No files matched your search
@@ -35,7 +35,8 @@ pri func decoder.decode_bitstream_slow?(dst: base.io_writer, src: base.io_reader
|
||||
var reptmp : base.u32[..= 0xFFFF_FFFE]
|
||||
var rep : base.u32
|
||||
|
||||
var pos : base.u64
|
||||
var pos : base.u64
|
||||
var pos_end : base.u64
|
||||
|
||||
var lc : base.u32[..= 8]
|
||||
var lp_mask : base.u64[..= 15]
|
||||
@@ -79,23 +80,24 @@ pri func decoder.decode_bitstream_slow?(dst: base.io_writer, src: base.io_reader
|
||||
var adj_dist : base.u32
|
||||
var wb_index : base.u64
|
||||
|
||||
// Initialize the bitstream.
|
||||
c8 = args.src.read_u8?()
|
||||
if c8 <> 0x00 {
|
||||
return "#bad code"
|
||||
}
|
||||
bits = args.src.read_u32be?()
|
||||
if bits == 0xFFFF_FFFF {
|
||||
return "#bad code"
|
||||
}
|
||||
range = 0xFFFF_FFFF
|
||||
prev_byte = this.stashed_bytes[0]
|
||||
match_byte = this.stashed_bytes[1]
|
||||
bits = this.stashed_bits
|
||||
range = this.stashed_range
|
||||
state = this.stashed_state
|
||||
rep0 = this.stashed_rep0
|
||||
rep1 = this.stashed_rep1
|
||||
rep2 = this.stashed_rep2
|
||||
rep3 = this.stashed_rep3
|
||||
pos = this.stashed_pos
|
||||
|
||||
lc = this.lc
|
||||
lp_mask = ((1 as base.u64) << this.lp) - 1
|
||||
pb_mask = ((1 as base.u64) << this.pb) - 1
|
||||
|
||||
// "outer" is the core loop discussed in README.md's "Algorithm Overview".
|
||||
while.outer pos < this.decoded_length {
|
||||
pos_end = pos ~sat+ this.decoded_length
|
||||
while.outer pos < pos_end {
|
||||
// AlgOve00 if decodeTheNextBym() == 0
|
||||
index_ao00 = (state << 4) | ((pos & pb_mask) as base.u32)
|
||||
prob = this.probs_ao00[index_ao00] as base.u32
|
||||
@@ -819,7 +821,7 @@ pri func decoder.decode_bitstream_slow?(dst: base.io_writer, src: base.io_reader
|
||||
assert len >= 1
|
||||
dist = rep0 + 1
|
||||
assert dist >= 1
|
||||
if ((dist as base.u64) > (pos ~mod+ this.lzma2_prior_decoded_length)) or
|
||||
if ((dist as base.u64) > pos) or
|
||||
((dist as base.u64) > (this.dict_size as base.u64)) {
|
||||
return "#bad distance"
|
||||
}
|
||||
@@ -847,7 +849,7 @@ pri func decoder.decode_bitstream_slow?(dst: base.io_writer, src: base.io_reader
|
||||
{
|
||||
wb_index ~mod+= this.dict_size as base.u64
|
||||
} endwhile
|
||||
if wb_index > args.workbuf.length() {
|
||||
if wb_index >= args.workbuf.length() {
|
||||
return "#internal error: inconsistent dictionary state"
|
||||
}
|
||||
|
||||
@@ -904,8 +906,18 @@ pri func decoder.decode_bitstream_slow?(dst: base.io_writer, src: base.io_reader
|
||||
|
||||
if bits <> 0 {
|
||||
return "#bad bitstream trailer"
|
||||
} else if (this.decoded_length <> 0xFFFF_FFFF_FFFF_FFFF) and
|
||||
(this.decoded_length <> pos) {
|
||||
} else if (this.decoded_length <> 0xFFFF_FFFF_FFFF_FFFF) and (pos <> pos_end) {
|
||||
return "#bad decoded length"
|
||||
}
|
||||
|
||||
this.stashed_bytes[0] = prev_byte
|
||||
this.stashed_bytes[1] = match_byte
|
||||
this.stashed_bits = bits
|
||||
this.stashed_range = range
|
||||
this.stashed_state = state
|
||||
this.stashed_rep0 = rep0
|
||||
this.stashed_rep1 = rep1
|
||||
this.stashed_rep2 = rep2
|
||||
this.stashed_rep3 = rep3
|
||||
this.stashed_pos = pos
|
||||
}
|
||||
+99
-14
@@ -52,11 +52,23 @@ pub struct decoder? implements base.io_transformer(
|
||||
|
||||
decoded_length : base.u64,
|
||||
|
||||
lzma2_prior_decoded_length : base.u64,
|
||||
lzma2_encoded_length_have : base.u64,
|
||||
lzma2_encoded_length_want : base.u64,
|
||||
lzma2_need_prob_reset : base.bool,
|
||||
lzma2_need_properties : base.bool,
|
||||
lzma2_encoded_length_have : base.u64,
|
||||
lzma2_encoded_length_want : base.u64,
|
||||
lzma2_need_prob_reset : base.bool,
|
||||
lzma2_need_properties : base.bool,
|
||||
lzma2_need_dict_reset : base.bool,
|
||||
|
||||
prev_lzma2_chunk_was_uncompressed : base.bool,
|
||||
|
||||
stashed_bytes : array[2] base.u8, // 0 is prev_byte, 1 is match_byte.
|
||||
stashed_bits : base.u32,
|
||||
stashed_range : base.u32,
|
||||
stashed_state : base.u32[..= 11],
|
||||
stashed_rep0 : base.u32[..= 0xFFFF_FFFE],
|
||||
stashed_rep1 : base.u32[..= 0xFFFF_FFFE],
|
||||
stashed_rep2 : base.u32[..= 0xFFFF_FFFE],
|
||||
stashed_rep3 : base.u32[..= 0xFFFF_FFFE],
|
||||
stashed_pos : base.u64,
|
||||
|
||||
util : base.utility,
|
||||
) + (
|
||||
@@ -317,6 +329,7 @@ pub func decoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf
|
||||
|
||||
pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, workbuf: roslice base.u8) {
|
||||
var header_byte : base.u8
|
||||
var c8 : base.u8
|
||||
var c32 : base.u32
|
||||
var prop_byte : base.u8
|
||||
|
||||
@@ -327,12 +340,12 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work
|
||||
var length : base.u32
|
||||
var n_copied : base.u32
|
||||
|
||||
var dmark : base.u64
|
||||
var smark : base.u64
|
||||
var status : base.status
|
||||
|
||||
this.lzma2_need_prob_reset = true
|
||||
this.lzma2_need_properties = true
|
||||
this.lzma2_need_dict_reset = true
|
||||
|
||||
// Loop over chunks. LZMA1 has only one (implicit) chunk. For LZMA2's chunk
|
||||
// header, see
|
||||
@@ -380,19 +393,21 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work
|
||||
header_byte = args.src.read_u8?()
|
||||
if header_byte < 0x80 {
|
||||
if header_byte < 0x02 {
|
||||
this.lzma2_prior_decoded_length = 0
|
||||
this.lzma2_need_prob_reset = true
|
||||
this.lzma2_need_properties = true
|
||||
} else if header_byte > 0x02 {
|
||||
this.lzma2_need_dict_reset = false
|
||||
this.initialize_dict!()
|
||||
} else if (header_byte > 0x02) or this.lzma2_need_dict_reset {
|
||||
return "#bad LZMA2 header"
|
||||
}
|
||||
this.prev_lzma2_chunk_was_uncompressed = true
|
||||
|
||||
c32 = args.src.read_u16be_as_u32?()
|
||||
length = 1 + c32
|
||||
|
||||
while true {
|
||||
n_copied = args.dst.limited_copy_u32_from_reader!(up_to: length, r: args.src)
|
||||
this.lzma2_prior_decoded_length ~sat+= n_copied as base.u64
|
||||
this.stashed_pos ~sat+= n_copied as base.u64
|
||||
if length <= n_copied {
|
||||
break
|
||||
}
|
||||
@@ -437,25 +452,39 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work
|
||||
}
|
||||
|
||||
if header_byte >= 0xE0 {
|
||||
this.lzma2_prior_decoded_length = 0
|
||||
this.lzma2_need_dict_reset = false
|
||||
this.initialize_dict!()
|
||||
} else if this.prev_lzma2_chunk_was_uncompressed {
|
||||
// Update the stashed prev_byte and matched_byte. We've copied
|
||||
// an uncompressed chunk since the last compressed chunk.
|
||||
this.update_stashed_bytes?(dst: args.dst, workbuf: args.workbuf)
|
||||
}
|
||||
this.prev_lzma2_chunk_was_uncompressed = false
|
||||
|
||||
if this.lzma2_need_prob_reset or this.lzma2_need_properties {
|
||||
if this.lzma2_need_prob_reset or this.lzma2_need_properties or this.lzma2_need_dict_reset {
|
||||
return "#bad LZMA2 header"
|
||||
}
|
||||
}
|
||||
|
||||
c8 = args.src.read_u8?()
|
||||
if c8 <> 0x00 {
|
||||
return "#bad code"
|
||||
}
|
||||
this.stashed_bits = args.src.read_u32be?()
|
||||
if this.stashed_bits == 0xFFFF_FFFF {
|
||||
return "#bad code"
|
||||
}
|
||||
this.stashed_range = 0xFFFF_FFFF
|
||||
this.lzma2_encoded_length_have = 5
|
||||
|
||||
while args.workbuf.length() < ((this.dict_size as base.u64) + 273) {
|
||||
yield? base."$short workbuf"
|
||||
} endwhile
|
||||
|
||||
this.lzma2_encoded_length_have = 0
|
||||
while true {
|
||||
dmark = args.dst.mark()
|
||||
smark = args.src.mark()
|
||||
status =? this.decode_bitstream?(dst: args.dst, src: args.src, workbuf: args.workbuf)
|
||||
this.lzma2_encoded_length_have ~sat+= args.src.count_since(mark: smark)
|
||||
this.lzma2_prior_decoded_length ~sat+= args.dst.count_since(mark: dmark)
|
||||
if status.is_ok() {
|
||||
break
|
||||
}
|
||||
@@ -474,6 +503,54 @@ pri func decoder.decode_bitstream?(dst: base.io_writer, src: base.io_reader, wor
|
||||
this.decode_bitstream_slow?(dst: args.dst, src: args.src, workbuf: args.workbuf)
|
||||
}
|
||||
|
||||
pri func decoder.update_stashed_bytes?(dst: base.io_writer, workbuf: roslice base.u8) {
|
||||
var dist : base.u32
|
||||
var which : base.u32
|
||||
var adj_dist : base.u32
|
||||
var wb_index : base.u64
|
||||
|
||||
while args.dst.length() <= 0 {
|
||||
yield? base."$short write"
|
||||
} endwhile
|
||||
|
||||
dist = 1
|
||||
which = 0
|
||||
while which < 2 {
|
||||
if (dist as base.u64) <= args.dst.history_length() {
|
||||
args.dst.limited_copy_u32_from_history!(up_to: 1, distance: dist)
|
||||
if not args.dst.can_undo_byte() {
|
||||
return "#internal error: inconsistent dictionary state"
|
||||
}
|
||||
this.stashed_bytes[which] = args.dst.peek_undo_byte()
|
||||
args.dst.undo_byte!()
|
||||
|
||||
} else {
|
||||
adj_dist = (((dist as base.u64) - args.dst.history_length()) & 0xFFFF_FFFF) as base.u32
|
||||
wb_index = (this.dict_workbuf_index as base.u64) ~mod- (adj_dist as base.u64)
|
||||
while wb_index >= 0x8000_0000_0000_0000,
|
||||
inv which < 2,
|
||||
{
|
||||
wb_index ~mod+= this.dict_size as base.u64
|
||||
} endwhile
|
||||
if wb_index >= args.workbuf.length() {
|
||||
return "#internal error: inconsistent dictionary state"
|
||||
}
|
||||
this.stashed_bytes[which] = args.workbuf[wb_index]
|
||||
}
|
||||
|
||||
dist = 1 + this.stashed_rep0
|
||||
which += 1
|
||||
} endwhile
|
||||
}
|
||||
|
||||
pri func decoder.initialize_dict!() {
|
||||
this.dict_workbuf_index = 0
|
||||
this.dict_seen = 0
|
||||
this.stashed_bytes[0] = 0
|
||||
this.stashed_bytes[1] = 0
|
||||
this.stashed_pos = 0
|
||||
}
|
||||
|
||||
// initialize_probs sets all of the probs_foo[i][j] values to 1024.
|
||||
pri func decoder.initialize_probs!() {
|
||||
var i : base.u32
|
||||
@@ -610,6 +687,14 @@ pri func decoder.initialize_probs!() {
|
||||
} endwhile
|
||||
i += 1
|
||||
} endwhile
|
||||
|
||||
// Reset some other LZMA state too. They're not probabilities, but they
|
||||
// need resetting whenever the probabilities do.
|
||||
this.stashed_state = 0
|
||||
this.stashed_rep0 = 0
|
||||
this.stashed_rep1 = 0
|
||||
this.stashed_rep2 = 0
|
||||
this.stashed_rep3 = 0
|
||||
}
|
||||
|
||||
pri const STATE_TRANSITION_LITERAL : roarray[12] base.u8[..= 11] = [
|
||||
|
||||
Reference in new issue
Block a user