std/xz: prepare to decode concatenated streams

This commit is contained in:
Nigel Tao committed 2024-02-26 21:48:31 +11:00
1 parent f06858f395
commit 8ff5693d9d
2 files changed
+866 -860

No files matched your search

+246 -242
View File
@@ -153,282 +153,286 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work
var c8 : base.u8
var footer_magic : base.u16
header_magic = args.src.read_u48le_as_u64?()
if header_magic <> '\xFD\x37\x7A\x58\x5A\x00'le {
return "#bad header"
}
// Read the 2-byte flags and the 4-byte CRC-32 checksum of those 2 bytes.
header_magic = args.src.read_u48le_as_u64?()
if header_magic == '\x00\x00\xFF\x12\xD9\x41'le {
this.checksummer = 0 // None.
} else if header_magic == '\x00\x01\x69\x22\xDE\x36'le {
this.checksummer = 1 // CRC-32.
} else if header_magic == '\x00\x04\xE6\xD6\xB4\x46'le {
this.checksummer = 2 // CRC-64.
} else if header_magic == '\x00\x0A\xE1\xFB\x0C\xA1'le {
this.checksummer = 3 // SHA-256.
} else if (header_magic & 0xF0FF) <> 0 {
// Section 2.1.1.2. Stream Flags: "If any reserved bit is set, the
// decoder MUST indicate an error."
return "#bad header"
} else {
header_magic = 0xF & (header_magic >> 8)
if (header_magic <> 0x0) and (header_magic <> 0x1) and (header_magic <> 0x4) and (header_magic <> 0xA) {
return "#unsupported checksum algorithm"
while.streams true {
header_magic = args.src.read_u48le_as_u64?()
if header_magic <> '\xFD\x37\x7A\x58\x5A\x00'le {
return "#bad header"
}
return "#bad checksum"
}
this.flags = (header_magic & 0xFFFF) as base.u16
while true {
if args.src.length() <= 0 {
yield? base."$short read"
continue
} else if args.src.peek_u8() == 0x00 {
// We've hit the Index instead of a Block.
break
// Read the 2-byte flags and the 4-byte CRC-32 checksum of those 2 bytes.
header_magic = args.src.read_u48le_as_u64?()
if header_magic == '\x00\x00\xFF\x12\xD9\x41'le {
this.checksummer = 0 // None.
} else if header_magic == '\x00\x01\x69\x22\xDE\x36'le {
this.checksummer = 1 // CRC-32.
} else if header_magic == '\x00\x04\xE6\xD6\xB4\x46'le {
this.checksummer = 2 // CRC-64.
} else if header_magic == '\x00\x0A\xE1\xFB\x0C\xA1'le {
this.checksummer = 3 // SHA-256.
} else if (header_magic & 0xF0FF) <> 0 {
// Section 2.1.1.2. Stream Flags: "If any reserved bit is set, the
// decoder MUST indicate an error."
return "#bad header"
} else {
header_magic = 0xF & (header_magic >> 8)
if (header_magic <> 0x0) and (header_magic <> 0x1) and (header_magic <> 0x4) and (header_magic <> 0xA) {
return "#unsupported checksum algorithm"
}
return "#bad checksum"
}
this.num_actual_blocks ~mod+= 1
this.flags = (header_magic & 0xFFFF) as base.u16
while.blocks true {
if args.src.length() <= 0 {
yield? base."$short read"
continue.blocks
} else if args.src.peek_u8() == 0x00 {
// We've hit the Index instead of a Block.
break.blocks
}
this.num_actual_blocks ~mod+= 1
if not this.ignore_checksum {
this.crc32.reset!()
}
this.compressed_size_for_index = 4 // For the upcoming 4-byte CRC-32.
while true {
smark = args.src.mark()
status =? this.decode_block_header_with_padding?(src: args.src)
this.compressed_size_for_index ~sat+= args.src.count_since(mark: smark)
if not this.ignore_checksum {
checksum32_have = this.crc32.update_u32!(x: args.src.since(mark: smark))
}
if status.is_ok() {
break
}
yield? status
} endwhile
checksum32_want = args.src.read_u32le?()
if this.ignore_checksum {
// No-op.
} else if checksum32_have <> checksum32_want {
return "#bad checksum"
} else {
this.crc32.reset!()
this.crc64.reset!()
this.sha256.reset!()
}
compressed_size = 0
uncompressed_size = 0
while true {
if (this.bcj_undo_index as base.u64) > args.dst.length() {
yield? base."$short write"
continue
}
dmark = args.dst.mark()
smark = args.src.mark()
if this.num_non_final_filters == 0 {
status =? this.lzma.transform_io?(dst: args.dst, src: args.src, workbuf: args.workbuf)
} else {
if this.bcj_undo_index > 0 {
args.dst.copy_from_slice!(s: this.filter_data[0][.. this.bcj_undo_index])
this.bcj_undo_index = 0
}
// The non-final filters will modify the args.dst contents so
// that its history isn't what this.lzma considers its
// (unfiltered) history.
io_forget_history (io: args.dst) {
status =? this.lzma.transform_io?(dst: args.dst, src: args.src, workbuf: args.workbuf)
}
this.bcj_undo_index = this.apply_non_final_filters!(dst_slice: args.dst.since(mark: dmark))
if (this.bcj_undo_index > 0) and (not status.is_ok()) {
i8 = this.bcj_undo_index - 1
while true {
if not args.dst.can_undo_byte() {
return "#internal error: inconsistent BCJ filter state"
}
this.filter_data[0][i8] = args.dst.peek_undo_byte()
args.dst.undo_byte!()
if i8 <= 0 {
break
}
i8 -= 1
} endwhile
}
}
compressed_size ~mod+= args.src.count_since(mark: smark)
uncompressed_size ~mod+= args.dst.count_since(mark: dmark)
if this.ignore_checksum {
// No-op.
} else if this.checksummer == 1 {
this.crc32.update!(x: args.dst.since(mark: dmark))
} else if this.checksummer == 2 {
this.crc64.update!(x: args.dst.since(mark: dmark))
} else if this.checksummer == 3 {
this.sha256.update!(x: args.dst.since(mark: dmark))
}
if status.is_ok() {
break
}
yield? status
} endwhile
if (this.block_has_compressed_size and (this.block_compressed_size <> compressed_size)) or
(this.block_has_uncompressed_size and (this.block_uncompressed_size <> uncompressed_size)) {
return "#bad block header"
}
// A block's 'compressed size' (as measured here) needs to match what's
// listed in the index at the end of the XZ file. Per the XZ file
// format specification, a block is:
//
// +==============+=================+===============+=======+
// | Block Header | Compressed Data | Block Padding | Check |
// +==============+=================+===============+=======+
//
// The compressed_size variable only measures the "Compressed Data"
// portion. The 'compressed size for indexing purposes' needs to add
// the size of the Block Header and the Check (but not the Block
// Padding). this.compressed_size_for_index was initialized and
// accumulated above to, at this point in time, hold the size of the
// Block Header. We need to add the other two parts.
this.compressed_size_for_index ~sat+= compressed_size
this.compressed_size_for_index ~sat+= CHECKSUM_LENGTH[this.checksummer] as base.u64
// Update the verification_have_etc values. The hash function is
// loosely based on https://en.wikipedia.org/wiki/MurmurHash#Algorithm
this.verification_have_total_sizes[0] ~mod+= this.compressed_size_for_index
hash = ((this.compressed_size_for_index ^ (this.compressed_size_for_index >> 32)) & 0xFFFF_FFFF) as base.u32
hash ~mod*= 0xCC9E_2D51
hash = (hash ~mod<< 15) | (hash >> 17)
hash ~mod*= 0x1B87_3593
hash ^= this.verification_have_hashed_sizes[0]
hash = (hash ~mod<< 13) | (hash >> 19)
this.verification_have_hashed_sizes[0] = (hash ~mod* 5) ~mod+ 0xE654_6B64
this.verification_have_total_sizes[1] ~mod+= uncompressed_size
hash = ((uncompressed_size ^ (uncompressed_size >> 32)) & 0xFFFF_FFFF) as base.u32
hash ~mod*= 0xCC9E_2D51
hash = (hash ~mod<< 15) | (hash >> 17)
hash ~mod*= 0x1B87_3593
hash ^= this.verification_have_hashed_sizes[1]
hash = (hash ~mod<< 13) | (hash >> 19)
this.verification_have_hashed_sizes[1] = (hash ~mod* 5) ~mod+ 0xE654_6B64
// Consume the "Block Padding".
while (compressed_size & 3) <> 0 {
c8 = args.src.read_u8?()
if c8 <> 0x00 {
return "#bad padding"
}
compressed_size ~mod+= 1
} endwhile
this.lzma_needs_reset = true
if this.ignore_checksum {
args.src.skip_u32?(n: CHECKSUM_LENGTH[this.checksummer] as base.u32)
} else if this.checksummer == 1 {
checksum32_want = args.src.read_u32le?()
checksum32_have = this.crc32.checksum_u32()
if checksum32_have <> checksum32_want {
return "#bad checksum"
}
} else if this.checksummer == 2 {
checksum64_want = args.src.read_u64le?()
checksum64_have = this.crc64.checksum_u64()
if checksum64_have <> checksum64_want {
return "#bad checksum"
}
} else if this.checksummer == 3 {
checksum256_have = this.sha256.checksum_bitvec256()
checksum64_want = args.src.read_u64be?()
if checksum256_have.get_u64(i: 3) <> checksum64_want {
return "#bad checksum"
}
checksum64_want = args.src.read_u64be?()
if checksum256_have.get_u64(i: 2) <> checksum64_want {
return "#bad checksum"
}
checksum64_want = args.src.read_u64be?()
if checksum256_have.get_u64(i: 1) <> checksum64_want {
return "#bad checksum"
}
checksum64_want = args.src.read_u64be?()
if checksum256_have.get_u64(i: 0) <> checksum64_want {
return "#bad checksum"
}
}
} endwhile.blocks
// Verify the index.
this.backwards_size = 0
if not this.ignore_checksum {
this.crc32.reset!()
}
this.compressed_size_for_index = 4 // For the upcoming 4-byte CRC-32.
while true {
smark = args.src.mark()
status =? this.decode_block_header_with_padding?(src: args.src)
this.compressed_size_for_index ~sat+= args.src.count_since(mark: smark)
status =? this.verify_index?(src: args.src)
this.backwards_size ~sat+= args.src.count_since(mark: smark)
if not this.ignore_checksum {
checksum32_have = this.crc32.update_u32!(x: args.src.since(mark: smark))
this.crc32.update!(x: args.src.since(mark: smark))
}
if status.is_ok() {
break
}
yield? status
} endwhile
if not this.ignore_checksum {
this.crc32.update!(x: ZEROES[.. 3 & (0 ~mod- (3 & this.backwards_size))])
}
while (this.backwards_size & 3) <> 0 {
c8 = args.src.read_u8?()
if c8 <> 0x00 {
return "#bad index"
}
this.backwards_size ~mod+= 1
} endwhile
this.backwards_size >>= 2
if (this.backwards_size == 0) or (this.backwards_size > 0xFFFF_FFFF) {
return "#bad index"
}
checksum32_want = args.src.read_u32le?()
if this.ignore_checksum {
// No-op.
} else if checksum32_have <> checksum32_want {
} else if checksum32_want <> this.crc32.checksum_u32() {
return "#bad checksum"
} else {
this.crc32.reset!()
this.crc64.reset!()
this.sha256.reset!()
}
compressed_size = 0
uncompressed_size = 0
// Verify the footer.
checksum32_want = args.src.read_u32le?()
while true {
if (this.bcj_undo_index as base.u64) > args.dst.length() {
yield? base."$short write"
continue
}
dmark = args.dst.mark()
smark = args.src.mark()
if this.num_non_final_filters == 0 {
status =? this.lzma.transform_io?(dst: args.dst, src: args.src, workbuf: args.workbuf)
} else {
if this.bcj_undo_index > 0 {
args.dst.copy_from_slice!(s: this.filter_data[0][.. this.bcj_undo_index])
this.bcj_undo_index = 0
}
// The non-final filters will modify the args.dst contents so
// that its history isn't what this.lzma considers its
// (unfiltered) history.
io_forget_history (io: args.dst) {
status =? this.lzma.transform_io?(dst: args.dst, src: args.src, workbuf: args.workbuf)
}
this.bcj_undo_index = this.apply_non_final_filters!(dst_slice: args.dst.since(mark: dmark))
if (this.bcj_undo_index > 0) and (not status.is_ok()) {
i8 = this.bcj_undo_index - 1
while true {
if not args.dst.can_undo_byte() {
return "#internal error: inconsistent BCJ filter state"
}
this.filter_data[0][i8] = args.dst.peek_undo_byte()
args.dst.undo_byte!()
if i8 <= 0 {
break
}
i8 -= 1
} endwhile
}
}
compressed_size ~mod+= args.src.count_since(mark: smark)
uncompressed_size ~mod+= args.dst.count_since(mark: dmark)
if this.ignore_checksum {
// No-op.
} else if this.checksummer == 1 {
this.crc32.update!(x: args.dst.since(mark: dmark))
} else if this.checksummer == 2 {
this.crc64.update!(x: args.dst.since(mark: dmark))
} else if this.checksummer == 3 {
this.sha256.update!(x: args.dst.since(mark: dmark))
status =? this.verify_footer?(src: args.src)
if not this.ignore_checksum {
this.crc32.update!(x: args.src.since(mark: smark))
}
if status.is_ok() {
break
}
yield? status
} endwhile
if (this.block_has_compressed_size and (this.block_compressed_size <> compressed_size)) or
(this.block_has_uncompressed_size and (this.block_uncompressed_size <> uncompressed_size)) {
return "#bad block header"
if (not this.ignore_checksum) and (checksum32_want <> this.crc32.checksum_u32()) {
return "#bad checksum"
}
footer_magic = args.src.read_u16le?()
if footer_magic <> '\x59\x5A'le {
return "#bad footer"
}
// A block's 'compressed size' (as measured here) needs to match what's
// listed in the index at the end of the XZ file. Per the XZ file
// format specification, a block is:
//
// +==============+=================+===============+=======+
// | Block Header | Compressed Data | Block Padding | Check |
// +==============+=================+===============+=======+
//
// The compressed_size variable only measures the "Compressed Data"
// portion. The 'compressed size for indexing purposes' needs to add
// the size of the Block Header and the Check (but not the Block
// Padding). this.compressed_size_for_index was initialized and
// accumulated above to, at this point in time, hold the size of the
// Block Header. We need to add the other two parts.
this.compressed_size_for_index ~sat+= compressed_size
this.compressed_size_for_index ~sat+= CHECKSUM_LENGTH[this.checksummer] as base.u64
// Update the verification_have_etc values. The hash function is
// loosely based on https://en.wikipedia.org/wiki/MurmurHash#Algorithm
this.verification_have_total_sizes[0] ~mod+= this.compressed_size_for_index
hash = ((this.compressed_size_for_index ^ (this.compressed_size_for_index >> 32)) & 0xFFFF_FFFF) as base.u32
hash ~mod*= 0xCC9E_2D51
hash = (hash ~mod<< 15) | (hash >> 17)
hash ~mod*= 0x1B87_3593
hash ^= this.verification_have_hashed_sizes[0]
hash = (hash ~mod<< 13) | (hash >> 19)
this.verification_have_hashed_sizes[0] = (hash ~mod* 5) ~mod+ 0xE654_6B64
this.verification_have_total_sizes[1] ~mod+= uncompressed_size
hash = ((uncompressed_size ^ (uncompressed_size >> 32)) & 0xFFFF_FFFF) as base.u32
hash ~mod*= 0xCC9E_2D51
hash = (hash ~mod<< 15) | (hash >> 17)
hash ~mod*= 0x1B87_3593
hash ^= this.verification_have_hashed_sizes[1]
hash = (hash ~mod<< 13) | (hash >> 19)
this.verification_have_hashed_sizes[1] = (hash ~mod* 5) ~mod+ 0xE654_6B64
// Consume the "Block Padding".
while (compressed_size & 3) <> 0 {
c8 = args.src.read_u8?()
if c8 <> 0x00 {
return "#bad padding"
}
compressed_size ~mod+= 1
} endwhile
this.lzma_needs_reset = true
if this.ignore_checksum {
args.src.skip_u32?(n: CHECKSUM_LENGTH[this.checksummer] as base.u32)
} else if this.checksummer == 1 {
checksum32_want = args.src.read_u32le?()
checksum32_have = this.crc32.checksum_u32()
if checksum32_have <> checksum32_want {
return "#bad checksum"
}
} else if this.checksummer == 2 {
checksum64_want = args.src.read_u64le?()
checksum64_have = this.crc64.checksum_u64()
if checksum64_have <> checksum64_want {
return "#bad checksum"
}
} else if this.checksummer == 3 {
checksum256_have = this.sha256.checksum_bitvec256()
checksum64_want = args.src.read_u64be?()
if checksum256_have.get_u64(i: 3) <> checksum64_want {
return "#bad checksum"
}
checksum64_want = args.src.read_u64be?()
if checksum256_have.get_u64(i: 2) <> checksum64_want {
return "#bad checksum"
}
checksum64_want = args.src.read_u64be?()
if checksum256_have.get_u64(i: 1) <> checksum64_want {
return "#bad checksum"
}
checksum64_want = args.src.read_u64be?()
if checksum256_have.get_u64(i: 0) <> checksum64_want {
return "#bad checksum"
}
}
} endwhile
// Verify the index.
this.backwards_size = 0
if not this.ignore_checksum {
this.crc32.reset!()
}
while true {
smark = args.src.mark()
status =? this.verify_index?(src: args.src)
this.backwards_size ~sat+= args.src.count_since(mark: smark)
if not this.ignore_checksum {
this.crc32.update!(x: args.src.since(mark: smark))
}
if status.is_ok() {
break
}
yield? status
} endwhile
if not this.ignore_checksum {
this.crc32.update!(x: ZEROES[.. 3 & (0 ~mod- (3 & this.backwards_size))])
}
while (this.backwards_size & 3) <> 0 {
c8 = args.src.read_u8?()
if c8 <> 0x00 {
return "#bad index"
}
this.backwards_size ~mod+= 1
} endwhile
this.backwards_size >>= 2
if (this.backwards_size == 0) or (this.backwards_size > 0xFFFF_FFFF) {
return "#bad index"
}
checksum32_want = args.src.read_u32le?()
if this.ignore_checksum {
// No-op.
} else if checksum32_want <> this.crc32.checksum_u32() {
return "#bad checksum"
} else {
this.crc32.reset!()
}
// Verify the footer.
checksum32_want = args.src.read_u32le?()
while true {
smark = args.src.mark()
status =? this.verify_footer?(src: args.src)
if not this.ignore_checksum {
this.crc32.update!(x: args.src.since(mark: smark))
}
if status.is_ok() {
break
}
yield? status
} endwhile
if (not this.ignore_checksum) and (checksum32_want <> this.crc32.checksum_u32()) {
return "#bad checksum"
}
footer_magic = args.src.read_u16le?()
if footer_magic <> '\x59\x5A'le {
return "#bad footer"
}
break.streams
} endwhile.streams
}
pri func decoder.decode_block_header_with_padding?(src: base.io_reader) {