mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-17 16:32:11 +01:00
SPDX is an ISO standard for describing licenses in a computer readable format, which is useful when applying license compliance tools and the like to source code. In addition to ensuring that the Wuffs C distribution is properly annotated to the benefit of projects using Wuffs and SPDX-supporting tools, add it everywhere else where there's a license in this repo for consistency, too.
776 lines
26 KiB
Go
776 lines
26 KiB
Go
// Copyright 2022 The Wuffs Authors.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
|
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
|
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
|
|
// option. This file may not be copied, modified, or distributed
|
|
// except according to those terms.
|
|
//
|
|
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
|
|
|
// ----------------
|
|
|
|
// Package litonlylzma provides a decoder and encoder for Literal Only LZMA (or
|
|
// Literal Only Xz), a subset of the LZMA (or Xz) compressed file formats.
|
|
//
|
|
// Subset means that:
|
|
// - the Encode methods' output is a valid LZMA/Xz file and can be
|
|
// decompressed by the tools available at https://www.7-zip.org/sdk.html or
|
|
// https://tukaani.org/xz/ (and available as /usr/bin/lzma or /usr/bin/xz
|
|
// on a Debian system).
|
|
// - the Decode methods supports the Encode methods' output but they do not
|
|
// support the full set of valid LZMA/Xz files.
|
|
// - this codec's compression ratios are not as good as full LZMA/Xz
|
|
// (although compression times are much faster). Moderate compression is
|
|
// still achieved, through range coding, but there are no Lempel Ziv
|
|
// back-references.
|
|
//
|
|
// The main benefit, compared to full LZMA/Xz, is that this implementation is
|
|
// much simpler and hence easier to study. It is around 800 lines of code.
|
|
//
|
|
// Example compression numbers on a small English text file (at
|
|
// https://github.com/google/wuffs/blob/main/test/data/romeo.txt):
|
|
// - romeo.txt is 942 bytes (100%).
|
|
// - romeo.txt.litonlyxz is 708 bytes (75%).
|
|
// - romeo.txt.litonlylzma is 659 bytes (70%).
|
|
// - romeo.txt.xz is 644 bytes (68%).
|
|
// - romeo.txt.lzma is 598 bytes (63%).
|
|
// - romeo.txt.bz2 is 568 bytes (60%).
|
|
// - romeo.txt.zst is 559 bytes (59%).
|
|
// - romeo.txt.gz is 558 bytes (59%).
|
|
//
|
|
// Example compression numbers on a large archive of source code (at
|
|
// https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.0.1.tar.xz):
|
|
// - linux-5.0.1.tar is 863313920 bytes (100%).
|
|
// - linux-5.0.1.tar.litonlyxz is 454480932 bytes (53%).
|
|
// - linux-5.0.1.tar.litonlylzma is 449726070 bytes (52%).
|
|
// - linux-5.0.1.tar.gz is 164575959 bytes (19%).
|
|
// - linux-5.0.1.tar.zst is 156959897 bytes (18%).
|
|
// - linux-5.0.1.tar.bz2 is 125873134 bytes (15%).
|
|
// - linux-5.0.1.tar.xz is 108233572 bytes (13%).
|
|
// - linux-5.0.1.tar.lzma is 108216601 bytes (13%).
|
|
//
|
|
// The various tools (/usr/bin/gzip, /usr/bin/xz, etc) were all ran with their
|
|
// default settings, not their "maximum compression" settings. This is why the
|
|
// linux-5.0.1.tar.xz file size above differs from what cdn.kernel.org served.
|
|
package litonlylzma
|
|
|
|
import (
|
|
"errors"
|
|
"hash/crc32"
|
|
)
|
|
|
|
var (
|
|
// ErrUnsupportedXxxData is potentially returned by Decode.
|
|
ErrUnsupportedLZMAData = errors.New("litonlylzma: unsupported LZMA data")
|
|
ErrUnsupportedXzData = errors.New("litonlylzma: unsupported Xz data")
|
|
|
|
errInvalidLZMAData = errors.New("litonlylzma: invalid LZMA data")
|
|
errInvalidXzData = errors.New("litonlylzma: invalid Xz data")
|
|
errUnexpectedEOF = errors.New("litonlylzma: unexpected EOF")
|
|
errUnsupportedFileFormat = errors.New("litonlylzma: unsupported FileFormat")
|
|
)
|
|
|
|
const (
|
|
// LZMA has a few configuration knobs but our subset-of-LZMA hard codes
|
|
// these to LZMA's default values:
|
|
// https://github.com/jljusten/LZMA-SDK/blob/781863cdf592da3e97420f50de5dac056ad352a5/C/LzmaEnc.h#L19-L21
|
|
// https://github.com/jljusten/LZMA-SDK/blob/781863cdf592da3e97420f50de5dac056ad352a5/DOC/lzma-specification.txt#L43-L45
|
|
lc = 3 // The number of Literal Context bits.
|
|
lp = 0 // The number of Literal Position bits.
|
|
pb = 2 // The number of Position Bits.
|
|
|
|
lpMask = (1 << lp) - 1
|
|
pbMask = (1 << pb) - 1
|
|
)
|
|
|
|
// lzmaHeader5 is the first 5 bytes of an LZMA file using the hard-coded
|
|
// configuration for the subset-of-LZMA that this package supports.
|
|
//
|
|
// 0x5D encodes the (lc, lp, pb) triple and the next four bytes hold the u32le
|
|
// dictionary size. 0x00001000 is LZMA's minimum dictionary size, although our
|
|
// subset-of-LZMA does not use a dictionary (also called a "sliding window").
|
|
const lzmaHeader5 = "\x5D\x00\x10\x00\x00"
|
|
|
|
// xzHeader24 is the first 24 bytes of an Xz file using the hard-coded
|
|
// configuration for the subset-of-Xz that this package supports.
|
|
const xzHeader24 = "" +
|
|
"\xFD\x37\x7A\x58\x5A\x00" + // Stream Header Magic Bytes.
|
|
// ----
|
|
"\x00\x01" + // Stream Flags (0x01 means CRC-32).
|
|
"\x69\x22\xDE\x36" + // CRC-32 of the previous 2 bytes.
|
|
// ----
|
|
"\x02" + // Block Header Size (0x02 * 4 = 8 bytes).
|
|
"\x00" + // Block Flags.
|
|
"\x21" + // Filter ID (0x21 means LZMA2).
|
|
"\x01" + // Size of Filter Properties (size of the next line).
|
|
"\x00" + // Filter Properties (0x00 means an 0x1000 dictionary size).
|
|
"\x00\x00\x00" + // Padding to 4-byte alignment.
|
|
"\x37\x27\x97\xD6" // CRC-32 of the previous 8 bytes.
|
|
|
|
// rangeDecoder and rangeEncoder hold the state for range coding (also known as
|
|
// arithmetic coding, roughly speaking). "range" is a keyword in Go, so in this
|
|
// implementation, what's traditionally called the "range" fields are called
|
|
// "width" instead.
|
|
//
|
|
// Conceptually, the encoded bitstream holds a high precision number (a binary
|
|
// fraction) in the range from zero to one. High precision could mean thousands
|
|
// or millions of bits, but when decoding, only 32 of these are 'paged in' at a
|
|
// time - this is the rangeDecoder.bits field. The encoder similarly works on
|
|
// only a small, fixed number of bits.
|
|
//
|
|
// When encoding, that high precision *number* isn't fully known until the end
|
|
// of the input is reached. *During* the encoding process, what's known is a
|
|
// high precision *interval* that gets narrowed down as the input bytes are
|
|
// consumed. That interval is represented by the rangeEncoder.low and
|
|
// rangeEncoder.width fields (the width is the difference between the
|
|
// interval's upper and lower bound). Both low and width are, conceptually,
|
|
// 32-bit values - they operate at the same scale - but low is an uint64
|
|
// because of potential overflow. We'll revisit that below.
|
|
//
|
|
// Width is initialized to 0xFFFF_FFFF and shrinks every time a bit is decoded
|
|
// or encoded. Similarly, low can only increase or stay the same on each bit
|
|
// step. When width drops below (1 << 24), the rangeDecoder or rangeEncoder
|
|
// 'zooms in', left-shifting (with truncation) by 8 the bits, low and width
|
|
// fields and reading a src byte or writing a dst byte of compressed data.
|
|
//
|
|
// Writing a byte may be delayed because the encoding interval could span
|
|
// multiple possible digits. To use a decimal fraction analogy, suppose that
|
|
// our interval is [0.7138872, 0.7146055] and the low field nominally holds 4
|
|
// digits. We could emit '7' and '1' but we have to hold back on emitting
|
|
// either '3' or '4' until we can narrow down the interval further. That '3'
|
|
// digit is stashed as the pendingHead field. low is set to 8872, width is set
|
|
// to (46055 - 38872) = 7183 and we progress until the width drops below 1000,
|
|
// causing a 'zoom in' event. There are three cases:
|
|
// - if low is less than 9000 (but starts with an '8'), we can emit the '3'
|
|
// with confidence (and then stash '8' in the pendingHead field).
|
|
// - if low overflows 9999, we can emit the '4' with confidence (and then
|
|
// stash low's 'thousands digit' in the pendingHead field).
|
|
// - if low is in 9000 ..= 9999 then we could still be unsure. Keep the '3'
|
|
// pending but extend it so that there are two pending digits: '39'. If
|
|
// this occurs again: '399', and so on.
|
|
//
|
|
// In all three cases, 'zooming in' causes us to shift out the most significant
|
|
// digit of the low field. This method is therefore named shiftLow here, the
|
|
// same name used in other range coding implementations, such as
|
|
// https://github.com/jljusten/LZMA-SDK/blob/781863cdf592da3e97420f50de5dac056ad352a5/C/LzmaEnc.c#L572-L602
|
|
//
|
|
// The pendingEtc fields combine to hold one or more digits ('0' ..= '9'
|
|
// decimal digits in our analogy, 0x00 ..= 0xFF bytes in our actual
|
|
// implementation). There are (1 + pendingExtra) digits in total and while the
|
|
// head digit might be any value, the extra digits must be all '9' (in the
|
|
// decimal analogy) or 0xFF (in our actual implementation).
|
|
//
|
|
// Some other implementations use the name "cache" instead of "pending", and
|
|
// store a "cacheSize" field equivalent to our (1 + pendingExtra).
|
|
//
|
|
// As above, the low field is conceptually a uint32, but it is a uint64 because
|
|
// of this potential overflow. Its high 32 bits are usually zero, rarely one
|
|
// and never anything higher. The high 32 bits essentially hold an overflow
|
|
// carry bit. If set, it flips "3999etc" to "4000etc".
|
|
//
|
|
// "Low is conceptually 4 bytes; pendingHead holds another byte" is why the
|
|
// shortest LZMA payload is 5 bytes even though the first byte is always 0x00
|
|
// and seems redundant.
|
|
|
|
type rangeDecoder struct {
|
|
src []byte
|
|
bits uint32
|
|
width uint32
|
|
}
|
|
|
|
type rangeEncoder struct {
|
|
dst []byte
|
|
low uint64
|
|
width uint32
|
|
pendingHead uint8
|
|
pendingExtra uint64
|
|
}
|
|
|
|
// Precondition: (rEnc.width < (1 << 24)) or we are at the end of our input
|
|
// (and are flushing the rangeEncoder).
|
|
func (rEnc *rangeEncoder) shiftLow() {
|
|
if rEnc.low < 0x0_FF00_0000 {
|
|
rEnc.dst = append(rEnc.dst, rEnc.pendingHead+0x00)
|
|
for ; rEnc.pendingExtra > 0; rEnc.pendingExtra-- {
|
|
rEnc.dst = append(rEnc.dst, 0xFF)
|
|
}
|
|
rEnc.pendingHead = uint8(rEnc.low >> 24)
|
|
rEnc.pendingExtra = 0
|
|
rEnc.low = (rEnc.low << 8) & 0xFFFF_FFFF
|
|
|
|
} else if rEnc.low < 0x1_0000_0000 {
|
|
rEnc.pendingExtra++
|
|
rEnc.low = (rEnc.low << 8) & 0xFFFF_FFFF
|
|
|
|
} else {
|
|
rEnc.dst = append(rEnc.dst, rEnc.pendingHead+0x01)
|
|
for ; rEnc.pendingExtra > 0; rEnc.pendingExtra-- {
|
|
rEnc.dst = append(rEnc.dst, 0x00)
|
|
}
|
|
rEnc.pendingHead = uint8(rEnc.low >> 24)
|
|
rEnc.pendingExtra = 0
|
|
rEnc.low = (rEnc.low << 8) & 0xFFFF_FFFF
|
|
}
|
|
}
|
|
|
|
// prob is a probability (out of 100%) whose uint16 value ranges from 0 to
|
|
// (1 << 11) = 2048 inclusive. For example, prob(1024) means a 50% probability
|
|
// and prob(384) means a (384 / 2048) = 18.75% probability.
|
|
//
|
|
// Specifically, it is the probability that the next bit is 0 (instead of 1).
|
|
type prob uint16
|
|
|
|
const (
|
|
// probBits is such that prob(1 << probBits) means a 100% probability.
|
|
probBits = 11
|
|
minProb = 0
|
|
maxProb = 1 << probBits
|
|
)
|
|
|
|
func setProbsToOneHalf(p []prob) {
|
|
for i := range p {
|
|
p[i] = 1 << (probBits - 1)
|
|
}
|
|
}
|
|
|
|
// adaptShift scales how much to adjust "the probability that the next bit is
|
|
// 0" after seeing the current bit. The probability is adaptive in that the
|
|
// current bit being 0 (or 1) increases (or decreases) the probability that the
|
|
// next bit is 0. The larger adaptShift is, the slower the adaptation.
|
|
const adaptShift = 5
|
|
|
|
func (p *prob) decodeBit(rDec *rangeDecoder) (bitValue uint32, retErr error) {
|
|
threshold := (rDec.width >> probBits) * uint32(*p)
|
|
if rDec.bits < threshold {
|
|
bitValue = 0
|
|
rDec.width = threshold
|
|
*p += (maxProb - *p) >> adaptShift
|
|
} else {
|
|
bitValue = 1
|
|
rDec.bits -= threshold
|
|
rDec.width -= threshold
|
|
*p -= (*p - minProb) >> adaptShift
|
|
}
|
|
|
|
if rDec.width < (1 << 24) {
|
|
if len(rDec.src) <= 0 {
|
|
return 0, errUnexpectedEOF
|
|
}
|
|
rDec.bits = rDec.bits<<8 | uint32(rDec.src[0])
|
|
rDec.width <<= 8
|
|
rDec.src = rDec.src[1:]
|
|
}
|
|
return bitValue, retErr
|
|
}
|
|
|
|
func (p *prob) encodeBit(rEnc *rangeEncoder, bitValue uint32) {
|
|
threshold := (rEnc.width >> probBits) * uint32(*p)
|
|
if bitValue == 0 {
|
|
rEnc.width = threshold
|
|
*p += (maxProb - *p) >> adaptShift
|
|
} else {
|
|
rEnc.low += uint64(threshold)
|
|
rEnc.width -= threshold
|
|
*p -= (*p - minProb) >> adaptShift
|
|
}
|
|
if rEnc.width < (1 << 24) {
|
|
rEnc.width <<= 8
|
|
rEnc.shiftLow()
|
|
}
|
|
}
|
|
|
|
// byteProbs hold probabilities for coding 8 bits (1 byte) of data. It is a 256
|
|
// element array such that:
|
|
// - The 0th element is unused.
|
|
// - The 1st element holds the probability that the 7th bit (the highest, most
|
|
// significant bit) is 0.
|
|
// - The 2nd element holds the probability that the 6th bit is 0, conditional
|
|
// on the high bit being 0.
|
|
// - The 3rd element holds the probability that the 6th bit is 0, conditional
|
|
// on the high bit being 1.
|
|
// - The 4th element holds the probability that the 5th bit is 0, conditional
|
|
// on the high two bits being 00.
|
|
// - The 5th element holds the probability that the 5th bit is 0, conditional
|
|
// on the high two bits being 01.
|
|
// - The 6th element holds the probability that the 5th bit is 0, conditional
|
|
// on the high two bits being 10.
|
|
// - The 7th element holds the probability that the 5th bit is 0, conditional
|
|
// on the high two bits being 11.
|
|
// - The 8th element holds the probability that the 4th bit is 0, conditional
|
|
// on the high three bits being 000.
|
|
// - etc
|
|
// - The 255th element holds the probability that the 0th bit (the lowest,
|
|
// least significant bit) is 0, conditional on the high seven bits being
|
|
// 1111111.
|
|
//
|
|
// Put another way, the 256 elements' value of N, as in "it's a probability for
|
|
// the Nth bit", looks like this (when arranged in 8 rows of 32 elements):
|
|
//
|
|
// u7665555444444443333333333333333
|
|
// 22222222222222222222222222222222
|
|
// 11111111111111111111111111111111
|
|
// 11111111111111111111111111111111
|
|
// 00000000000000000000000000000000
|
|
// 00000000000000000000000000000000
|
|
// 00000000000000000000000000000000
|
|
// 00000000000000000000000000000000
|
|
//
|
|
// The 'u' means that the 0th element is unused.
|
|
type byteProbs [0x100]prob
|
|
|
|
func (p *byteProbs) decodeByte(rDec *rangeDecoder) (byteValue byte, retErr error) {
|
|
index := uint32(1)
|
|
for index < 0x100 {
|
|
if bitValue, err := p[index].decodeBit(rDec); err != nil {
|
|
return 0, err
|
|
} else {
|
|
index = index<<1 | bitValue
|
|
}
|
|
}
|
|
return byte(index), nil
|
|
}
|
|
|
|
func (p *byteProbs) encodeByte(rEnc *rangeEncoder, byteValue byte) {
|
|
b := uint32(byteValue)
|
|
index := uint32(1)
|
|
for i := 7; i >= 0; i-- {
|
|
bitValue := (b >> uint(i)) & 1
|
|
p[index].encodeBit(rEnc, bitValue)
|
|
index = index<<1 | bitValue
|
|
}
|
|
}
|
|
|
|
// FileFormat is a compressed file format, either the original LZMA format
|
|
// itself or something like Xz that builds on or varies that.
|
|
//
|
|
// Each valid FileFormatXxx value does not distinguish between the full file
|
|
// format (as spoken by numerous Xxx software tools) and the subset-of-Xxx
|
|
// implemented by this package. Specifically, FileFormatLZMA.Encode will
|
|
// produce compressed data that is valid full-LZMA (and is also valid
|
|
// subset-of-LZMA). FileFormatLZMA.Decode can decode subset-of-LZMA (but not
|
|
// full-LZMA).
|
|
type FileFormat uint32
|
|
|
|
const (
|
|
FileFormatInvalid = FileFormat(0)
|
|
FileFormatLZMA = FileFormat(1)
|
|
FileFormatXz = FileFormat(2)
|
|
)
|
|
|
|
func (f FileFormat) String() string {
|
|
switch f {
|
|
case FileFormatLZMA:
|
|
return "FileFormatLZMA"
|
|
case FileFormatXz:
|
|
return "FileFormatXz"
|
|
}
|
|
return "FileFormatInvalid"
|
|
}
|
|
|
|
// Decode converts src from the Literal Only LZMA/Xz compressed file format,
|
|
// appending the decoding to dst.
|
|
//
|
|
// It is valid to pass a nil dst, like the way it's valid to pass nil to the
|
|
// built-in append function.
|
|
//
|
|
// Decode also returns the suffix of src that was not used during decoding.
|
|
//
|
|
// It can also return partial results. The appendedDst and remainingSrc return
|
|
// values may be non-zero even if retErr is non-zero.
|
|
//
|
|
// It returns ErrUnsupportedXxxData if the source bytes look like Xxx formatted
|
|
// data that is outside the subset-of-Xxx that this package implements.
|
|
func (f FileFormat) Decode(dst []byte, src []byte) (appendedDst []byte, remainingSrc []byte, retErr error) {
|
|
switch f {
|
|
case FileFormatLZMA:
|
|
return decodeLZMA(dst, src)
|
|
case FileFormatXz:
|
|
return decodeXz(dst, src)
|
|
}
|
|
return nil, nil, errUnsupportedFileFormat
|
|
}
|
|
|
|
func decodeLZMA(dst []byte, src []byte) (appendedDst []byte, remainingSrc []byte, retErr error) {
|
|
// LZMA consists of a 13 byte header and then at least a 5 byte payload
|
|
// (range coded data). The 13 byte header is:
|
|
// - 1 byte for the (lc, lp, pb) triple with max values (8, 4, 4),
|
|
// - 4 bytes u32le dictionary size and
|
|
// - 8 bytes i64le uncompressed size.
|
|
if (len(src) < 18) || (src[0] >= (9 * 5 * 5)) {
|
|
return dst, nil, errInvalidLZMAData
|
|
} else if string(src[:5]) != lzmaHeader5 {
|
|
return dst, nil, ErrUnsupportedLZMAData
|
|
}
|
|
|
|
size := uint64(0)
|
|
for i := 0; i < 8; i++ {
|
|
size |= uint64(src[5+i]) << uint(8*i)
|
|
}
|
|
if int64(size) < -1 {
|
|
return dst, nil, errInvalidLZMAData
|
|
} else if int64(size) == -1 {
|
|
return dst, nil, ErrUnsupportedLZMAData
|
|
}
|
|
|
|
return decodeRaw(dst, src[13:], size, ErrUnsupportedLZMAData)
|
|
}
|
|
|
|
func decodeXz(dst []byte, src []byte) (appendedDst []byte, remainingSrc []byte, retErr error) {
|
|
originalDstLen := len(dst)
|
|
originalSrcLen := len(src)
|
|
|
|
// Decode the headers.
|
|
if (len(src) < 24) || (string(src[:6]) != xzHeader24[:6]) {
|
|
return dst, src, errInvalidXzData
|
|
} else if string(src[6:24]) != xzHeader24[6:24] {
|
|
return dst, src, ErrUnsupportedXzData
|
|
}
|
|
src = src[24:]
|
|
|
|
// Decode the payload as multiple chunks.
|
|
for {
|
|
if len(src) == 0 {
|
|
return dst, src, errInvalidXzData
|
|
|
|
} else if src[0] == 0x00 { // No more chunks.
|
|
src = src[1:]
|
|
break
|
|
|
|
} else if src[0] == 0x01 { // Independent uncompressed chunk.
|
|
if len(src) < 3 {
|
|
return dst, src, errInvalidXzData
|
|
}
|
|
uncompressedSize := (uint32(src[1]) << 8) + (uint32(src[2]) << 0) + 1
|
|
src = src[3:]
|
|
if uint64(uncompressedSize) > uint64(len(src)) {
|
|
return dst, src, errInvalidXzData
|
|
}
|
|
dst = append(dst, src[:uncompressedSize]...)
|
|
src = src[uncompressedSize:]
|
|
|
|
} else if src[0] == 0xE0 { // Independent LZMA chunk.
|
|
if len(src) < 6 {
|
|
return dst, src, errInvalidXzData
|
|
} else if src[5] != 0x5D { // Like lzmaHeader5[0], this encodes lc=3, lp=0, pb=2.
|
|
return dst, src, ErrUnsupportedXzData
|
|
}
|
|
uncompressedSize := (uint32(src[1]) << 8) + (uint32(src[2]) << 0) + 1
|
|
compressedSize := (uint32(src[3]) << 8) + (uint32(src[4]) << 0) + 1
|
|
src = src[6:]
|
|
if uint64(compressedSize) > uint64(len(src)) {
|
|
return dst, src, errInvalidXzData
|
|
}
|
|
lenSrc := len(src)
|
|
dst, src, retErr = decodeRaw(dst, src, uint64(uncompressedSize), ErrUnsupportedXzData)
|
|
if retErr != nil {
|
|
return dst, src, retErr
|
|
} else if lenSrc != (len(src) + int(compressedSize)) {
|
|
return dst, src, errInvalidXzData
|
|
}
|
|
|
|
} else {
|
|
return dst, src, ErrUnsupportedXzData
|
|
}
|
|
}
|
|
unpaddedSizeWant := uint64((originalSrcLen - 12) - len(src) + 4)
|
|
for i := unpaddedSizeWant & 3; (i & 3) != 0; i++ {
|
|
if (len(src) == 0) || (src[0] != 0x00) {
|
|
return dst, src, errInvalidXzData
|
|
}
|
|
src = src[1:]
|
|
}
|
|
if len(src) < 4 {
|
|
return dst, src, errInvalidXzData
|
|
}
|
|
checksum := crc32.ChecksumIEEE(dst[originalDstLen:])
|
|
if (src[0] != uint8(checksum>>0)) ||
|
|
(src[1] != uint8(checksum>>8)) ||
|
|
(src[2] != uint8(checksum>>16)) ||
|
|
(src[3] != uint8(checksum>>24)) {
|
|
return dst, src, errInvalidXzData
|
|
}
|
|
src = src[4:]
|
|
|
|
// Decode the index.
|
|
srcCheckpoint1 := src
|
|
if (len(src) < 2) || (src[0] != 0x00) || (src[1] != 0x01) {
|
|
return dst, src, errInvalidXzData
|
|
}
|
|
src = src[2:]
|
|
src, unpaddedSizeHave, ok := decodeUvarint(src)
|
|
if !ok || (unpaddedSizeHave != unpaddedSizeWant) {
|
|
return dst, src, errInvalidXzData
|
|
}
|
|
src, uncompressedSize, ok := decodeUvarint(src)
|
|
if !ok || (uncompressedSize != uint64(len(dst)-originalDstLen)) {
|
|
return dst, src, errInvalidXzData
|
|
}
|
|
for i := (len(srcCheckpoint1) - len(src)) & 3; (i & 3) != 0; i++ {
|
|
if (len(src) == 0) || (src[0] != 0x00) {
|
|
return dst, src, errInvalidXzData
|
|
}
|
|
src = src[1:]
|
|
}
|
|
backwardSize := (len(srcCheckpoint1) - len(src)) >> 2
|
|
if len(src) < 4 {
|
|
return dst, src, errInvalidXzData
|
|
}
|
|
checksum = crc32.ChecksumIEEE(srcCheckpoint1[:len(srcCheckpoint1)-len(src)])
|
|
if (src[0] != uint8(checksum>>0)) ||
|
|
(src[1] != uint8(checksum>>8)) ||
|
|
(src[2] != uint8(checksum>>16)) ||
|
|
(src[3] != uint8(checksum>>24)) {
|
|
return dst, src, errInvalidXzData
|
|
}
|
|
src = src[4:]
|
|
|
|
// Decode the footer.
|
|
if len(src) < 12 {
|
|
return dst, src, errInvalidXzData
|
|
}
|
|
checksum = crc32.ChecksumIEEE(src[4:10])
|
|
if (src[0] != uint8(checksum>>0)) || // Checksum.
|
|
(src[1] != uint8(checksum>>8)) ||
|
|
(src[2] != uint8(checksum>>16)) ||
|
|
(src[3] != uint8(checksum>>24)) ||
|
|
(src[4] != uint8(backwardSize>>0)) || // Backward Size.
|
|
(src[5] != uint8(backwardSize>>8)) ||
|
|
(src[6] != uint8(backwardSize>>16)) ||
|
|
(src[7] != uint8(backwardSize>>24)) ||
|
|
(src[8] != xzHeader24[6]) || // Stream Flags again.
|
|
(src[9] != xzHeader24[7]) ||
|
|
(src[10] != 0x59) || // Stream Footer Magic Bytes.
|
|
(src[11] != 0x5A) {
|
|
return dst, src, errInvalidXzData
|
|
}
|
|
return dst, src[12:], nil
|
|
}
|
|
|
|
func decodeRaw(dst []byte, src []byte, size uint64, errUnsupported error) (appendedDst []byte, remainingSrc []byte, retErr error) {
|
|
if (len(src) < 5) || (src[0] != 0x00) {
|
|
return dst, src, errUnsupported
|
|
}
|
|
|
|
rDec := rangeDecoder{
|
|
src: src[5:],
|
|
bits: (uint32(src[1]) << 24) |
|
|
(uint32(src[2]) << 16) |
|
|
(uint32(src[3]) << 8) |
|
|
(uint32(src[4]) << 0),
|
|
width: 0xFFFF_FFFF,
|
|
}
|
|
|
|
posProbs := [1 << pb]prob{}
|
|
setProbsToOneHalf(posProbs[:])
|
|
|
|
litProbs := [1 << (lc + lp)]byteProbs{}
|
|
for ij := range litProbs {
|
|
setProbsToOneHalf(litProbs[ij][:])
|
|
}
|
|
|
|
pos := uint32(0)
|
|
prev := byte(0)
|
|
curr := byte(0)
|
|
for ; size > 0; size-- {
|
|
if bitValue, err := posProbs[pos&pbMask].decodeBit(&rDec); err != nil {
|
|
return dst, rDec.src, err
|
|
} else if bitValue != 0 {
|
|
// A full-LZMA decoder would implement Lempel Ziv matches (or the
|
|
// End of Stream marker) here. Our simpler subset-of-LZMA decoder
|
|
// only uses literals.
|
|
return dst, rDec.src, errUnsupported
|
|
}
|
|
|
|
i := (pos & lpMask) << lc
|
|
j := uint32(prev) >> (8 - lc)
|
|
if curr, retErr = litProbs[i|j].decodeByte(&rDec); retErr != nil {
|
|
return dst, rDec.src, retErr
|
|
}
|
|
dst = append(dst, curr)
|
|
pos++
|
|
prev = curr
|
|
}
|
|
|
|
return dst, rDec.src, nil
|
|
}
|
|
|
|
func decodeUvarint(src []byte) (remainingSrc []byte, x uint64, ok bool) {
|
|
for i := uint32(0); (i < 63) && (len(src) > 0); i += 7 {
|
|
s := src[0]
|
|
src = src[1:]
|
|
x |= uint64(s&0x7F) << i
|
|
if (s & 0x80) == 0 {
|
|
return src, x, true
|
|
}
|
|
}
|
|
return src, 0, false
|
|
}
|
|
|
|
// Encode converts src to the Literal Only LZMA compressed file format,
|
|
// appending the encoding to dst.
|
|
//
|
|
// It is valid to pass a nil dst, like the way it's valid to pass nil to the
|
|
// built-in append function.
|
|
func (f FileFormat) Encode(dst []byte, src []byte) (appendedDst []byte, retErr error) {
|
|
switch f {
|
|
case FileFormatLZMA:
|
|
return encodeLZMA(dst, src)
|
|
case FileFormatXz:
|
|
return encodeXz(dst, src)
|
|
}
|
|
return nil, errUnsupportedFileFormat
|
|
}
|
|
|
|
func encodeLZMA(dst []byte, src []byte) (appendedDst []byte, retErr error) {
|
|
dst = append(dst, lzmaHeader5...)
|
|
size := len(src)
|
|
for i := 0; i < 8; i++ {
|
|
dst = append(dst, byte(size))
|
|
size >>= 8
|
|
}
|
|
return encodeRaw(dst, src), nil
|
|
}
|
|
|
|
func encodeXz(dst []byte, src []byte) (appendedDst []byte, retErr error) {
|
|
// Encode the headers (12 byte stream header then 12 byte block header).
|
|
// The block's unpaddedSize calculation ignores the stream header.
|
|
dstLen0 := len(dst) + 12
|
|
dst = append(dst, xzHeader24...)
|
|
|
|
// Encode the payload as multiple chunks.
|
|
//
|
|
// Later XZ chunks can depend on earlier ones (and doing so can help
|
|
// compression ratios) but for simplicity, this encoder emits independent
|
|
// chunks. Each chunk's maximum compressed and uncompressed size are both
|
|
// equal to 0x10000 = 65536 bytes, roughly speaking. The encoder partitions
|
|
// the src into 65536 byte sub-slices. If a sub-slice's LZMA-compressed
|
|
// form would be longer, the encoder emits an uncompressed chunk instead.
|
|
rawLZMA := []byte(nil)
|
|
for remaining := src; len(remaining) > 0; {
|
|
srcChunk := []byte(nil)
|
|
if len(remaining) > 0x10000 {
|
|
srcChunk, remaining = remaining[:0x10000], remaining[0x10000:]
|
|
} else {
|
|
srcChunk, remaining = remaining, nil
|
|
}
|
|
|
|
rawLZMA = encodeRaw(rawLZMA[:0], srcChunk)
|
|
if (len(srcChunk) + 3) <= (len(rawLZMA) + 6) {
|
|
dst = append(dst,
|
|
0x01, // Independent uncompressed chunk.
|
|
uint8((len(srcChunk)-1)>>8),
|
|
uint8((len(srcChunk)-1)>>0),
|
|
)
|
|
dst = append(dst, srcChunk...)
|
|
} else {
|
|
dst = append(dst,
|
|
0xE0, // Independent LZMA chunk.
|
|
uint8((len(srcChunk)-1)>>8),
|
|
uint8((len(srcChunk)-1)>>0),
|
|
uint8((len(rawLZMA)-1)>>8),
|
|
uint8((len(rawLZMA)-1)>>0),
|
|
0x5D, // Like lzmaHeader5[0], this encodes lc=3, lp=0, pb=2.
|
|
)
|
|
dst = append(dst, rawLZMA...)
|
|
}
|
|
}
|
|
dst = append(dst, 0x00) // No more chunks.
|
|
unpaddedSize := uint64(len(dst)-dstLen0) + 4 // +4 for the upcoming checksum.
|
|
for 0 != 3&(len(dst)-dstLen0) {
|
|
dst = append(dst, 0x00) // Padding
|
|
}
|
|
checksum := crc32.ChecksumIEEE(src)
|
|
dst = append(dst,
|
|
uint8(checksum>>0),
|
|
uint8(checksum>>8),
|
|
uint8(checksum>>16),
|
|
uint8(checksum>>24),
|
|
)
|
|
|
|
// Encode the index.
|
|
dstLen1 := len(dst)
|
|
dst = append(dst,
|
|
0x00, // Index Indicator.
|
|
0x01, // Number of Records.
|
|
)
|
|
dst = encodeUvarint(dst, unpaddedSize)
|
|
dst = encodeUvarint(dst, uint64(len(src)))
|
|
for 0 != 3&(len(dst)-dstLen1) {
|
|
dst = append(dst, 0x00) // Padding
|
|
}
|
|
backwardSize := (len(dst) - dstLen1) >> 2
|
|
checksum = crc32.ChecksumIEEE(dst[dstLen1:])
|
|
dst = append(dst,
|
|
uint8(checksum>>0),
|
|
uint8(checksum>>8),
|
|
uint8(checksum>>16),
|
|
uint8(checksum>>24),
|
|
)
|
|
|
|
// Encode the footer.
|
|
dstLen2 := len(dst)
|
|
dst = append(dst,
|
|
0, 0, 0, 0, // Space for the checksum.
|
|
uint8(backwardSize>>0), // Backward Size.
|
|
uint8(backwardSize>>8),
|
|
uint8(backwardSize>>16),
|
|
uint8(backwardSize>>24),
|
|
xzHeader24[6], // Stream Flags again.
|
|
xzHeader24[7],
|
|
0x59, // Stream Footer Magic Bytes.
|
|
0x5A,
|
|
)
|
|
checksum = crc32.ChecksumIEEE(dst[dstLen2+4 : dstLen2+10])
|
|
dst[dstLen2+0] = uint8(checksum >> 0)
|
|
dst[dstLen2+1] = uint8(checksum >> 8)
|
|
dst[dstLen2+2] = uint8(checksum >> 16)
|
|
dst[dstLen2+3] = uint8(checksum >> 24)
|
|
return dst, nil
|
|
}
|
|
|
|
func encodeRaw(dst []byte, src []byte) (appendedDst []byte) {
|
|
rEnc := rangeEncoder{
|
|
dst: dst,
|
|
width: 0xFFFF_FFFF,
|
|
}
|
|
|
|
posProbs := [1 << pb]prob{}
|
|
setProbsToOneHalf(posProbs[:])
|
|
|
|
litProbs := [1 << (lc + lp)]byteProbs{}
|
|
for ij := range litProbs {
|
|
setProbsToOneHalf(litProbs[ij][:])
|
|
}
|
|
|
|
pos := uint32(0)
|
|
prev := byte(0)
|
|
for _, curr := range src {
|
|
// A full-LZMA encoder would choose between Lempel Ziv literals and
|
|
// matches. Our simpler subset-of-LZMA encoder only uses literals, and
|
|
// it is not a streaming encoder (it does not need to emit an End of
|
|
// Stream marker), so we always encode a 0 bit here.
|
|
posProbs[pos&pbMask].encodeBit(&rEnc, 0)
|
|
|
|
i := (pos & lpMask) << lc
|
|
j := uint32(prev) >> (8 - lc)
|
|
litProbs[i|j].encodeByte(&rEnc, curr)
|
|
pos++
|
|
prev = curr
|
|
}
|
|
|
|
// Flush the rangeEncoder.
|
|
for i := 0; i < 5; i++ {
|
|
rEnc.shiftLow()
|
|
}
|
|
|
|
return rEnc.dst
|
|
}
|
|
|
|
func encodeUvarint(dst []byte, x uint64) []byte {
|
|
for ; x >= 0x80; x >>= 7 {
|
|
dst = append(dst, byte(x)|0x80)
|
|
}
|
|
return append(dst, byte(x))
|
|
}
|