Compare commits

...
Author SHA1 Message Date
Nigel Tao 315b2e5262 wuffs gen -version=0.3.0-rc.1 2022-07-10 10:30:15 +10:00
Nigel Tao fac934b272 Fix lib/readerat ReadSeeker.Seek offset bug
Fixes #80
2022-06-29 11:40:26 +10:00
Nigel Tao b8d44be783 Touch fuzz/c/std/bzip2_fuzzer.c again
This undoes commit c1283e49 now that issue #79 was closed.

Updates #79
2022-06-24 13:32:04 +10:00
Nigel Tao d0f0591f64 Add lib/litonlylzma 2022-06-24 13:22:36 +10:00
Nigel Tao c1283e4989 Touch fuzz/c/std/bzip2_fuzzer.c
Updates #79
2022-06-20 19:13:40 +10:00
Nigel Tao c226ed60f5 Use memcpy-based peeks and pokes on Microsoft C
I don't have a Windows machine readily available, but according to
https://godbolt.org/z/q4MfjzTPh and the https://imgur.com/UD5a7MF
profile mentioned in #72, this could improve inner loop performance.

Updates #72
2022-06-12 17:18:49 +10:00
Nigel Tao 45a79923e6 Tighten std/bzip2 Huffman leaf node bit packing 2022-06-10 14:31:53 +10:00
Nigel Tao 67aa63297c Rename std/bzip2 num_nodes variable 2022-06-10 14:06:27 +10:00
Nigel Tao 8c33dd503b Tighten std/bzip2 huffman_trees array size 2022-06-10 14:02:17 +10:00
Nigel Tao 5d7b113172 Optimize imageviewer xcb_clear_area calls 2022-06-08 17:22:29 +10:00
Nigel Tao 886e7f3e8f Simplify fuzzers' io_buffer construction 2022-06-08 17:08:49 +10:00
Nigel Tao fbc686f9b7 Add memrandomize to fuzz/c/fuzzlib 2022-06-08 17:06:59 +10:00
Nigel Tao 9256fe5e54 Have std/bzip2 check the number of sections
Updates #79
2022-06-08 16:41:46 +10:00
Nigel Tao e1b43cd767 Add test/data/artificial-bzip2/abraca.txt.bz2 2022-06-08 14:04:09 +10:00
Nigel Tao d90dde4049 Add .allstar/binary_artifacts.yaml
Fixes #78
2022-06-06 14:17:57 +10:00
27 changed files with 1367 additions and 187 deletions

No files matched your search

+6
View File
@@ -0,0 +1,6 @@
# Allstar (https://github.com/ossf/allstar/) is a GitHub App to set and enforce
# security policies.
# test/data/archive.iso is test data, not a binary to be executed.
ignorePaths:
- test/data/archive.iso
+31
View File
@@ -490,11 +490,35 @@ load(xcb_connection_t* c, xcb_window_t w, const char* filename) {
return true;
}
bool //
intersects(int32_t old_x,
int32_t old_y,
int32_t width,
int32_t height,
int32_t new_x,
int32_t new_y) {
if ((width <= 0) || (height <= 0)) {
return false;
} else if ((old_x > new_x) && ((old_x - new_x) >= width)) {
return false;
} else if ((new_x > old_x) && ((new_x - old_x) >= width)) {
return false;
} else if ((old_y > new_y) && ((old_y - new_y) >= height)) {
return false;
} else if ((new_y > old_y) && ((new_y - old_y) >= height)) {
return false;
}
return true;
}
// clear_area clears the L-shaped difference between old and new rectangles (of
// equal width and height). It does this in up to two xcb_clear_area calls,
// labeled A and B in the example below (with old_x=0, old_y=0, width=5,
// height=4, new_x=2, new_y=2).
//
// If the old and new rectangles do not intersect then the old rectangle (not
// an L-shape) will be cleared.
//
// AAAAA
// AAAAA
// BB+---+
@@ -510,6 +534,13 @@ clear_area(xcb_connection_t* c,
int32_t height,
int32_t new_x,
int32_t new_y) {
if ((width <= 0) || (height <= 0)) {
return;
} else if (!intersects(old_x, old_y, width, height, new_x, new_y)) {
xcb_clear_area(c, 1, w, old_x, old_y, width, height);
return;
}
int32_t dy = new_y - old_y;
if (dy < 0) {
xcb_clear_area(c, 1, w, old_x, old_y + height + dy, width, -dy);
+15 -12
View File
@@ -44,6 +44,19 @@ jenkins_hash_u32(const uint8_t* data, size_t size) {
return hash;
}
// memrandomize is like memcpy or memset but it writes pseudo-random values.
static void* //
memrandomize(void* dest, uint64_t seed, size_t n) {
unsigned short xsubi[3]; // See "man 3 nrand48".
xsubi[0] = (seed >> 0) ^ (seed >> 48);
xsubi[1] = (seed >> 16);
xsubi[2] = (seed >> 32);
for (uint8_t* ptr = (uint8_t*)dest; n--;) {
*ptr++ = nrand48(xsubi);
}
return dest;
}
const char* //
fuzz(wuffs_base__io_buffer* src, uint64_t hash);
@@ -55,18 +68,8 @@ llvmFuzzerTestOneInput(const uint8_t* data, size_t size) {
uint32_t hash1 = jenkins_hash_u32(data + s2, size - s2);
uint64_t hash = (((uint64_t)hash0) << 32) | ((uint64_t)hash1);
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = ((wuffs_base__slice_u8){
.ptr = (uint8_t*)(data),
.len = size,
}),
.meta = ((wuffs_base__io_buffer_meta){
.wi = size,
.ri = 0,
.pos = 0,
.closed = true,
}),
});
wuffs_base__io_buffer src =
wuffs_base__ptr_u8__reader((uint8_t*)data, size, true);
const char* msg = fuzz(&src, hash);
if (msg) {
+2 -6
View File
@@ -101,12 +101,8 @@ fuzz(wuffs_base__io_buffer* src, uint64_t hash) {
&dec, WUFFS_BASE__QUIRK_IGNORE_CHECKSUM, hash & 0xFFFE);
uint8_t dst_buffer[DST_BUFFER_ARRAY_SIZE];
wuffs_base__io_buffer dst = ((wuffs_base__io_buffer){
.data = ((wuffs_base__slice_u8){
.ptr = dst_buffer,
.len = DST_BUFFER_ARRAY_SIZE,
}),
});
wuffs_base__io_buffer dst =
wuffs_base__ptr_u8__writer(&dst_buffer[0], DST_BUFFER_ARRAY_SIZE);
while (true) {
dst.meta.wi = 0;
+2 -6
View File
@@ -103,12 +103,8 @@ fuzz(wuffs_base__io_buffer* src, uint64_t hash) {
&dec, WUFFS_BASE__QUIRK_IGNORE_CHECKSUM, hash & 0xFFFE);
uint8_t dst_buffer[DST_BUFFER_ARRAY_SIZE];
wuffs_base__io_buffer dst = ((wuffs_base__io_buffer){
.data = ((wuffs_base__slice_u8){
.ptr = dst_buffer,
.len = DST_BUFFER_ARRAY_SIZE,
}),
});
wuffs_base__io_buffer dst =
wuffs_base__ptr_u8__writer(&dst_buffer[0], DST_BUFFER_ARRAY_SIZE);
while (true) {
dst.meta.wi = 0;
+58 -3
View File
@@ -797,6 +797,22 @@ wuffs_base__count_leading_zeroes_u64(uint64_t u) {
// --------
// Normally, the wuffs_base__peek_etc and wuffs_base__poke_etc implementations
// are both (1) correct regardless of CPU endianness and (2) very fast (e.g. an
// inlined wuffs_base__peek_u32le__no_bounds_check call, in an optimized clang
// or gcc build, is a single MOV instruction on x86_64).
//
// However, the endian-agnostic implementations are slow on Microsoft's C
// compiler (MSC). Alternative memcpy-based implementations restore speed, but
// they are only correct on little-endian CPU architectures. Defining
// WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE opts in to these implementations.
//
// https://godbolt.org/z/q4MfjzTPh
#if defined(_MSC_VER) && !defined(__clang__) && \
(defined(_M_ARM64) || defined(_M_X64))
#define WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE
#endif
#define wuffs_base__peek_u8be__no_bounds_check \
wuffs_base__peek_u8__no_bounds_check
#define wuffs_base__peek_u8le__no_bounds_check \
@@ -809,12 +825,24 @@ wuffs_base__peek_u8__no_bounds_check(const uint8_t* p) {
static inline uint16_t //
wuffs_base__peek_u16be__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint16_t x;
memcpy(&x, p, 2);
return _byteswap_ushort(x);
#else
return (uint16_t)(((uint16_t)(p[0]) << 8) | ((uint16_t)(p[1]) << 0));
#endif
}
static inline uint16_t //
wuffs_base__peek_u16le__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint16_t x;
memcpy(&x, p, 2);
return x;
#else
return (uint16_t)(((uint16_t)(p[0]) << 0) | ((uint16_t)(p[1]) << 8));
#endif
}
static inline uint32_t //
@@ -831,14 +859,26 @@ wuffs_base__peek_u24le__no_bounds_check(const uint8_t* p) {
static inline uint32_t //
wuffs_base__peek_u32be__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint32_t x;
memcpy(&x, p, 4);
return _byteswap_ulong(x);
#else
return ((uint32_t)(p[0]) << 24) | ((uint32_t)(p[1]) << 16) |
((uint32_t)(p[2]) << 8) | ((uint32_t)(p[3]) << 0);
#endif
}
static inline uint32_t //
wuffs_base__peek_u32le__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint32_t x;
memcpy(&x, p, 4);
return x;
#else
return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) |
((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24);
#endif
}
static inline uint64_t //
@@ -887,18 +927,30 @@ wuffs_base__peek_u56le__no_bounds_check(const uint8_t* p) {
static inline uint64_t //
wuffs_base__peek_u64be__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint64_t x;
memcpy(&x, p, 8);
return _byteswap_uint64(x);
#else
return ((uint64_t)(p[0]) << 56) | ((uint64_t)(p[1]) << 48) |
((uint64_t)(p[2]) << 40) | ((uint64_t)(p[3]) << 32) |
((uint64_t)(p[4]) << 24) | ((uint64_t)(p[5]) << 16) |
((uint64_t)(p[6]) << 8) | ((uint64_t)(p[7]) << 0);
#endif
}
static inline uint64_t //
wuffs_base__peek_u64le__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint64_t x;
memcpy(&x, p, 8);
return x;
#else
return ((uint64_t)(p[0]) << 0) | ((uint64_t)(p[1]) << 8) |
((uint64_t)(p[2]) << 16) | ((uint64_t)(p[3]) << 24) |
((uint64_t)(p[4]) << 32) | ((uint64_t)(p[5]) << 40) |
((uint64_t)(p[6]) << 48) | ((uint64_t)(p[7]) << 56);
#endif
}
// --------
@@ -921,7 +973,8 @@ wuffs_base__poke_u16be__no_bounds_check(uint8_t* p, uint16_t x) {
static inline void //
wuffs_base__poke_u16le__no_bounds_check(uint8_t* p, uint16_t x) {
#if defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__)
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE) || \
(defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__))
// This seems to perform better on gcc 10 (but not clang 9). Clang also
// defines "__GNUC__".
memcpy(p, &x, 2);
@@ -955,7 +1008,8 @@ wuffs_base__poke_u32be__no_bounds_check(uint8_t* p, uint32_t x) {
static inline void //
wuffs_base__poke_u32le__no_bounds_check(uint8_t* p, uint32_t x) {
#if defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__)
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE) || \
(defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__))
// This seems to perform better on gcc 10 (but not clang 9). Clang also
// defines "__GNUC__".
memcpy(p, &x, 4);
@@ -1041,7 +1095,8 @@ wuffs_base__poke_u64be__no_bounds_check(uint8_t* p, uint64_t x) {
static inline void //
wuffs_base__poke_u64le__no_bounds_check(uint8_t* p, uint64_t x) {
#if defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__)
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE) || \
(defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__))
// This seems to perform better on gcc 10 (but not clang 9). Clang also
// defines "__GNUC__".
memcpy(p, &x, 8);
+45
View File
@@ -0,0 +1,45 @@
// Copyright 2022 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package litonlylzma_test
import (
"fmt"
"log"
"github.com/google/wuffs/lib/litonlylzma"
)
func ExampleRoundTrip() {
original := []byte("Hello world.\n")
compressed, err := litonlylzma.FileFormatLZMA.Encode(nil, original)
if err != nil {
log.Fatalf("Encode: %v", err)
}
recovered, _, err := litonlylzma.FileFormatLZMA.Decode(nil, compressed)
if err != nil {
log.Fatalf("Decode: %v", err)
}
for i, c := range compressed {
if i > 0 {
fmt.Printf(" ")
}
fmt.Printf("%02X", c)
}
fmt.Printf("\n%s\n", recovered)
// Output:
// 5D 00 10 00 00 0D 00 00 00 00 00 00 00 00 24 19 49 86 E7 D5 E5 E2 56 ED 6A E6 0E 81 FE B8 00 00
// Hello world.
}
+491
View File
@@ -0,0 +1,491 @@
// Copyright 2022 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------
// Package litonlylzma provides a decoder and encoder for Literal Only LZMA, a
// subset of the LZMA compressed file format.
//
// Subset means that the Encode method's output is a valid LZMA file and can be
// decompressed by the tools available at https://www.7-zip.org/sdk.html (and
// available as /usr/bin/lzma on a Debian system).
//
// Subset also means that this codec's compression ratios are not as good as
// full-LZMA (although compression times are much faster). Moderate compression
// is still achieved, through range coding, but there are no Lempel Ziv
// back-references. The main benefit, compared to full-LZMA, is that this
// implementation is much simpler and hence easier to study. It is only a few
// hundred lines of code.
//
// Example compression numbers on a small English text file (at
// https://github.com/google/wuffs/blob/main/test/data/romeo.txt):
// - romeo.txt is 942 bytes (100%).
// - romeo.txt.litonlylzma is 659 bytes (70%).
// - romeo.txt.xz is 644 bytes (68%).
// - romeo.txt.lzma is 598 bytes (63%).
// - romeo.txt.bz2 is 568 bytes (60%).
// - romeo.txt.zst is 559 bytes (59%).
// - romeo.txt.gz is 558 bytes (59%).
//
// Example compression numbers on a large archive of source code (at
// https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.0.1.tar.xz):
// - linux-5.0.1.tar is 863313920 bytes (100%).
// - linux-5.0.1.tar.litonlylzma is 449726070 bytes (52%).
// - linux-5.0.1.tar.gz is 164575959 bytes (19%).
// - linux-5.0.1.tar.zst is 156959897 bytes (18%).
// - linux-5.0.1.tar.bz2 is 125873134 bytes (15%).
// - linux-5.0.1.tar.xz is 108233572 bytes (13%).
// - linux-5.0.1.tar.lzma is 108216601 bytes (13%).
//
// The various tools (/usr/bin/gzip, /usr/bin/xz, etc) were all ran with their
// default settings, not their "maximum compression" settings. This is why the
// linux-5.0.1.tar.xz file size above differs from what cdn.kernel.org served.
package litonlylzma
import (
"errors"
)
var (
// ErrUnsupportedLZMAData is potentially returned by Decode.
ErrUnsupportedLZMAData = errors.New("litonlylzma: unsupported LZMA data")
errInvalidLZMAData = errors.New("litonlylzma: invalid LZMA data")
errUnexpectedEOF = errors.New("litonlylzma: unexpected EOF")
errUnsupportedFileFormat = errors.New("litonlylzma: unsupported FileFormat")
)
const (
// LZMA has a few configuration knobs but our subset-of-LZMA hard codes
// these to LZMA's default values:
// https://github.com/jljusten/LZMA-SDK/blob/781863cdf592da3e97420f50de5dac056ad352a5/C/LzmaEnc.h#L19-L21
// https://github.com/jljusten/LZMA-SDK/blob/781863cdf592da3e97420f50de5dac056ad352a5/DOC/lzma-specification.txt#L43-L45
lc = 3 // The number of Literal Context bits.
lp = 0 // The number of Literal Position bits.
pb = 2 // The number of Position Bits.
lpMask = (1 << lp) - 1
pbMask = (1 << pb) - 1
)
// lzmaHeader5 is the first 5 bytes of the LZMA header using the hard-coded
// configuration for the subset-of-LZMA that this package supports. 0x5D
// encodes the (lc, lp, pb) triple and the next four bytes hold the u32le
// dictionary size. 0x1000 is LZMA's minimum dictionary size, although our
// subset-of-LZMA does not use a dictionary (also called a "sliding window").
const lzmaHeader5 = "\x5D\x00\x10\x00\x00"
// rangeDecoder and rangeEncoder hold the state for range coding (also known as
// arithmetic coding, roughly speaking). "range" is a keyword in Go, so in this
// implementation, what's traditionally called the "range" fields are called
// "width" instead.
//
// Conceptually, the encoded bitstream holds a high precision number (a binary
// fraction) in the range from zero to one. High precision could mean thousands
// or millions of bits, but when decoding, only 32 of these are 'paged in' at a
// time - this is the rangeDecoder.bits field. The encoder similarly works on
// only a small, fixed number of bits.
//
// When encoding, that high precision *number* isn't fully known until the end
// of the input is reached. *During* the encoding process, what's known is a
// high precision *interval* that gets narrowed down as the input bytes are
// consumed. That interval is represented by the rangeEncoder.low and
// rangeEncoder.width fields (the width is the difference between the
// interval's upper and lower bound). Both low and width are, conceptually,
// 32-bit values - they operate at the same scale - but low is an uint64
// because of potential overflow. We'll revisit that below.
//
// Width is initialized to 0xFFFF_FFFF and shrinks every time a bit is decoded
// or encoded. Similarly, low can only increase or stay the same on each bit
// step. When width drops below (1 << 24), the rangeDecoder or rangeEncoder
// 'zooms in', left-shifting (with truncation) by 8 the bits, low and width
// fields and reading a src byte or writing a dst byte of compressed data.
//
// Writing a byte may be delayed because the encoding interval could span
// multiple possible digits. To use a decimal fraction analogy, suppose that
// our interval is [0.7138872, 0.7146055] and the low field nominally holds 4
// digits. We could emit '7' and '1' but we have to hold back on emitting
// either '3' or '4' until we can narrow down the interval further. That '3'
// digit is stashed as the pendingHead field. low is set to 8872, width is set
// to (46055 - 38872) = 7183 and we progress until the width drops below 1000,
// causing a 'zoom in' event. There are three cases:
// - if low is less than 9000 (but starts with an '8'), we can emit the '3'
// with confidence (and then stash '8' in the pendingHead field).
// - if low overflows 9999, we can emit the '4' with confidence (and then stash
// low's 'thousands digit' in the pendingHead field).
// - if low is in 9000 ..= 9999 then we could still be unsure. Keep the '3'
// pending but extend it so that there are two pending digits: '39'. If this
// occurs again: '399', and so on.
//
// In all three cases, 'zooming in' causes us to shift out the most significant
// digit of the low field. This method is therefore named shiftLow here, the
// same name used in other range coding implementations, such as
// https://github.com/jljusten/LZMA-SDK/blob/781863cdf592da3e97420f50de5dac056ad352a5/C/LzmaEnc.c#L572-L602
//
// The pendingEtc fields combine to hold one or more digits ('0' ..= '9'
// decimal digits in our analogy, 0x00 ..= 0xFF bytes in our actual
// implementation). There are (1 + pendingExtra) digits in total and while the
// head digit might be any value, the extra digits must be all '9' (in the
// decimal analogy) or 0xFF (in our actual implementation).
//
// Some other implementations use the name "cache" instead of "pending", and
// store a "cacheSize" field equivalent to our (1 + pendingExtra).
//
// As above, the low field is conceptually a uint32, but it is a uint64 because
// of this potential overflow. Its high 32 bits are usually zero, rarely one
// and never anything higher. The high 32 bits essentially hold an overflow
// carry bit. If set, it flips "3999etc" to "4000etc".
//
// "Low is conceptually 4 bytes; pendingHead holds another byte" is why the
// shortest LZMA payload is 5 bytes even though the first byte is always 0x00
// and seems redundant.
type rangeDecoder struct {
src []byte
bits uint32
width uint32
}
type rangeEncoder struct {
dst []byte
low uint64
width uint32
pendingHead uint8
pendingExtra uint64
}
// Precondition: (rEnc.width < (1 << 24)) or we are at the end of our input
// (and are flushing the rangeEncoder).
func (rEnc *rangeEncoder) shiftLow() {
if rEnc.low < 0x0_FF00_0000 {
rEnc.dst = append(rEnc.dst, rEnc.pendingHead+0x00)
for ; rEnc.pendingExtra > 0; rEnc.pendingExtra-- {
rEnc.dst = append(rEnc.dst, 0xFF)
}
rEnc.pendingHead = uint8(rEnc.low >> 24)
rEnc.pendingExtra = 0
rEnc.low = (rEnc.low << 8) & 0xFFFF_FFFF
} else if rEnc.low < 0x1_0000_0000 {
rEnc.pendingExtra++
rEnc.low = (rEnc.low << 8) & 0xFFFF_FFFF
} else {
rEnc.dst = append(rEnc.dst, rEnc.pendingHead+0x01)
for ; rEnc.pendingExtra > 0; rEnc.pendingExtra-- {
rEnc.dst = append(rEnc.dst, 0x00)
}
rEnc.pendingHead = uint8(rEnc.low >> 24)
rEnc.pendingExtra = 0
rEnc.low = (rEnc.low << 8) & 0xFFFF_FFFF
}
}
// prob is a probability (out of 100%) whose uint16 value ranges from 0 to
// (1 << 11) = 2048 inclusive. For example, prob(1024) means a 50% probability
// and prob(384) means a (384 / 2048) = 18.75% probability.
//
// Specifically, it is the probability that the next bit is 0 (instead of 1).
type prob uint16
const (
// probBits is such that prob(1 << probBits) means a 100% probability.
probBits = 11
minProb = 0
maxProb = 1 << probBits
)
func setProbsToOneHalf(p []prob) {
for i := range p {
p[i] = 1 << (probBits - 1)
}
}
// adaptShift scales how much to adjust "the probability that the next bit is
// 0" after seeing the current bit. The probability is adaptive in that the
// current bit being 0 (or 1) increases (or decreases) the probability that the
// next bit is 0. The larger adaptShift is, the slower the adaptation.
const adaptShift = 5
func (p *prob) decodeBit(rDec *rangeDecoder) (bitValue uint32, retErr error) {
threshold := (rDec.width >> probBits) * uint32(*p)
if rDec.bits < threshold {
bitValue = 0
rDec.width = threshold
*p += (maxProb - *p) >> adaptShift
} else {
bitValue = 1
rDec.bits -= threshold
rDec.width -= threshold
*p -= (*p - minProb) >> adaptShift
}
if rDec.width < (1 << 24) {
if len(rDec.src) <= 0 {
return 0, errUnexpectedEOF
}
rDec.bits = rDec.bits<<8 | uint32(rDec.src[0])
rDec.width <<= 8
rDec.src = rDec.src[1:]
}
return bitValue, retErr
}
func (p *prob) encodeBit(rEnc *rangeEncoder, bitValue uint32) {
threshold := (rEnc.width >> probBits) * uint32(*p)
if bitValue == 0 {
rEnc.width = threshold
*p += (maxProb - *p) >> adaptShift
} else {
rEnc.low += uint64(threshold)
rEnc.width -= threshold
*p -= (*p - minProb) >> adaptShift
}
if rEnc.width < (1 << 24) {
rEnc.width <<= 8
rEnc.shiftLow()
}
}
// byteProbs hold probabilities for coding 8 bits (1 byte) of data. It is a 256
// element array such that:
// - The 0th element is unused.
// - The 1st element holds the probability that the 7th bit (the highest, most
// significant bit) is 0.
// - The 2nd element holds the probability that the 6th bit is 0, conditional
// on the high bit being 0.
// - The 3rd element holds the probability that the 6th bit is 0, conditional
// on the high bit being 1.
// - The 4th element holds the probability that the 5th bit is 0, conditional
// on the high two bits being 00.
// - The 5th element holds the probability that the 5th bit is 0, conditional
// on the high two bits being 01.
// - The 6th element holds the probability that the 5th bit is 0, conditional
// on the high two bits being 10.
// - The 7th element holds the probability that the 5th bit is 0, conditional
// on the high two bits being 11.
// - The 8th element holds the probability that the 4th bit is 0, conditional
// on the high three bits being 000.
// - etc
// - The 255th element holds the probability that the 0th bit (the lowest,
// least significant bit) is 0, conditional on the high seven bits being
// 1111111.
//
// Put another way, the 256 elements' value of N, as in "it's a probability for
// the Nth bit", looks like this (when arranged in 8 rows of 32 elements):
//
// u7665555444444443333333333333333
// 22222222222222222222222222222222
// 11111111111111111111111111111111
// 11111111111111111111111111111111
// 00000000000000000000000000000000
// 00000000000000000000000000000000
// 00000000000000000000000000000000
// 00000000000000000000000000000000
//
// The 'u' means that the 0th element is unused.
type byteProbs [0x100]prob
func (p *byteProbs) decodeByte(rDec *rangeDecoder) (byteValue byte, retErr error) {
index := uint32(1)
for index < 0x100 {
if bitValue, err := p[index].decodeBit(rDec); err != nil {
return 0, err
} else {
index = index<<1 | bitValue
}
}
return byte(index), nil
}
func (p *byteProbs) encodeByte(rEnc *rangeEncoder, byteValue byte) {
b := uint32(byteValue)
index := uint32(1)
for i := 7; i >= 0; i-- {
bitValue := (b >> uint(i)) & 1
p[index].encodeBit(rEnc, bitValue)
index = index<<1 | bitValue
}
}
// FileFormat is a compressed file format, either the original LZMA format
// itself or something that builds on or varies that.
//
// Each valid FileFormatXxx value does not distinguish between the full file
// format (as spoken by numerous Xxx software tools) and the subset-of-Xxx
// implemented by this package. Specifically, FileFormatLZMA.Encode will
// produce compressed data that is valid full-LZMA (and is also valid
// subset-of-LZMA). FileFormatLZMA.Decode can decode subset-of-LZMA (but not
// full-LZMA).
//
// This package currently defines only one valid FileFormatXxx value:
// FileFormatLZMA. Future versions of this package could define e.g.
// FileFormatLZMA2 or FileFormatXz such that FileFormatXz.Encode would produce
// valid full-XZ data: a valid XZ file (that could be decoded by official XZ
// tools) that used the LZMA compression filter, but elected not to use the
// full capabilities of that filter (using LZ literals only, not matches).
type FileFormat uint32
const (
FileFormatInvalid = FileFormat(0)
FileFormatLZMA = FileFormat(1)
)
// Decode converts src from the Literal Only LZMA compressed file format,
// appending the decoding to dst.
//
// f should be FileFormatLZMA but future versions of this package may support
// other file formats.
//
// It is valid to pass a nil dst, like the way it's valid to pass nil to the
// built-in append function.
//
// Decode also returns the suffix of src that was not used during decoding.
//
// It can also return partial results. The appendedDst and remainingSrc return
// values may be non-zero even if retErr is non-zero.
//
// It returns ErrUnsupportedLZMAData if the source bytes look like LZMA
// formatted data that is outside the subset-of-LZMA that this package
// implements.
func (f FileFormat) Decode(dst []byte, src []byte) (appendedDst []byte, remainingSrc []byte, retErr error) {
if f != FileFormatLZMA {
return dst, nil, errUnsupportedFileFormat
}
// LZMA consists of a 13 byte header and then at least a 5 byte payload
// (range coded data). The 13 byte header is:
// - 1 byte for the (lc, lp, pb) triple with max values (8, 4, 4),
// - 4 bytes u32le dictionary size and
// - 8 bytes i64le uncompressed size.
if (len(src) < 18) || (src[0] >= (9 * 5 * 5)) {
return dst, nil, errInvalidLZMAData
} else if (src[0] != lzmaHeader5[0]) ||
(src[1] != lzmaHeader5[1]) ||
(src[2] != lzmaHeader5[2]) ||
(src[3] != lzmaHeader5[3]) ||
(src[4] != lzmaHeader5[4]) ||
(src[13] != 0x00) {
return dst, nil, ErrUnsupportedLZMAData
}
size := uint64(0)
for i := 0; i < 8; i++ {
size |= uint64(src[5+i]) << uint(8*i)
}
if int64(size) < -1 {
return dst, nil, errInvalidLZMAData
} else if int64(size) == -1 {
return dst, nil, ErrUnsupportedLZMAData
}
rDec := rangeDecoder{
src: src[18:],
bits: (uint32(src[14]) << 24) |
(uint32(src[15]) << 16) |
(uint32(src[16]) << 8) |
(uint32(src[17]) << 0),
width: 0xFFFF_FFFF,
}
posProbs := [1 << pb]prob{}
setProbsToOneHalf(posProbs[:])
litProbs := [1 << (lc + lp)]byteProbs{}
for ij := range litProbs {
setProbsToOneHalf(litProbs[ij][:])
}
pos := uint32(0)
prev := byte(0)
curr := byte(0)
for ; size > 0; size-- {
if bitValue, err := posProbs[pos&pbMask].decodeBit(&rDec); err != nil {
return dst, nil, err
} else if bitValue != 0 {
// A full-LZMA decoder would implement Lempel Ziv matches (or the
// End of Stream marker) here. Our simpler subset-of-LZMA decoder
// only uses literals.
return dst, rDec.src, ErrUnsupportedLZMAData
}
i := (pos & lpMask) << lc
j := uint32(prev) >> (8 - lc)
if curr, retErr = litProbs[i|j].decodeByte(&rDec); retErr != nil {
return dst, rDec.src, retErr
}
dst = append(dst, curr)
pos++
prev = curr
}
return dst, rDec.src, nil
}
// Encode converts src to the Literal Only LZMA compressed file format,
// appending the encoding to dst.
//
// f should be FileFormatLZMA but future versions of this package may support
// other file formats.
//
// It is valid to pass a nil dst, like the way it's valid to pass nil to the
// built-in append function.
func (f FileFormat) Encode(dst []byte, src []byte) (appendedDst []byte, retErr error) {
if f != FileFormatLZMA {
return nil, errUnsupportedFileFormat
}
dst = append(dst, lzmaHeader5...)
size := len(src)
for i := 0; i < 8; i++ {
dst = append(dst, byte(size))
size >>= 8
}
rEnc := rangeEncoder{
dst: dst,
width: 0xFFFF_FFFF,
}
posProbs := [1 << pb]prob{}
setProbsToOneHalf(posProbs[:])
litProbs := [1 << (lc + lp)]byteProbs{}
for ij := range litProbs {
setProbsToOneHalf(litProbs[ij][:])
}
pos := uint32(0)
prev := byte(0)
for _, curr := range src {
// A full-LZMA encoder would choose between Lempel Ziv literals and
// matches. Our simpler subset-of-LZMA encoder only uses literals, and
// it is not a streaming encoder (it does not need to emit an End of
// Stream marker), so we always encode a 0 bit here.
posProbs[pos&pbMask].encodeBit(&rEnc, 0)
i := (pos & lpMask) << lc
j := uint32(prev) >> (8 - lc)
litProbs[i|j].encodeByte(&rEnc, curr)
pos++
prev = curr
}
// Flush the rangeEncoder.
for i := 0; i < 5; i++ {
rEnc.shiftLow()
}
return rEnc.dst, nil
}
+64
View File
@@ -0,0 +1,64 @@
// Copyright 2022 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package litonlylzma
import (
"bytes"
"os"
"testing"
)
const (
oneThousand0x00s = "oneThousand0x00s"
oneThousand0xFFs = "oneThousand0xFFs"
)
func testRoundTrip(tt *testing.T, filename string) {
original := []byte(nil)
switch filename {
case oneThousand0x00s:
original = make([]byte, 1000)
case oneThousand0xFFs:
original = make([]byte, 1000)
for i := range original {
original[i] = 0xFF
}
default:
o, err := os.ReadFile("../../test/data/" + filename)
if err != nil {
tt.Fatalf("ReadFile: %v", err)
}
original = o
}
compressed, err := FileFormatLZMA.Encode(nil, original)
if err != nil {
tt.Fatalf("Encode: %v", err)
}
recovered, _, err := FileFormatLZMA.Decode(nil, compressed)
if err != nil {
tt.Fatalf("Decode: %v", err)
}
if !bytes.Equal(original, recovered) {
tt.Fatalf("round trip produced different bytes")
}
}
func TestRoundTrip0Bytes(tt *testing.T) { testRoundTrip(tt, "0.bytes") }
func TestRoundTripArchiveTar(tt *testing.T) { testRoundTrip(tt, "archive.tar") }
func TestRoundTripOneThousand0x00s(tt *testing.T) { testRoundTrip(tt, oneThousand0x00s) }
func TestRoundTripOneThousand0xFFs(tt *testing.T) { testRoundTrip(tt, oneThousand0xFFs) }
func TestRoundTripPiTxt(tt *testing.T) { testRoundTrip(tt, "pi.txt") }
func TestRoundTripRomeoTxt(tt *testing.T) { testRoundTrip(tt, "romeo.txt") }
+1 -1
View File
@@ -87,7 +87,7 @@ func (r *ReadSeeker) Seek(offset int64, whence int) (int64, error) {
return 0, errSeekToInvalidWhence
}
if r.offset < 0 {
if offset < 0 {
return 0, errSeekToNegativePosition
}
r.offset = offset
+90 -31
View File
@@ -1064,6 +1064,22 @@ wuffs_base__count_leading_zeroes_u64(uint64_t u) {
// --------
// Normally, the wuffs_base__peek_etc and wuffs_base__poke_etc implementations
// are both (1) correct regardless of CPU endianness and (2) very fast (e.g. an
// inlined wuffs_base__peek_u32le__no_bounds_check call, in an optimized clang
// or gcc build, is a single MOV instruction on x86_64).
//
// However, the endian-agnostic implementations are slow on Microsoft's C
// compiler (MSC). Alternative memcpy-based implementations restore speed, but
// they are only correct on little-endian CPU architectures. Defining
// WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE opts in to these implementations.
//
// https://godbolt.org/z/q4MfjzTPh
#if defined(_MSC_VER) && !defined(__clang__) && \
(defined(_M_ARM64) || defined(_M_X64))
#define WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE
#endif
#define wuffs_base__peek_u8be__no_bounds_check \
wuffs_base__peek_u8__no_bounds_check
#define wuffs_base__peek_u8le__no_bounds_check \
@@ -1076,12 +1092,24 @@ wuffs_base__peek_u8__no_bounds_check(const uint8_t* p) {
static inline uint16_t //
wuffs_base__peek_u16be__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint16_t x;
memcpy(&x, p, 2);
return _byteswap_ushort(x);
#else
return (uint16_t)(((uint16_t)(p[0]) << 8) | ((uint16_t)(p[1]) << 0));
#endif
}
static inline uint16_t //
wuffs_base__peek_u16le__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint16_t x;
memcpy(&x, p, 2);
return x;
#else
return (uint16_t)(((uint16_t)(p[0]) << 0) | ((uint16_t)(p[1]) << 8));
#endif
}
static inline uint32_t //
@@ -1098,14 +1126,26 @@ wuffs_base__peek_u24le__no_bounds_check(const uint8_t* p) {
static inline uint32_t //
wuffs_base__peek_u32be__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint32_t x;
memcpy(&x, p, 4);
return _byteswap_ulong(x);
#else
return ((uint32_t)(p[0]) << 24) | ((uint32_t)(p[1]) << 16) |
((uint32_t)(p[2]) << 8) | ((uint32_t)(p[3]) << 0);
#endif
}
static inline uint32_t //
wuffs_base__peek_u32le__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint32_t x;
memcpy(&x, p, 4);
return x;
#else
return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) |
((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24);
#endif
}
static inline uint64_t //
@@ -1154,18 +1194,30 @@ wuffs_base__peek_u56le__no_bounds_check(const uint8_t* p) {
static inline uint64_t //
wuffs_base__peek_u64be__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint64_t x;
memcpy(&x, p, 8);
return _byteswap_uint64(x);
#else
return ((uint64_t)(p[0]) << 56) | ((uint64_t)(p[1]) << 48) |
((uint64_t)(p[2]) << 40) | ((uint64_t)(p[3]) << 32) |
((uint64_t)(p[4]) << 24) | ((uint64_t)(p[5]) << 16) |
((uint64_t)(p[6]) << 8) | ((uint64_t)(p[7]) << 0);
#endif
}
static inline uint64_t //
wuffs_base__peek_u64le__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint64_t x;
memcpy(&x, p, 8);
return x;
#else
return ((uint64_t)(p[0]) << 0) | ((uint64_t)(p[1]) << 8) |
((uint64_t)(p[2]) << 16) | ((uint64_t)(p[3]) << 24) |
((uint64_t)(p[4]) << 32) | ((uint64_t)(p[5]) << 40) |
((uint64_t)(p[6]) << 48) | ((uint64_t)(p[7]) << 56);
#endif
}
// --------
@@ -1188,7 +1240,8 @@ wuffs_base__poke_u16be__no_bounds_check(uint8_t* p, uint16_t x) {
static inline void //
wuffs_base__poke_u16le__no_bounds_check(uint8_t* p, uint16_t x) {
#if defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__)
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE) || \
(defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__))
// This seems to perform better on gcc 10 (but not clang 9). Clang also
// defines "__GNUC__".
memcpy(p, &x, 2);
@@ -1222,7 +1275,8 @@ wuffs_base__poke_u32be__no_bounds_check(uint8_t* p, uint32_t x) {
static inline void //
wuffs_base__poke_u32le__no_bounds_check(uint8_t* p, uint32_t x) {
#if defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__)
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE) || \
(defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__))
// This seems to perform better on gcc 10 (but not clang 9). Clang also
// defines "__GNUC__".
memcpy(p, &x, 4);
@@ -1308,7 +1362,8 @@ wuffs_base__poke_u64be__no_bounds_check(uint8_t* p, uint64_t x) {
static inline void //
wuffs_base__poke_u64le__no_bounds_check(uint8_t* p, uint64_t x) {
#if defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__)
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE) || \
(defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__))
// This seems to perform better on gcc 10 (but not clang 9). Clang also
// defines "__GNUC__".
memcpy(p, &x, 8);
@@ -6827,9 +6882,7 @@ extern const char wuffs_bzip2__error__bad_block_header[];
extern const char wuffs_bzip2__error__bad_block_length[];
extern const char wuffs_bzip2__error__bad_checksum[];
extern const char wuffs_bzip2__error__bad_header[];
extern const char wuffs_bzip2__error__bad_number_of_huffman_codes[];
extern const char wuffs_bzip2__error__bad_number_of_sections[];
extern const char wuffs_bzip2__error__unsupported_huffman_code[];
extern const char wuffs_bzip2__error__unsupported_block_randomization[];
// ---------------- Public Consts
@@ -6967,7 +7020,7 @@ struct wuffs_bzip2__decoder__struct {
uint8_t f_presence[256];
uint8_t f_mtft[256];
uint8_t f_huffman_selectors[32768];
uint16_t f_huffman_trees[6][1024][2];
uint16_t f_huffman_trees[6][257][2];
uint16_t f_huffman_tables[6][256];
uint32_t f_bwt[1048576];
@@ -24866,9 +24919,7 @@ const char wuffs_bzip2__error__bad_block_header[] = "#bzip2: bad block header";
const char wuffs_bzip2__error__bad_block_length[] = "#bzip2: bad block length";
const char wuffs_bzip2__error__bad_checksum[] = "#bzip2: bad checksum";
const char wuffs_bzip2__error__bad_header[] = "#bzip2: bad header";
const char wuffs_bzip2__error__bad_number_of_huffman_codes[] = "#bzip2: bad number of Huffman codes";
const char wuffs_bzip2__error__bad_number_of_sections[] = "#bzip2: bad number of sections";
const char wuffs_bzip2__error__unsupported_huffman_code[] = "#bzip2: unsupported Huffman code";
const char wuffs_bzip2__error__unsupported_block_randomization[] = "#bzip2: unsupported block randomization";
const char wuffs_bzip2__error__internal_error_inconsistent_huffman_decoder_state[] = "#bzip2: internal error: inconsistent Huffman decoder state";
@@ -25804,7 +25855,7 @@ wuffs_bzip2__decoder__build_huffman_tree(
uint32_t a_which) {
uint32_t v_code_length = 0;
uint32_t v_symbol_index = 0;
uint32_t v_num_nodes = 0;
uint32_t v_num_branch_nodes = 0;
uint32_t v_stack_height = 0;
uint32_t v_stack_values[21] = {0};
uint32_t v_node_index = 0;
@@ -25812,7 +25863,7 @@ wuffs_bzip2__decoder__build_huffman_tree(
self->private_data.f_huffman_trees[a_which][0][0] = 0;
self->private_data.f_huffman_trees[a_which][0][1] = 0;
v_num_nodes = 1;
v_num_branch_nodes = 1;
v_stack_height = 1;
v_stack_values[0] = 0;
v_code_length = 1;
@@ -25837,27 +25888,27 @@ wuffs_bzip2__decoder__build_huffman_tree(
}
v_node_index = v_stack_values[(v_stack_height - 1)];
if (self->private_data.f_huffman_trees[a_which][v_node_index][0] == 0) {
self->private_data.f_huffman_trees[a_which][v_node_index][0] = ((uint16_t)(v_num_nodes));
self->private_data.f_huffman_trees[a_which][v_node_index][0] = ((uint16_t)(v_num_branch_nodes));
} else {
self->private_data.f_huffman_trees[a_which][v_node_index][1] = ((uint16_t)(v_num_nodes));
self->private_data.f_huffman_trees[a_which][v_node_index][1] = ((uint16_t)(v_num_branch_nodes));
}
if (v_num_nodes >= 1023) {
return wuffs_base__make_status(wuffs_bzip2__error__unsupported_huffman_code);
if (v_num_branch_nodes >= 257) {
return wuffs_base__make_status(wuffs_bzip2__error__bad_huffman_code_under_subscribed);
}
v_stack_values[v_stack_height] = v_num_nodes;
self->private_data.f_huffman_trees[a_which][v_num_nodes][0] = 0;
self->private_data.f_huffman_trees[a_which][v_num_nodes][1] = 0;
v_num_nodes += 1;
v_stack_values[v_stack_height] = v_num_branch_nodes;
self->private_data.f_huffman_trees[a_which][v_num_branch_nodes][0] = 0;
self->private_data.f_huffman_trees[a_which][v_num_branch_nodes][1] = 0;
v_num_branch_nodes += 1;
v_stack_height += 1;
}
label__2__break:;
v_node_index = v_stack_values[(v_stack_height - 1)];
if (v_symbol_index < 2) {
v_leaf_value = ((uint16_t)((1280 + v_symbol_index)));
v_leaf_value = ((uint16_t)((769 + v_symbol_index)));
} else if ((v_symbol_index + 1) < self->private_impl.f_num_symbols) {
v_leaf_value = ((uint16_t)((1023 + v_symbol_index)));
v_leaf_value = ((uint16_t)((511 + v_symbol_index)));
} else {
v_leaf_value = 2047;
v_leaf_value = 768;
}
if (self->private_data.f_huffman_trees[a_which][v_node_index][0] == 0) {
self->private_data.f_huffman_trees[a_which][v_node_index][0] = v_leaf_value;
@@ -25898,7 +25949,7 @@ wuffs_bzip2__decoder__build_huffman_table(
v_bits = (v_i << 24);
v_n_bits = 0;
v_child = 0;
while ((v_child < 1024) && (v_n_bits < 8)) {
while ((v_child < 257) && (v_n_bits < 8)) {
v_child = self->private_data.f_huffman_trees[a_which][v_child][(v_bits >> 31)];
v_bits <<= 1;
#if defined(__GNUC__)
@@ -26192,6 +26243,10 @@ wuffs_bzip2__decoder__decode_huffman_fast(
} else {
v_ticks = 49;
v_section += 1;
if (v_section >= self->private_impl.f_num_sections) {
status = wuffs_base__make_status(wuffs_bzip2__error__bad_number_of_sections);
goto exit;
}
v_which = WUFFS_BZIP2__CLAMP_TO_5[(self->private_data.f_huffman_selectors[(v_section & 32767)] & 7)];
}
v_bits |= (wuffs_base__peek_u32be__no_bounds_check(iop_a_src) >> v_n_bits);
@@ -26200,8 +26255,8 @@ wuffs_bzip2__decoder__decode_huffman_fast(
v_table_entry = self->private_data.f_huffman_tables[v_which][(v_bits >> 24)];
v_bits <<= (v_table_entry >> 12);
v_n_bits -= ((uint32_t)((v_table_entry >> 12)));
v_child = (v_table_entry & 2047);
while (v_child < 1024) {
v_child = (v_table_entry & 1023);
while (v_child < 257) {
v_child = self->private_data.f_huffman_trees[v_which][v_child][(v_bits >> 31)];
v_bits <<= 1;
if (v_n_bits <= 0) {
@@ -26210,7 +26265,7 @@ wuffs_bzip2__decoder__decode_huffman_fast(
}
v_n_bits -= 1;
}
if (v_child < 1280) {
if (v_child < 768) {
v_child_ff = ((uint32_t)((v_child & 255)));
v_output = ((uint32_t)(self->private_data.f_mtft[v_child_ff]));
wuffs_base__slice_u8__copy_from_slice(wuffs_base__make_slice_u8_ij(self->private_data.f_mtft, 1, (1 + v_child_ff)), wuffs_base__make_slice_u8(self->private_data.f_mtft, v_child_ff));
@@ -26224,7 +26279,7 @@ wuffs_bzip2__decoder__decode_huffman_fast(
v_block_size += 1;
v_run_shift = 0;
goto label__outer__continue;
} else if (v_child > 1281) {
} else if (v_child == 768) {
self->private_impl.f_decode_huffman_finished = true;
goto label__outer__break;
}
@@ -26232,7 +26287,7 @@ wuffs_bzip2__decoder__decode_huffman_fast(
status = wuffs_base__make_status(wuffs_bzip2__error__bad_block_length);
goto exit;
}
v_run = (((uint32_t)((v_child - 1279))) << v_run_shift);
v_run = ((((uint32_t)(v_child)) & 3) << v_run_shift);
v_run_shift += 1;
v_i = v_block_size;
v_j = (v_run + v_block_size);
@@ -26311,6 +26366,10 @@ wuffs_bzip2__decoder__decode_huffman_slow(
} else {
self->private_impl.f_decode_huffman_ticks = 49;
self->private_impl.f_decode_huffman_section += 1;
if (self->private_impl.f_decode_huffman_section >= self->private_impl.f_num_sections) {
status = wuffs_base__make_status(wuffs_bzip2__error__bad_number_of_sections);
goto exit;
}
self->private_impl.f_decode_huffman_which = WUFFS_BZIP2__CLAMP_TO_5[(self->private_data.f_huffman_selectors[(self->private_impl.f_decode_huffman_section & 32767)] & 7)];
}
v_node_index = 0;
@@ -26332,10 +26391,10 @@ wuffs_bzip2__decoder__decode_huffman_slow(
v_child = self->private_data.f_huffman_trees[self->private_impl.f_decode_huffman_which][v_node_index][(self->private_impl.f_bits >> 31)];
self->private_impl.f_bits <<= 1;
self->private_impl.f_n_bits -= 1;
if (v_child < 1024) {
if (v_child < 257) {
v_node_index = ((uint32_t)(v_child));
goto label__0__continue;
} else if (v_child < 1280) {
} else if (v_child < 768) {
v_child_ff = ((uint32_t)((v_child & 255)));
v_output = ((uint32_t)(self->private_data.f_mtft[v_child_ff]));
wuffs_base__slice_u8__copy_from_slice(wuffs_base__make_slice_u8_ij(self->private_data.f_mtft, 1, (1 + v_child_ff)), wuffs_base__make_slice_u8(self->private_data.f_mtft, v_child_ff));
@@ -26349,7 +26408,7 @@ wuffs_bzip2__decoder__decode_huffman_slow(
self->private_impl.f_block_size += 1;
self->private_impl.f_decode_huffman_run_shift = 0;
goto label__0__break;
} else if (v_child > 1281) {
} else if (v_child == 768) {
self->private_impl.f_decode_huffman_finished = true;
goto label__outer__break;
}
@@ -26357,7 +26416,7 @@ wuffs_bzip2__decoder__decode_huffman_slow(
status = wuffs_base__make_status(wuffs_bzip2__error__bad_block_length);
goto exit;
}
v_run = (((uint32_t)((v_child - 1279))) << self->private_impl.f_decode_huffman_run_shift);
v_run = ((((uint32_t)(v_child)) & 3) << self->private_impl.f_decode_huffman_run_shift);
self->private_impl.f_decode_huffman_run_shift += 1;
v_i = self->private_impl.f_block_size;
v_j = (v_run + self->private_impl.f_block_size);
+95 -36
View File
@@ -85,15 +85,15 @@ extern "C" {
// each major.minor branch, the commit count should increase monotonically.
//
// WUFFS_VERSION was overridden by "wuffs gen -version" based on revision
// 3ff4bf46746fff230d4704f8c5a35a1665ddbd56 committed on 2022-06-03.
// fac934b27259f37191285a8d9d1d9629ed2d9f5d committed on 2022-06-29.
#define WUFFS_VERSION 0x000030000
#define WUFFS_VERSION_MAJOR 0
#define WUFFS_VERSION_MINOR 3
#define WUFFS_VERSION_PATCH 0
#define WUFFS_VERSION_PRE_RELEASE_LABEL "beta.15"
#define WUFFS_VERSION_BUILD_METADATA_COMMIT_COUNT 3345
#define WUFFS_VERSION_BUILD_METADATA_COMMIT_DATE 20220603
#define WUFFS_VERSION_STRING "0.3.0-beta.15+3345.20220603"
#define WUFFS_VERSION_PRE_RELEASE_LABEL "rc.1"
#define WUFFS_VERSION_BUILD_METADATA_COMMIT_COUNT 3360
#define WUFFS_VERSION_BUILD_METADATA_COMMIT_DATE 20220629
#define WUFFS_VERSION_STRING "0.3.0-rc.1+3360.20220629"
// ---------------- Configuration
@@ -1065,6 +1065,22 @@ wuffs_base__count_leading_zeroes_u64(uint64_t u) {
// --------
// Normally, the wuffs_base__peek_etc and wuffs_base__poke_etc implementations
// are both (1) correct regardless of CPU endianness and (2) very fast (e.g. an
// inlined wuffs_base__peek_u32le__no_bounds_check call, in an optimized clang
// or gcc build, is a single MOV instruction on x86_64).
//
// However, the endian-agnostic implementations are slow on Microsoft's C
// compiler (MSC). Alternative memcpy-based implementations restore speed, but
// they are only correct on little-endian CPU architectures. Defining
// WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE opts in to these implementations.
//
// https://godbolt.org/z/q4MfjzTPh
#if defined(_MSC_VER) && !defined(__clang__) && \
(defined(_M_ARM64) || defined(_M_X64))
#define WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE
#endif
#define wuffs_base__peek_u8be__no_bounds_check \
wuffs_base__peek_u8__no_bounds_check
#define wuffs_base__peek_u8le__no_bounds_check \
@@ -1077,12 +1093,24 @@ wuffs_base__peek_u8__no_bounds_check(const uint8_t* p) {
static inline uint16_t //
wuffs_base__peek_u16be__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint16_t x;
memcpy(&x, p, 2);
return _byteswap_ushort(x);
#else
return (uint16_t)(((uint16_t)(p[0]) << 8) | ((uint16_t)(p[1]) << 0));
#endif
}
static inline uint16_t //
wuffs_base__peek_u16le__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint16_t x;
memcpy(&x, p, 2);
return x;
#else
return (uint16_t)(((uint16_t)(p[0]) << 0) | ((uint16_t)(p[1]) << 8));
#endif
}
static inline uint32_t //
@@ -1099,14 +1127,26 @@ wuffs_base__peek_u24le__no_bounds_check(const uint8_t* p) {
static inline uint32_t //
wuffs_base__peek_u32be__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint32_t x;
memcpy(&x, p, 4);
return _byteswap_ulong(x);
#else
return ((uint32_t)(p[0]) << 24) | ((uint32_t)(p[1]) << 16) |
((uint32_t)(p[2]) << 8) | ((uint32_t)(p[3]) << 0);
#endif
}
static inline uint32_t //
wuffs_base__peek_u32le__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint32_t x;
memcpy(&x, p, 4);
return x;
#else
return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) |
((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24);
#endif
}
static inline uint64_t //
@@ -1155,18 +1195,30 @@ wuffs_base__peek_u56le__no_bounds_check(const uint8_t* p) {
static inline uint64_t //
wuffs_base__peek_u64be__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint64_t x;
memcpy(&x, p, 8);
return _byteswap_uint64(x);
#else
return ((uint64_t)(p[0]) << 56) | ((uint64_t)(p[1]) << 48) |
((uint64_t)(p[2]) << 40) | ((uint64_t)(p[3]) << 32) |
((uint64_t)(p[4]) << 24) | ((uint64_t)(p[5]) << 16) |
((uint64_t)(p[6]) << 8) | ((uint64_t)(p[7]) << 0);
#endif
}
static inline uint64_t //
wuffs_base__peek_u64le__no_bounds_check(const uint8_t* p) {
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE)
uint64_t x;
memcpy(&x, p, 8);
return x;
#else
return ((uint64_t)(p[0]) << 0) | ((uint64_t)(p[1]) << 8) |
((uint64_t)(p[2]) << 16) | ((uint64_t)(p[3]) << 24) |
((uint64_t)(p[4]) << 32) | ((uint64_t)(p[5]) << 40) |
((uint64_t)(p[6]) << 48) | ((uint64_t)(p[7]) << 56);
#endif
}
// --------
@@ -1189,7 +1241,8 @@ wuffs_base__poke_u16be__no_bounds_check(uint8_t* p, uint16_t x) {
static inline void //
wuffs_base__poke_u16le__no_bounds_check(uint8_t* p, uint16_t x) {
#if defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__)
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE) || \
(defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__))
// This seems to perform better on gcc 10 (but not clang 9). Clang also
// defines "__GNUC__".
memcpy(p, &x, 2);
@@ -1223,7 +1276,8 @@ wuffs_base__poke_u32be__no_bounds_check(uint8_t* p, uint32_t x) {
static inline void //
wuffs_base__poke_u32le__no_bounds_check(uint8_t* p, uint32_t x) {
#if defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__)
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE) || \
(defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__))
// This seems to perform better on gcc 10 (but not clang 9). Clang also
// defines "__GNUC__".
memcpy(p, &x, 4);
@@ -1309,7 +1363,8 @@ wuffs_base__poke_u64be__no_bounds_check(uint8_t* p, uint64_t x) {
static inline void //
wuffs_base__poke_u64le__no_bounds_check(uint8_t* p, uint64_t x) {
#if defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__)
#if defined(WUFFS_BASE__USE_MEMCPY_LE_PEEK_POKE) || \
(defined(__GNUC__) && !defined(__clang__) && defined(__x86_64__))
// This seems to perform better on gcc 10 (but not clang 9). Clang also
// defines "__GNUC__".
memcpy(p, &x, 8);
@@ -6828,9 +6883,7 @@ extern const char wuffs_bzip2__error__bad_block_header[];
extern const char wuffs_bzip2__error__bad_block_length[];
extern const char wuffs_bzip2__error__bad_checksum[];
extern const char wuffs_bzip2__error__bad_header[];
extern const char wuffs_bzip2__error__bad_number_of_huffman_codes[];
extern const char wuffs_bzip2__error__bad_number_of_sections[];
extern const char wuffs_bzip2__error__unsupported_huffman_code[];
extern const char wuffs_bzip2__error__unsupported_block_randomization[];
// ---------------- Public Consts
@@ -6968,7 +7021,7 @@ struct wuffs_bzip2__decoder__struct {
uint8_t f_presence[256];
uint8_t f_mtft[256];
uint8_t f_huffman_selectors[32768];
uint16_t f_huffman_trees[6][1024][2];
uint16_t f_huffman_trees[6][257][2];
uint16_t f_huffman_tables[6][256];
uint32_t f_bwt[1048576];
@@ -24867,9 +24920,7 @@ const char wuffs_bzip2__error__bad_block_header[] = "#bzip2: bad block header";
const char wuffs_bzip2__error__bad_block_length[] = "#bzip2: bad block length";
const char wuffs_bzip2__error__bad_checksum[] = "#bzip2: bad checksum";
const char wuffs_bzip2__error__bad_header[] = "#bzip2: bad header";
const char wuffs_bzip2__error__bad_number_of_huffman_codes[] = "#bzip2: bad number of Huffman codes";
const char wuffs_bzip2__error__bad_number_of_sections[] = "#bzip2: bad number of sections";
const char wuffs_bzip2__error__unsupported_huffman_code[] = "#bzip2: unsupported Huffman code";
const char wuffs_bzip2__error__unsupported_block_randomization[] = "#bzip2: unsupported block randomization";
const char wuffs_bzip2__error__internal_error_inconsistent_huffman_decoder_state[] = "#bzip2: internal error: inconsistent Huffman decoder state";
@@ -25805,7 +25856,7 @@ wuffs_bzip2__decoder__build_huffman_tree(
uint32_t a_which) {
uint32_t v_code_length = 0;
uint32_t v_symbol_index = 0;
uint32_t v_num_nodes = 0;
uint32_t v_num_branch_nodes = 0;
uint32_t v_stack_height = 0;
uint32_t v_stack_values[21] = {0};
uint32_t v_node_index = 0;
@@ -25813,7 +25864,7 @@ wuffs_bzip2__decoder__build_huffman_tree(
self->private_data.f_huffman_trees[a_which][0][0] = 0;
self->private_data.f_huffman_trees[a_which][0][1] = 0;
v_num_nodes = 1;
v_num_branch_nodes = 1;
v_stack_height = 1;
v_stack_values[0] = 0;
v_code_length = 1;
@@ -25838,27 +25889,27 @@ wuffs_bzip2__decoder__build_huffman_tree(
}
v_node_index = v_stack_values[(v_stack_height - 1)];
if (self->private_data.f_huffman_trees[a_which][v_node_index][0] == 0) {
self->private_data.f_huffman_trees[a_which][v_node_index][0] = ((uint16_t)(v_num_nodes));
self->private_data.f_huffman_trees[a_which][v_node_index][0] = ((uint16_t)(v_num_branch_nodes));
} else {
self->private_data.f_huffman_trees[a_which][v_node_index][1] = ((uint16_t)(v_num_nodes));
self->private_data.f_huffman_trees[a_which][v_node_index][1] = ((uint16_t)(v_num_branch_nodes));
}
if (v_num_nodes >= 1023) {
return wuffs_base__make_status(wuffs_bzip2__error__unsupported_huffman_code);
if (v_num_branch_nodes >= 257) {
return wuffs_base__make_status(wuffs_bzip2__error__bad_huffman_code_under_subscribed);
}
v_stack_values[v_stack_height] = v_num_nodes;
self->private_data.f_huffman_trees[a_which][v_num_nodes][0] = 0;
self->private_data.f_huffman_trees[a_which][v_num_nodes][1] = 0;
v_num_nodes += 1;
v_stack_values[v_stack_height] = v_num_branch_nodes;
self->private_data.f_huffman_trees[a_which][v_num_branch_nodes][0] = 0;
self->private_data.f_huffman_trees[a_which][v_num_branch_nodes][1] = 0;
v_num_branch_nodes += 1;
v_stack_height += 1;
}
label__2__break:;
v_node_index = v_stack_values[(v_stack_height - 1)];
if (v_symbol_index < 2) {
v_leaf_value = ((uint16_t)((1280 + v_symbol_index)));
v_leaf_value = ((uint16_t)((769 + v_symbol_index)));
} else if ((v_symbol_index + 1) < self->private_impl.f_num_symbols) {
v_leaf_value = ((uint16_t)((1023 + v_symbol_index)));
v_leaf_value = ((uint16_t)((511 + v_symbol_index)));
} else {
v_leaf_value = 2047;
v_leaf_value = 768;
}
if (self->private_data.f_huffman_trees[a_which][v_node_index][0] == 0) {
self->private_data.f_huffman_trees[a_which][v_node_index][0] = v_leaf_value;
@@ -25899,7 +25950,7 @@ wuffs_bzip2__decoder__build_huffman_table(
v_bits = (v_i << 24);
v_n_bits = 0;
v_child = 0;
while ((v_child < 1024) && (v_n_bits < 8)) {
while ((v_child < 257) && (v_n_bits < 8)) {
v_child = self->private_data.f_huffman_trees[a_which][v_child][(v_bits >> 31)];
v_bits <<= 1;
#if defined(__GNUC__)
@@ -26193,6 +26244,10 @@ wuffs_bzip2__decoder__decode_huffman_fast(
} else {
v_ticks = 49;
v_section += 1;
if (v_section >= self->private_impl.f_num_sections) {
status = wuffs_base__make_status(wuffs_bzip2__error__bad_number_of_sections);
goto exit;
}
v_which = WUFFS_BZIP2__CLAMP_TO_5[(self->private_data.f_huffman_selectors[(v_section & 32767)] & 7)];
}
v_bits |= (wuffs_base__peek_u32be__no_bounds_check(iop_a_src) >> v_n_bits);
@@ -26201,8 +26256,8 @@ wuffs_bzip2__decoder__decode_huffman_fast(
v_table_entry = self->private_data.f_huffman_tables[v_which][(v_bits >> 24)];
v_bits <<= (v_table_entry >> 12);
v_n_bits -= ((uint32_t)((v_table_entry >> 12)));
v_child = (v_table_entry & 2047);
while (v_child < 1024) {
v_child = (v_table_entry & 1023);
while (v_child < 257) {
v_child = self->private_data.f_huffman_trees[v_which][v_child][(v_bits >> 31)];
v_bits <<= 1;
if (v_n_bits <= 0) {
@@ -26211,7 +26266,7 @@ wuffs_bzip2__decoder__decode_huffman_fast(
}
v_n_bits -= 1;
}
if (v_child < 1280) {
if (v_child < 768) {
v_child_ff = ((uint32_t)((v_child & 255)));
v_output = ((uint32_t)(self->private_data.f_mtft[v_child_ff]));
wuffs_base__slice_u8__copy_from_slice(wuffs_base__make_slice_u8_ij(self->private_data.f_mtft, 1, (1 + v_child_ff)), wuffs_base__make_slice_u8(self->private_data.f_mtft, v_child_ff));
@@ -26225,7 +26280,7 @@ wuffs_bzip2__decoder__decode_huffman_fast(
v_block_size += 1;
v_run_shift = 0;
goto label__outer__continue;
} else if (v_child > 1281) {
} else if (v_child == 768) {
self->private_impl.f_decode_huffman_finished = true;
goto label__outer__break;
}
@@ -26233,7 +26288,7 @@ wuffs_bzip2__decoder__decode_huffman_fast(
status = wuffs_base__make_status(wuffs_bzip2__error__bad_block_length);
goto exit;
}
v_run = (((uint32_t)((v_child - 1279))) << v_run_shift);
v_run = ((((uint32_t)(v_child)) & 3) << v_run_shift);
v_run_shift += 1;
v_i = v_block_size;
v_j = (v_run + v_block_size);
@@ -26312,6 +26367,10 @@ wuffs_bzip2__decoder__decode_huffman_slow(
} else {
self->private_impl.f_decode_huffman_ticks = 49;
self->private_impl.f_decode_huffman_section += 1;
if (self->private_impl.f_decode_huffman_section >= self->private_impl.f_num_sections) {
status = wuffs_base__make_status(wuffs_bzip2__error__bad_number_of_sections);
goto exit;
}
self->private_impl.f_decode_huffman_which = WUFFS_BZIP2__CLAMP_TO_5[(self->private_data.f_huffman_selectors[(self->private_impl.f_decode_huffman_section & 32767)] & 7)];
}
v_node_index = 0;
@@ -26333,10 +26392,10 @@ wuffs_bzip2__decoder__decode_huffman_slow(
v_child = self->private_data.f_huffman_trees[self->private_impl.f_decode_huffman_which][v_node_index][(self->private_impl.f_bits >> 31)];
self->private_impl.f_bits <<= 1;
self->private_impl.f_n_bits -= 1;
if (v_child < 1024) {
if (v_child < 257) {
v_node_index = ((uint32_t)(v_child));
goto label__0__continue;
} else if (v_child < 1280) {
} else if (v_child < 768) {
v_child_ff = ((uint32_t)((v_child & 255)));
v_output = ((uint32_t)(self->private_data.f_mtft[v_child_ff]));
wuffs_base__slice_u8__copy_from_slice(wuffs_base__make_slice_u8_ij(self->private_data.f_mtft, 1, (1 + v_child_ff)), wuffs_base__make_slice_u8(self->private_data.f_mtft, v_child_ff));
@@ -26350,7 +26409,7 @@ wuffs_bzip2__decoder__decode_huffman_slow(
self->private_impl.f_block_size += 1;
self->private_impl.f_decode_huffman_run_shift = 0;
goto label__0__break;
} else if (v_child > 1281) {
} else if (v_child == 768) {
self->private_impl.f_decode_huffman_finished = true;
goto label__outer__break;
}
@@ -26358,7 +26417,7 @@ wuffs_bzip2__decoder__decode_huffman_slow(
status = wuffs_base__make_status(wuffs_bzip2__error__bad_block_length);
goto exit;
}
v_run = (((uint32_t)((v_child - 1279))) << self->private_impl.f_decode_huffman_run_shift);
v_run = ((((uint32_t)(v_child)) & 3) << self->private_impl.f_decode_huffman_run_shift);
self->private_impl.f_decode_huffman_run_shift += 1;
v_i = self->private_impl.f_block_size;
v_j = (v_run + self->private_impl.f_block_size);
+69
View File
@@ -0,0 +1,69 @@
// Copyright 2022 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build ignore
// +build ignore
package main
// litonlylzma.go converts to and from the Literal Only LZMA compressed file
// format, a subset of the LZMA compressed file format (e.g. as spoken by
// /usr/bin/lzma on a Debian system).
//
// Usage:
// go run litonlylzma.go -encode < foo.txt > foo.txt.litonlylzma
// go run litonlylzma.go -decode < foo.txt.litonlylzma
import (
"flag"
"fmt"
"io"
"os"
"github.com/google/wuffs/lib/litonlylzma"
)
var (
decode = flag.Bool("decode", false, "decode from LZMA")
encode = flag.Bool("encode", false, "encode to LZMA")
)
func main() {
if err := main1(); err != nil {
os.Stderr.WriteString(err.Error() + "\n")
os.Exit(1)
}
}
func main1() error {
flag.Parse()
if *decode == *encode {
return fmt.Errorf("exactly one of -decode and -encode needed")
}
src, err := io.ReadAll(os.Stdin)
if err != nil {
return err
}
dst, err := []byte(nil), error(nil)
if *decode {
dst, _, err = litonlylzma.FileFormatLZMA.Decode(nil, src)
} else {
dst, err = litonlylzma.FileFormatLZMA.Encode(nil, src)
}
if err != nil {
return err
}
_, err = os.Stdout.Write(dst)
return err
}
+112 -28
View File
@@ -88,7 +88,7 @@ func main1() error {
const rep = "repeat "
if strings.HasPrefix(s, rep) {
args := s[len(rep):]
count, args, ok := parseNum(args)
count, args, ok := parseNum32(args)
if !ok || count <= 0 || args != "[" {
return fmt.Errorf("bad repeat command: %q", s)
}
@@ -157,7 +157,7 @@ func log2(u uint32) (i int32) {
return -1
}
func parseHex(s string) (num uint32, remaining string, ok bool) {
func parseHex32(s string) (num uint32, remaining string, ok bool) {
if i := strings.IndexByte(s, ' '); i >= 0 {
s, remaining = s[:i], s[i+1:]
for len(remaining) > 0 && remaining[0] == ' ' {
@@ -177,7 +177,27 @@ func parseHex(s string) (num uint32, remaining string, ok bool) {
return uint32(u), remaining, true
}
func parseNum(s string) (num uint32, remaining string, ok bool) {
func parseHex64(s string) (num uint64, remaining string, ok bool) {
if i := strings.IndexByte(s, ' '); i >= 0 {
s, remaining = s[:i], s[i+1:]
for len(remaining) > 0 && remaining[0] == ' ' {
remaining = remaining[1:]
}
}
if len(s) < 2 || s[0] != '0' || s[1] != 'x' {
return 0, "", false
}
s = s[2:]
u, err := strconv.ParseUint(s, 16, 64)
if err != nil {
return 0, "", false
}
return u, remaining, true
}
func parseNum32(s string) (num uint32, remaining string, ok bool) {
if i := strings.IndexByte(s, ' '); i >= 0 {
s, remaining = s[:i], s[i+1:]
for len(remaining) > 0 && remaining[0] == ' ' {
@@ -237,6 +257,70 @@ var reverse8 = [256]byte{
// ----
func init() {
formats["bzip2"] = stateBzip2
}
var bzip2Globals struct {
stream bzip2BitStream
}
func stateBzip2(line string) (stateFunc, error) {
g := &bzip2Globals
const (
cmdB = "bits "
)
switch {
case line == "":
g.stream.flush()
return stateBzip2, nil
case strings.HasPrefix(line, cmdB):
s := strings.TrimSpace(line[len(cmdB):])
n, s, ok := parseNum32(s)
if !ok || s == "" {
break
}
x, s, ok := parseHex64(s)
if !ok {
break
}
g.stream.writeBits64(x, n)
return stateBzip2, nil
}
return nil, fmt.Errorf("bad stateBzip2 command: %q", line)
}
type bzip2BitStream struct {
bits uint64
nBits uint32 // Always within [0, 7].
}
// writeBits64 writes the low n bits of b to z.
func (z *bzip2BitStream) writeBits64(b uint64, n uint32) {
if n > 56 {
panic("writeBits64: n is too large")
}
z.bits |= b << (64 - n - z.nBits)
z.nBits += n
for z.nBits >= 8 {
out = append(out, uint8(z.bits>>56))
z.bits <<= 8
z.nBits -= 8
}
}
func (z *bzip2BitStream) flush() {
if z.nBits > 0 {
out = append(out, uint8(z.bits>>56))
z.bits = 0
z.nBits = 0
}
}
// ----
func init() {
formats["deflate"] = stateDeflate
}
@@ -529,7 +613,7 @@ func stateDeflate(line string) (stateFunc, error) {
s := line[len(cmdB):]
for s != "" {
x, ok := uint32(0), false
x, s, ok = parseHex(s)
x, s, ok = parseHex32(s)
if !ok {
return nil, fmt.Errorf("bad stateDeflate command: %q", line)
}
@@ -728,7 +812,7 @@ outer:
default:
s := line
n, s, ok := parseNum(s)
n, s, ok := parseNum32(s)
if !ok || s == "" {
break
}
@@ -743,7 +827,7 @@ outer:
if g.etcetera {
g.etcetera = false
n0, s0, ok := parseNum(g.prevLine)
n0, s0, ok := parseNum32(g.prevLine)
if !ok {
return nil, fmt.Errorf("bad etcetera command")
}
@@ -965,10 +1049,10 @@ func deflateParseLenDist(line string) (l uint32, d uint32, ok bool) {
s := line
if strings.HasPrefix(s, lStr) {
s = s[len(lStr):]
if l, s, ok := parseNum(s); ok && 3 <= l && l <= 258 {
if l, s, ok := parseNum32(s); ok && 3 <= l && l <= 258 {
if strings.HasPrefix(s, dStr) {
s = s[len(dStr):]
if d, s, ok := parseNum(s); ok && 1 <= d && d <= 32768 && s == "" {
if d, s, ok := parseNum32(s); ok && 1 <= d && d <= 32768 && s == "" {
return l, d, true
}
}
@@ -1035,7 +1119,7 @@ outer:
s := line[len(cmdB):]
for s != "" {
x, ok := uint32(0), false
x, s, ok = parseHex(s)
x, s, ok = parseHex32(s)
if !ok {
break outer
}
@@ -1076,7 +1160,7 @@ outer:
flags |= 0x01
transparentIndex = uint8(num)
case strings.HasSuffix(term, ms):
num, remaining, ok := parseNum(term[:len(term)-len(ms)])
num, remaining, ok := parseNum32(term[:len(term)-len(ms)])
if !ok || remaining != "" {
break outer
}
@@ -1098,7 +1182,7 @@ outer:
case strings.HasPrefix(line, cmdL):
s := line[len(cmdL):]
litWidth, s, ok := parseNum(s)
litWidth, s, ok := parseNum32(s)
if !ok || litWidth < 2 || 8 < litWidth {
break
}
@@ -1107,7 +1191,7 @@ outer:
uncompressed := []byte(nil)
for s != "" {
x := uint32(0)
x, s, ok = parseHex(s)
x, s, ok = parseHex32(s)
if !ok {
break outer
}
@@ -1140,7 +1224,7 @@ outer:
case strings.HasPrefix(line, cmdLC):
s := line[len(cmdLC):]
loopCount, _, ok := parseNum(s)
loopCount, _, ok := parseNum32(s)
if !ok || 0xFFFF < loopCount {
break
}
@@ -1191,15 +1275,15 @@ func stateGifImage(line string) (stateFunc, error) {
switch {
case strings.HasPrefix(line, cmdBCI):
s := line[len(cmdBCI):]
if i, _, ok := parseNum(s); ok {
if i, _, ok := parseNum32(s); ok {
g.imageBackgroundColorIndex = i
}
return stateGifImage, nil
case strings.HasPrefix(line, cmdIWH):
s := line[len(cmdIWH):]
if w, s, ok := parseNum(s); ok {
if h, _, ok := parseNum(s); ok {
if w, s, ok := parseNum32(s); ok {
if h, _, ok := parseNum32(s); ok {
g.imageWidth = w
g.imageHeight = h
return stateGifImage, nil
@@ -1248,10 +1332,10 @@ func stateGifFrame(line string) (stateFunc, error) {
case strings.HasPrefix(line, cmdFLTWH):
s := line[len(cmdFLTWH):]
if l, s, ok := parseNum(s); ok {
if t, s, ok := parseNum(s); ok {
if w, s, ok := parseNum(s); ok {
if h, _, ok := parseNum(s); ok {
if l, s, ok := parseNum32(s); ok {
if t, s, ok := parseNum32(s); ok {
if w, s, ok := parseNum32(s); ok {
if h, _, ok := parseNum32(s); ok {
g.frameLeft = l
g.frameTop = t
g.frameWidth = w
@@ -1276,9 +1360,9 @@ func stateGifImagePalette(line string) (stateFunc, error) {
}
s := line
if rgb0, s, ok := parseHex(s); ok {
if rgb1, s, ok := parseHex(s); ok {
if rgb2, _, ok := parseHex(s); ok {
if rgb0, s, ok := parseHex32(s); ok {
if rgb1, s, ok := parseHex32(s); ok {
if rgb2, _, ok := parseHex32(s); ok {
g.globalPalette = append(g.globalPalette,
[4]uint8{uint8(rgb0), uint8(rgb1), uint8(rgb2), 0xFF})
return stateGifImagePalette, nil
@@ -1296,9 +1380,9 @@ func stateGifFramePalette(line string) (stateFunc, error) {
}
s := line
if rgb0, s, ok := parseHex(s); ok {
if rgb1, s, ok := parseHex(s); ok {
if rgb2, _, ok := parseHex(s); ok {
if rgb0, s, ok := parseHex32(s); ok {
if rgb1, s, ok := parseHex32(s); ok {
if rgb2, _, ok := parseHex32(s); ok {
g.localPalette = append(g.localPalette,
[4]uint8{uint8(rgb0), uint8(rgb1), uint8(rgb2), 0xFF})
return stateGifFramePalette, nil
@@ -1381,7 +1465,7 @@ func statePngRaw(line string) (stateFunc, error) {
}
for s := line; s != ""; {
if x, remaining, ok := parseHex(s); ok {
if x, remaining, ok := parseHex32(s); ok {
g.chunkData.WriteByte(byte(x))
s = remaining
} else {
@@ -1401,7 +1485,7 @@ func statePngZlib(line string) (stateFunc, error) {
}
for s := line; s != ""; {
if x, remaining, ok := parseHex(s); ok {
if x, remaining, ok := parseHex32(s); ok {
g.scratch[0] = byte(x)
g.zlibWriter.Write(g.scratch[:1])
s = remaining
+62 -52
View File
@@ -18,9 +18,7 @@ pub status "#bad block header"
pub status "#bad block length"
pub status "#bad checksum"
pub status "#bad header"
pub status "#bad number of Huffman codes"
pub status "#bad number of sections"
pub status "#unsupported Huffman code"
pub status "#unsupported block randomization"
pri status "#internal error: inconsistent Huffman decoder state"
@@ -74,20 +72,30 @@ pub struct decoder? implements base.io_transformer(
// 50-symbol section.
huffman_selectors : array[32768] base.u8,
// Each array[2] base.u16 is a Huffman tree node. The build_huffman_tree
// method comment details what the base.u16 values mean.
// Each array[2] base.u16 is a branch node in the Huffman tree. The
// build_huffman_tree method comment details what the base.u16 values mean.
//
// 1024 is just a guess for the worst case number of nodes needed. TODO: be
// more rigorous about deriving this upper bound.
huffman_trees : array[6] array[1024] array[2] base.u16,
// Each Huffman tree has num_symbols leaf nodes and hence (num_symbols - 1)
// branch nodes, also called internal nodes, as per
// https://stackoverflow.com/questions/69399994/maximum-number-of-nodes-in-a-huffman-tree/69401083#69401083
//
// This can be proven inductively. The base case Huffman code (two symbols)
// has one branch node (the root node). The inductive case, adding one
// symbol to a (complete but possibly non-canonical) Huffman code replaces
// one leaf node with one branch node (and two leaf nodes). The net gain in
// branch nodes is +1.
//
// Since num_symbols' inclusive maximum is 258 (for RUNA, RUNB, 255 × mXYZ,
// EOB), there are at most 257 branch nodes.
huffman_trees : array[6] array[257] array[2] base.u16,
// huffman_tables[which] caches the result of walking huffman_trees[which]
// for each possible 8-bit input. The base.u16 value decomposes as:
// - The low 11 bits are the final node, where finality comes either from
// - The low 10 bits are the final node, where finality comes either from
// hitting a leaf node or from spending all 8 bits of the input. As per
// the build_huffman_tree method, a value of 0x3FF = 1023 or less means
// a branch node and 0x400 = 1024 or more means a leaf node.
// - The middle bit is unused.
// the build_huffman_tree method, a value of 0x100 = 256 or less means
// a branch node and 0x201 = 513 or more means a leaf node.
// - The middle 2 bits are unused.
// - The high 4 bits are the number of input bits spent.
huffman_tables : array[6] array[256] base.u16,
@@ -536,29 +544,31 @@ pri func decoder.read_code_lengths?(src: base.io_reader) {
// build_huffman_tree builds a canonical Huffman tree given the symbols' code
// lengths. For the "abraca.txt.bz2" example, the code lengths are:
// - symbol=0x500 (RUNA) cl=3
// - symbol=0x501 (RUNB) cl=3
// - symbol=0x401 (m01) cl=2
// - symbol=0x402 (m02) cl=3
// - symbol=0x403 (m03) cl=2
// - symbol=0x7FF (EOB) cl=3
// - symbol=0x301 (RUNA) cl=3
// - symbol=0x302 (RUNB) cl=3
// - symbol=0x201 (m01) cl=2
// - symbol=0x202 (m02) cl=3
// - symbol=0x203 (m03) cl=2
// - symbol=0x300 (EOB) cl=3
//
// The bitstring mapping is:
// - "00" 0x401 (m01)
// - "01" 0x403 (m03)
// - "100" 0x500 (RUNA)
// - "101" 0x501 (RUNB)
// - "110" 0x402 (m02)
// - "111" 0x7FF (EOB)
// - "00" 0x201 (m01)
// - "01" 0x203 (m03)
// - "100" 0x301 (RUNA)
// - "101" 0x302 (RUNB)
// - "110" 0x202 (m02)
// - "111" 0x300 (EOB)
//
// The Huffman tree has 5 nodes. Each node has two children, represented by a
// base.u16 value. 1023 or less means that the child is a branch node. 1024 =
// 0x400 or above means a leaf node (with the symbol values as above).
// The Huffman tree has 5 branch nodes (where 5+1 is both the number of leaf
// nodes and the number of symbols). Each branch node has two children,
// represented by a base.u16 value. 0x100 = 256 or less means that the child is
// a branch node. 0x201 ..= 0x2FF means an mNN leaf node. 0x300, 0x301 and
// 0x302 mean EOB, RUNA and RUNB leaf nodes.
// - node0 = [0x001 (node1), 0x002 (node2)]
// - node1 = [0x401 (m01) , 0x403 (m03) ]
// - node1 = [0x201 (m01) , 0x203 (m03) ]
// - node2 = [0x003 (node3), 0x004 (node4)]
// - node3 = [0x500 (RUNA) , 0x501 (RUNB) ]
// - node4 = [0x402 (m02) , 0x7FF (EOB) ]
// - node3 = [0x301 (RUNA) , 0x302 (RUNB) ]
// - node4 = [0x202 (m02) , 0x300 (EOB) ]
//
// For example, the tree walk from node0 right (1) to node2 left (0) to node3
// left (0) to RUNA means that the RUNA bitstring is "100".
@@ -567,30 +577,30 @@ pri func decoder.read_code_lengths?(src: base.io_reader) {
// length order, maintaining a stack of nodes that's the path from the root
// (node0) to the latest created-but-unfinished node. Created means that their
// left child is assigned (non-zero). Unfinished means that their right child
// is pending (zero). For example, after adding the 0x401, 0x403 and 0x500
// is pending (zero). For example, after adding the 0x201, 0x203 and 0x301
// symbols, we would have this (in-progress) tree:
// - node0 = [0x001 (node1), 0x002 (node2)]
// - node1 = [0x401 (m01) , 0x403 (m03) ]
// - node1 = [0x201 (m01) , 0x203 (m03) ]
// - node2 = [0x003 (node3), 0 ]
// - node3 = [0x500 (RUNA) , 0 ]
// - node3 = [0x301 (RUNA) , 0 ]
//
// The stack would be [node0, node2, node3]. The next symbol would fill in the
// top of the stack, node3. If that symbol's code length matches the stack
// height (here, 3) then it would pop the stack. Otherwise it would push new
// nodes onto the stack until the stack height matched that code length.
pri func decoder.build_huffman_tree!(which: base.u32[..= 5]) base.status {
var code_length : base.u32
var symbol_index : base.u32
var num_nodes : base.u32[..= 1023]
var stack_height : base.u32[..= 21]
var stack_values : array[21] base.u32[..= 1023]
var node_index : base.u32[..= 1023]
var leaf_value : base.u16
var code_length : base.u32
var symbol_index : base.u32
var num_branch_nodes : base.u32[..= 257]
var stack_height : base.u32[..= 21]
var stack_values : array[21] base.u32[..= 256]
var node_index : base.u32[..= 256]
var leaf_value : base.u16
// Push a root node.
this.huffman_trees[args.which][0][0] = 0
this.huffman_trees[args.which][0][1] = 0
num_nodes = 1
num_branch_nodes = 1
stack_height = 1
stack_values[0] = 0
@@ -624,27 +634,27 @@ pri func decoder.build_huffman_tree!(which: base.u32[..= 5]) base.status {
node_index = stack_values[stack_height - 1]
if this.huffman_trees[args.which][node_index][0] == 0 {
this.huffman_trees[args.which][node_index][0] = num_nodes as base.u16
this.huffman_trees[args.which][node_index][0] = num_branch_nodes as base.u16
} else {
this.huffman_trees[args.which][node_index][1] = num_nodes as base.u16
this.huffman_trees[args.which][node_index][1] = num_branch_nodes as base.u16
}
if num_nodes >= 1023 {
return "#unsupported Huffman code"
if num_branch_nodes >= 257 {
return "#bad Huffman code (under-subscribed)"
}
stack_values[stack_height] = num_nodes
this.huffman_trees[args.which][num_nodes][0] = 0
this.huffman_trees[args.which][num_nodes][1] = 0
num_nodes += 1
stack_values[stack_height] = num_branch_nodes
this.huffman_trees[args.which][num_branch_nodes][0] = 0
this.huffman_trees[args.which][num_branch_nodes][1] = 0
num_branch_nodes += 1
stack_height += 1
} endwhile
node_index = stack_values[stack_height - 1]
if symbol_index < 2 {
leaf_value = (0x500 + symbol_index) as base.u16
leaf_value = (0x301 + symbol_index) as base.u16
} else if (symbol_index + 1) < this.num_symbols {
leaf_value = (0x3FF + symbol_index) as base.u16
leaf_value = (0x1FF + symbol_index) as base.u16
} else {
leaf_value = 0x7FF
leaf_value = 0x300
}
if this.huffman_trees[args.which][node_index][0] == 0 {
@@ -685,7 +695,7 @@ pri func decoder.build_huffman_table!(which: base.u32[..= 5]) {
bits = i << 24
n_bits = 0
child = 0
while (child < 1024) and (n_bits < 8),
while (child < 257) and (n_bits < 8),
inv i < 256,
{
child = this.huffman_trees[args.which][child][bits >> 31]
+9 -6
View File
@@ -27,7 +27,7 @@ pri func decoder.decode_huffman_fast!(src: base.io_reader) base.status {
var i : base.u32
var j : base.u32
var output : base.u32[..= 255]
var run : base.u32[..= 16_777216] // 16_777216 = (2 << 23)
var run : base.u32[..= 12_582912] // 12_582912 = (3 << 22)
var mtft0 : base.u32[..= 255]
bits = this.bits
@@ -45,6 +45,9 @@ pri func decoder.decode_huffman_fast!(src: base.io_reader) base.status {
} else {
ticks = 49
section ~mod+= 1
if section >= this.num_sections {
return "#bad number of sections"
}
which = CLAMP_TO_5[this.huffman_selectors[section & 32767] & 7]
}
@@ -55,8 +58,8 @@ pri func decoder.decode_huffman_fast!(src: base.io_reader) base.status {
table_entry = this.huffman_tables[which][bits >> 24]
bits ~mod<<= table_entry >> 12
n_bits -= (table_entry >> 12) as base.u32
child = table_entry & 0x7FF
while child < 1024 {
child = table_entry & 0x3FF
while child < 257 {
child = this.huffman_trees[which][child][bits >> 31]
bits ~mod<<= 1
if n_bits <= 0 {
@@ -65,7 +68,7 @@ pri func decoder.decode_huffman_fast!(src: base.io_reader) base.status {
n_bits -= 1
} endwhile
if child < 1280 { // mNN symbol.
if child < 0x300 { // mNN symbol.
// Move to front.
child_ff = (child & 0xFF) as base.u32
output = this.mtft[child_ff] as base.u32
@@ -82,7 +85,7 @@ pri func decoder.decode_huffman_fast!(src: base.io_reader) base.status {
block_size += 1
run_shift = 0
continue.outer
} else if child > 1281 { // EOB symbol.
} else if child == 0x300 { // EOB symbol.
this.decode_huffman_finished = true
break.outer
}
@@ -91,7 +94,7 @@ pri func decoder.decode_huffman_fast!(src: base.io_reader) base.status {
if run_shift >= 23 {
return "#bad block length"
}
run = ((child - 1279) as base.u32) << run_shift
run = ((child as base.u32) & 3) << run_shift
run_shift += 1
i = block_size
j = run + block_size
+9 -6
View File
@@ -14,14 +14,14 @@
pri func decoder.decode_huffman_slow?(src: base.io_reader) {
var c : base.u8
var node_index : base.u32[..= 1023]
var node_index : base.u32[..= 256]
var child : base.u16
var child_ff : base.u32[..= 255]
var i : base.u32
var j : base.u32
var output : base.u32[..= 255]
var run : base.u32[..= 16_777216] // 16_777216 = (2 << 23)
var run : base.u32[..= 12_582912] // 12_582912 = (3 << 22)
var mtft0 : base.u32[..= 255]
// Apply the Huffman trees.
@@ -31,6 +31,9 @@ pri func decoder.decode_huffman_slow?(src: base.io_reader) {
} else {
this.decode_huffman_ticks = 49
this.decode_huffman_section ~mod+= 1
if this.decode_huffman_section >= this.num_sections {
return "#bad number of sections"
}
this.decode_huffman_which = CLAMP_TO_5[this.huffman_selectors[this.decode_huffman_section & 32767] & 7]
}
@@ -46,11 +49,11 @@ pri func decoder.decode_huffman_slow?(src: base.io_reader) {
this.bits ~mod<<= 1
this.n_bits -= 1
if child < 1024 { // Branch node.
if child < 257 { // Branch node.
node_index = child as base.u32
continue
} else if child < 1280 { // mNN symbol.
} else if child < 0x300 { // mNN symbol.
// Move to front.
child_ff = (child & 0xFF) as base.u32
output = this.mtft[child_ff] as base.u32
@@ -68,7 +71,7 @@ pri func decoder.decode_huffman_slow?(src: base.io_reader) {
this.decode_huffman_run_shift = 0
break
} else if child > 1281 { // EOB symbol.
} else if child == 0x300 { // EOB symbol.
this.decode_huffman_finished = true
break.outer
}
@@ -77,7 +80,7 @@ pri func decoder.decode_huffman_slow?(src: base.io_reader) {
if this.decode_huffman_run_shift >= 23 {
return "#bad block length"
}
run = ((child - 1279) as base.u32) << this.decode_huffman_run_shift
run = ((child as base.u32) & 3) << this.decode_huffman_run_shift
this.decode_huffman_run_shift += 1
i = this.block_size
j = run + this.block_size
+32
View File
@@ -74,6 +74,16 @@ golden_test g_bzip2_256_bytes_gt = {
.src_filename = "test/data/256.bytes.bz2",
};
golden_test g_bzip2_bad_number_of_sections_gt = {
.want_filename = "test/data/0.bytes",
.src_filename = "test/data/artificial-bzip2/bad-number-of-sections.bz2",
};
golden_test g_bzip2_huffman_258_gt = {
.want_filename = "test/data/abraca.txt",
.src_filename = "test/data/artificial-bzip2/huffman-258.bz2",
};
golden_test g_bzip2_midsummer_gt = {
.want_filename = "test/data/midsummer.txt",
.src_filename = "test/data/midsummer.txt.bz2",
@@ -137,6 +147,26 @@ test_wuffs_bzip2_decode_256_bytes() {
UINT64_MAX, UINT64_MAX);
}
const char* //
test_wuffs_bzip2_decode_bad_number_of_sections() {
CHECK_FOCUS(__func__);
const char* have =
do_test_io_buffers(wuffs_bzip2_decode, &g_bzip2_bad_number_of_sections_gt,
UINT64_MAX, UINT64_MAX);
const char* want = wuffs_bzip2__error__bad_number_of_sections;
if (have != want) {
RETURN_FAIL("have \"%s\", want \"%s\"", have, want);
}
return NULL;
}
const char* //
test_wuffs_bzip2_decode_huffman_258() {
CHECK_FOCUS(__func__);
return do_test_io_buffers(wuffs_bzip2_decode, &g_bzip2_huffman_258_gt,
UINT64_MAX, UINT64_MAX);
}
const char* //
test_wuffs_bzip2_decode_midsummer() {
CHECK_FOCUS(__func__);
@@ -227,6 +257,8 @@ bench_mimic_bzip2_decode_100k() {
proc g_tests[] = {
test_wuffs_bzip2_decode_256_bytes,
test_wuffs_bzip2_decode_bad_number_of_sections,
test_wuffs_bzip2_decode_huffman_258,
test_wuffs_bzip2_decode_interface,
test_wuffs_bzip2_decode_midsummer,
test_wuffs_bzip2_decode_pi,
Binary file not shown.
@@ -0,0 +1,49 @@
# Feed this file to script/make-artificial.go
# This script generates test/data/abraca.txt.bz2 exactly, following the
# description in std/bzip2/README.md and its bitslices (a) ..= (s).
#
# By itself, this isn't a very interesting foo.make-artificial.txt file, in
# that it doesn't make *artificial* bzip2 test data (as opposed to test data
# *naturally* created by the /usr/bin/bzip2 tool). But this file is a useful
# diff-base for other bar.make-artificial.txt files in the same directory.
make bzip2
# Bitslices (a) ..= (g)
bits 16 0x425A
bits 8 0x68
bits 8 0x39
bits 48 0x314159265359
bits 32 0x76A70995
bits 1 0x0
bits 24 0x000001
# Bitslices (h) ..= (j)
bits 16 0x0300
bits 16 0x7000
bits 16 0x2000
# Bitslices (k) ..= (m)
bits 3 0x2
bits 15 0x0001
bits 1 0x0
# Bitslices (n) ..= (o)
repeat 2 [
bits 19 0x0CD34
]
# Bitslice (p)
bits 3 0x6
bits 2 0x0
bits 2 0x1
bits 2 0x0
bits 3 0x4
bits 2 0x1
bits 3 0x7
# Bitslices (q) ..= (s)
bits 48 0x177245385090
bits 32 0x76A70995
bits 5 0x00
@@ -0,0 +1,42 @@
# Feed this file to script/make-artificial.go
# This modifies abraca.txt.bz2.make-artificial.txt to have 51 Huffman-coded
# symbols (bitslice (p)) but only 1 section (bitslice (l)). At 50 symbols per
# section, this should be invalid.
make bzip2
# Bitslices (a) ..= (g)
bits 16 0x425A
bits 8 0x68
bits 8 0x39
bits 48 0x314159265359
bits 32 0x76A70995
bits 1 0x0
bits 24 0x000001
# Bitslices (h) ..= (j)
bits 16 0x0300
bits 16 0x7000
bits 16 0x2000
# Bitslices (k) ..= (m)
bits 3 0x2
bits 15 0x0001
bits 1 0x0
# Bitslices (n) ..= (o)
repeat 2 [
bits 19 0x0CD34
]
# Bitslice (p)
repeat 50 [
bits 2 0x0
]
bits 3 0x7
# Bitslices (q) ..= (s)
bits 48 0x177245385090
bits 32 0x76A70995
bits 5 0x00
Binary file not shown.
@@ -0,0 +1,83 @@
# Feed this file to script/make-artificial.go
# This modifies abraca.txt.bz2.make-artificial.txt to produce the same
# decompressed output, "abraca", but in a more elaborate way. The Huffman codes
# here have the full 258 symbols instead of just 6.
make bzip2
# Bitslices (a) ..= (g)
bits 16 0x425A
bits 8 0x68
bits 8 0x39
bits 48 0x314159265359
bits 32 0x76A70995
bits 1 0x0
bits 24 0x000001
# Bitslices (h) ..= (j)
bits 16 0xFFFF
repeat 16 [
bits 16 0xFFFF
]
# Bitslices (k) ..= (m)
bits 3 0x2
bits 15 0x0001
bits 1 0x0
# Bitslices (n) ..= (o)
#
# Both Huffman codes have 258 symbols: 254 × 8-bit symbols and then 4 × 9-bit
# symbols.
#
# For small XYZ (where XYZ < 253), the mXYZ symbol's bitstring is (XYZ+1) in
# binary. For example, the m099 symbol's bitstring is "01100100", because the
# binary representation of 0d099+1 = 0d100 = 0x64 = 0b01100100.
#
# RUNA is the 8-bit bitstring "00000000".
# RUNB is the 8-bit bitstring "00000001".
# m001 is the 8-bit bitstring "00000010".
# m002 is the 8-bit bitstring "00000011".
# m003 is the 8-bit bitstring "00000100".
# etc
# m250 is the 8-bit bitstring "11111011".
# m251 is the 8-bit bitstring "11111100".
# m252 is the 8-bit bitstring "11111101".
# m253 is the 9-bit bitstring "111111100".
# m254 is the 9-bit bitstring "111111101".
# m255 is the 9-bit bitstring "111111110".
# EOB is the 9-bit bitstring "111111111".
repeat 2 [
bits 5 0x08
repeat 254 [
bits 1 0x0
]
bits 3 0x4
bits 1 0x0
bits 1 0x0
bits 1 0x0
]
# Bitslice (p)
#
# There are 7 symbols.
# Order = [0x00, 0x01, 0x02, 0x03, ... 0xFF] m099 picks 'c'. 0d099+1 = 0x64.
# Order = ['c', 0x00, 0x01, 0x02, ... 0xFF] m098 picks 'a'. 0d098+1 = 0x63.
# Order = ['a', 'c', 0x00, 0x01, ... 0xFF] m114 picks 'r'. 0d114+1 = 0x73.
# Order = ['r', 'a', 'c', 0x00, ... 0xFF] m001 picks 'a'. 0d001+1 = 0x02.
# Order = ['a', 'r', 'c', 0x00, ... 0xFF] RUNA picks 'a'.
# Order = ['a', 'r', 'c', 0x00, ... 0xFF] m100 picks 'b'. 0d100+1 = 0x65.
# Order = ['b', 'a', 'r', 'c', ... 0xFF] EOB.
bits 8 0x64
bits 8 0x63
bits 8 0x73
bits 8 0x02
bits 8 0x00
bits 8 0x65
bits 9 0x1FF
# Bitslices (q) ..= (s)
bits 48 0x177245385090
bits 32 0x76A70995
bits 1 0x0
Binary file not shown.
Binary file not shown.
Binary file not shown.