From b0c72867e66267063ad4ac4cfad770e4555dbac4 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sun, 21 Oct 2018 16:03:47 +1100 Subject: [PATCH] Update std/lzw/README.md for algorithmic detail --- script/print-lzw-example.go | 293 ++++++++++++++++++++++++++++++++++++ std/gif/README.md | 11 +- std/lzw/README.md | 256 ++++++++++++++++++++++++++++++- 3 files changed, 553 insertions(+), 7 deletions(-) create mode 100644 script/print-lzw-example.go diff --git a/script/print-lzw-example.go b/script/print-lzw-example.go new file mode 100644 index 00000000..ab09d6a5 --- /dev/null +++ b/script/print-lzw-example.go @@ -0,0 +1,293 @@ +// Copyright 2018 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build ignore + +package main + +// print-lzw-example.go prints a worked example discussed in std/lzw/README.md, +// based on three implementations of a simplified core of the LZW algorithm. +// Simplifications include assuming LSB-first, 8 bit literals, no clear codes, +// no full tables and that the input is given in terms of a code stream, not a +// bit stream. +// +// The implementations are designed for ease of studying, and for e.g. a +// minimal diff between the suf1 and sufQ implementations, and are not +// optimized for performance. +// +// Usage: go run print-lzw-example.go +// +// You can also play with the code (e.g. modify Q and re-run) online: +// https://play.golang.org/p/1aLC_XHZzoA + +import ( + "fmt" +) + +const ( + expectedOutput = "TOBEORNOTTOBEORTOBEORNOTXOTXOTXOOTXOOOTXOOOTOBEY" + + clearCode = 0x100 + endCode = 0x101 + + // Neither clearCode or endCode can ever be a valid prefix code. We + // arbitrarily pick clearCode to represent "no prefix". + noPrefix = clearCode + + // Q is the maximum possible suffix length, in bytes, which equals 1 plus + // the maximum possible "suffix length minus 1". Q == 1 means that sufQ is + // equivalent to suf1. + // + // In std/lzw/README.md, Q is 3 to keep the example short and simple. In + // practice, especially on modern CPUs that can do unaligned 32 or 64 bit + // loads and stores, Q is 4 or 8. + Q = 3 +) + +var codes = []int{ + 'T', 'O', 'B', 'E', 'O', 'R', 'N', 'O', 'T', + 0x102, 0x104, 0x106, 0x10B, 0x105, 0x107, 0x109, + 'X', + 0x111, 0x113, 0x114, 0x115, 0x10E, + 'Y', + 0x101, +} + +func codeString(code int) string { + if code < clearCode { + return fmt.Sprintf("%5c", code) + } else if code == noPrefix { + return " -" + } + return fmt.Sprintf("0x%03X", code) +} + +func main() { + n, s, q := new(naive), new(suf1), new(sufQ) + decode(n) + decode(s) + decode(q) + + key := endCode + fmt.Printf(" Code Emits Key Value Pre1+Suf1 LM1 /Q %%Q PreQ+SufQ\n") + for _, code := range codes { + // Code + fmt.Printf("%s", codeString(code)) + + // Emits + if code == endCode { + fmt.Printf(" -end-\n") + break + } + fmt.Printf(" %6s", n[code]) + + if key != endCode { + fmt.Printf(" 0x%03X %7s %s %c %3d %3d %3d %s %s", + key, // Key + n[key], // Value + codeString(int(s.prefixes[key])), // Pre1 + s.suffixes[key], // Suf1 + q.lengthM1s[key], // LM1 + q.lengthM1s[key]/Q, // /Q + q.lengthM1s[key]%Q, // %Q + codeString(int(q.prefixes[key])), // PreQ + string(q.suffixes[key][:]), // SufQ + ) + } + + fmt.Println() + key++ + } +} + +type implementation interface { + initialize() + emit(buffer []byte, code int, prevCode int, n int) []byte +} + +func decode(impl implementation) { + impl.initialize() + + output, prevCode, n := []byte(nil), -1, 0x101 + for _, code := range codes { + if code == clearCode { + panic("unimplemented clearCode") + } else if code == endCode { + if string(output) != expectedOutput { + panic("unexpected output") + } + return + } else if code > n { + panic("invalid code") + } + + // We have a literal or copy code. + output = impl.emit(output, code, prevCode, n) + + prevCode, n = code, n+1 + if n >= 4096 { + panic("unimplemented table-is-full") + } + } + panic("missing endCode") +} + +// naive is a simple implementation that, in the worst case, requires O(N*N) +// memory. +type naive [4096]string + +func (t *naive) initialize() { + for i := 0; i < clearCode; i++ { + t[i] = string([]byte{byte(i)}) + } +} + +func (t *naive) emit(output []byte, code int, prevCode int, n int) []byte { + if prevCode >= 0 { + prevOutput := t[prevCode] + if code < n { + t[n] = prevOutput + t[code][:1] + } else { + t[n] = prevOutput + prevOutput[:1] + } + } + return append(output, t[code]...) +} + +// suf1 keeps separate prefix and suffix tables, 1 byte per suffix. +type suf1 struct { + prefixes [4096]uint16 + suffixes [4096]byte +} + +func (t *suf1) initialize() { + for i := 0; i < clearCode; i++ { + t.prefixes[i] = noPrefix + t.suffixes[i] = byte(i) + } +} + +func (t *suf1) emit(output []byte, code int, prevCode int, n int) []byte { + // Fill an intermediate buffer from back to front, 1 byte at a time. + buffer := [4096]byte{} + i := len(buffer) + + c := code + if code == n { + c = prevCode + i-- + // buffer[4095] will be set later. + } + + firstByte := byte(0) + for { + suffix := t.suffixes[c] + i-- + buffer[i] = suffix + c = int(t.prefixes[c]) + if c == noPrefix { + firstByte = suffix + break + } + } + + if code == n { + buffer[4095] = firstByte + } + + // The "if prevCode >= 0" guard, and initializing prevCode to -1 instead of + // 0, is correct conceptually, but unnecessary in practice. Look for "fake + // key-value pair" in std/lzw/README.md. + if prevCode >= 0 { + t.prefixes[n] = uint16(prevCode) + t.suffixes[n] = firstByte + } + + return append(output, buffer[i:4096]...) +} + +// sufQ keeps separate prefix and suffix tables, up to Q bytes per suffix. +type sufQ struct { + lengthM1s [4096]uint16 + prefixes [4096]uint16 + suffixes [4096][Q]byte +} + +func (t *sufQ) initialize() { + for i := 0; i < clearCode; i++ { + t.lengthM1s[i] = 0 + t.prefixes[i] = noPrefix + t.suffixes[i][0] = byte(i) + for j := 1; j < Q; j++ { + // Unnecessary for correct output, but makes the printed table prettier. + t.suffixes[i][j] = '.' + } + } +} + +func (t *sufQ) emit(output []byte, code int, prevCode int, n int) []byte { + // Fill an intermediate buffer from back to front, Q bytes at a time. + buffer := [4096 + Q - 1]byte{} + i := len(buffer) + + c := code + if code == n { + c = prevCode + i-- + // buffer[4095] will be set later. + } + + i -= int(t.lengthM1s[c] % Q) + + firstByte := byte(0) + for { + suffix := t.suffixes[c] + i -= Q + copy(buffer[i:i+Q], suffix[:]) + c = int(t.prefixes[c]) + if c == noPrefix { + firstByte = suffix[0] + break + } + } + + if code == n { + buffer[4095] = firstByte + } + + // The "if prevCode >= 0" guard, and initializing prevCode to -1 instead of + // 0, is correct conceptually, but unnecessary in practice. Look for "fake + // key-value pair" in std/lzw/README.md. + if prevCode >= 0 { + lm1 := t.lengthM1s[prevCode] + 1 + t.lengthM1s[n] = lm1 + lm1 %= Q + if lm1 != 0 { + // Copy the prevCode's prefix and suffix, and then extend the suffix. + t.prefixes[n] = t.prefixes[prevCode] + t.suffixes[n] = t.suffixes[prevCode] + t.suffixes[n][lm1] = firstByte + } else { + // Start a new suffix. + t.prefixes[n] = uint16(prevCode) + t.suffixes[n][0] = firstByte + for j := 1; j < Q; j++ { + // Unnecessary for correct output, but makes the printed table prettier. + t.suffixes[n][j] = '.' + } + } + } + + return append(output, buffer[i:4096]...) +} diff --git a/std/gif/README.md b/std/gif/README.md index 90f9bcf0..21eda131 100644 --- a/std/gif/README.md +++ b/std/gif/README.md @@ -200,10 +200,9 @@ The final byte is 0x3B. # LZW (Lempel Ziv Welch) Compression -LZW is a general purpose compression algorithm, not specific to GIF or even to -graphics. In practice, there are two incompatible implementations, LSB (Least -Significant Bits) and MSB (Most Significant Bits) first. The GIF format uses -LSB first. The PDF and TIFF formats use MSB first, but are not discussed here. +See `std/lzw/README.md` for a description of LZW: a general purpose compression +algorithm. Below is a detailed decoding of the LZW data from the Wire Format +Worked Example, deconstructed above. It is not an official format, but the `test/data/bricks-nodither.indexes.giflzw` file contains the extracted "Pixel @@ -266,8 +265,8 @@ Each code decodes to either a literal value (one byte), a back-reference data stream. The first `1<> 3" or "& 7". When Q is 1 (a degenerate +case), the "%Q" column is all zeroes and all suffixes are 1 byte long, which is +unsurprisingly equivalent to the 1-byte suffix representation described above. + +When copying the suffix to a buffer (either an intermediate buffer or a final +buffer), it is often unnecessary to copy *exactly* suffix length bytes, +provided that there is enough destination buffer space for all Q bytes. Even if +we write excess bytes, decoding subsequent codes will overwrite the excess, or +else we'll hit an end code and the excess will be ignored. Again, on modern +CPUs, it can be faster to write *at least 7* bytes (i.e. *exactly 8*) than to +write *exactly 7*. + +Even so, the (LM1 % Q) value is still necessary to calculate where to start +writing those Q-byte chunk. After that, reconstructing the value proceeds in +the same way as in the 1-byte suffix algorithm. It just involves larger chunks. +For example, the value for the key 0x116 is the suffix "T" preceded by the +value for 0x115, which is the suffix "OOO" preceded by the value for 0x111, +which is the suffix "OTX" with no prefix: the value is "OTX"+"OOO"+"T". + +The PreQ column is more complicated than the Pre1 column. It is no longer just +a vertically shifted Code column, and can now contain "no prefix" values (or +equivalently, the "/Q" column is 0). When inserting a new key-value pair, we +only set the N'th prefix equal to the previously seen code (the Code column +shifted by one row) if the suffix for that previous code was a full Q bytes (or +equivalently, if the current row's "%Q" value is 0). Otherwise, we copy the +previous code's prefix and suffix, and then extend the suffix with one more +byte. For example, the PreQ+SufQ value for the 0x117 key is based on the +PreQ+SufQ value (the same PreQ, an extended SufQ) of the previously seen code, +namely 0x10E. + + +# Varying Bit Widths + +The description above has discussed an LZW stream as a stream of codes. In the +actual wire format, those codes are packed into a stream of bytes, and a code +can straddle byte boundaries. + +Specifically, the next code in the stream takes B bits, where B is the smallest +number of that bits that can distinguish all valid codes: those codes in the +range [0, N]. If N is 0x101, B is 9. If N is 0x3FF, B is 10 bits. If N is +0x400, B is 11. Remember that each successive code increments N, up to a +maximum N of 0xFFF, or 12 bits. + +The literal width can also vary in the range [2..8] for LZW as used by GIF. In +that case, B starts off as 1 more than the literal width. For example, with a +literal width of 2, the four literal codes are 0x0, 0x1, 0x2 and 0x3, which are +followed by a clear code of 0x4 and an end code of 0x5, so N is 0x5 and B is 3. + + +## Early Change + +Unfortunately, some implementations misinterpreted the algorithm, and +incremented B *before* instead of *after* emitting a bit-packed code for N == +((1 << B) - 1). Accordingly, LZW as used in TIFF and sometimes in PDF +(depending on the EarlyChange bit) increment B one iteration earlier. The +maximum value of B is still 12. + +This misinterpretation is unfixable due to backwards compatibility, + + +## LSB and MSB + +The bits are then packed into bytes in a predetermined ordering. For GIF, the +bits are packed Least Significant Bits first. For PDF and TIFF, Most +Significant Bits first. With a 8 bit literal width (and therefore a 9 bit +initial code width), the two literal codes T (0x54) and O (0x4F) followed by an +end code 0x101 would be encoded as 27 bits, or 4 bytes (with 5 padding bits, +which are ignored but are conventionally set to zero). + +In LSB first, it would be four bytes, 0x54 0x9E 0x04 0x04: + + offset xoffset ASCII hex binary + 000000 0x0000 T 0x54 0b_0101_0100 + 000001 0x0001 . 0x9E 0b_1001_1110 + 000002 0x0002 . 0x04 0b_0000_0100 + 000003 0x0003 . 0x04 0b_0000_0100 + +These four bytes contain three 9-bit codes: + + binary width code + 0b_...._...._...._...0_0101_0100 9 0x0054 + 0b_...._...._...._..00_1001_111. 9 0x004F + 0b_...._...._...._.100_0000_01.. 9 0x0101 + +In MSB first, it would again be four bytes, 0x2A 0x13 0xE0 0x20: + + offset xoffset ASCII hex binary + 000000 0x0000 * 0x2A 0b_0010_1010 + 000001 0x0001 . 0x13 0b_0001_0011 + 000002 0x0002 . 0xE0 0b_1110_0000 + 000003 0x0003 0x20 0b_0010_0000 + +Again, these four bytes contain three 9-bit codes: + + binary width code + 0b_0010_1010_0..._...._...._.... 9 0x0054 + 0b_.001_0011_11.._...._...._.... 9 0x004F + 0b_..10_0000_001._...._...._.... 9 0x0101 + +See `std/gif/README.md` for a complete worked example of LSB-first encoded LZW +data, including a varying B width.