Add more GIF benchmarks

The existing benchmarks decoded realistic photos. The new benchmarks
also decode artificially generated images, which compress better.

Also adjust script/extract-giflzw.go to optionally analyze and print a
histogram of LZW code emission lengths. "Compresses better" is highly
correlated with "a lower proportion of literal (1 length) codes".
This commit is contained in:
Nigel Tao committed 2018-10-21 11:46:37 +11:00
1 parent 6945bdc75c
commit 8d7bd36179
19 files changed
+521 -40

No files matched your search

+196 -21
View File
@@ -16,20 +16,37 @@
package main
// extract-giflzw.go extracts the LZW-compressed block data in a GIF image. The
// initial byte in each written file is the LZW literal width.
// extract-giflzw.go extracts or analyzes the LZW-compressed block data in a
// GIF image.
//
// Usage: go run extract-giflzw.go foo.gif bar.gif
// - Add the "-extract" flag to write the LZW data to a file. The initial
// byte in each written file is the LZW literal width.
//
// - Add the "-histogram" flag to print emission length histograms instead of
// writing out *.indexes.giflzw files.
//
// Add the optional "-allframes" flag to process all frames of an animated GIF,
// not just the first frame. It applies to both -extract and -histogram.
//
// Usage: go run extract-giflzw.go -extract foo.gif bar.gif
// go run extract-giflzw.go -histogram -allframes foo.gif bar.gif
import (
"bytes"
"compress/lzw"
"flag"
"fmt"
"io/ioutil"
"os"
"path/filepath"
)
var (
allframesFlag = flag.Bool("allframes", false, "process all frames, not just the first")
extractFlag = flag.Bool("extract", false, "write LZW data to a file")
histogramFlag = flag.Bool("histogram", false, "print emission length histograms")
)
func main() {
if err := main1(); err != nil {
os.Stderr.WriteString(err.Error() + "\n")
@@ -38,24 +55,67 @@ func main() {
}
func main1() error {
for _, a := range os.Args[1:] {
data, err := decode(a)
flag.Parse()
if !*extractFlag && !*histogramFlag {
return fmt.Errorf("none of -extract or -histogram given")
}
for argIndex, arg := range flag.Args() {
frames, err := extractLZWFrames(arg)
if err != nil {
return err
}
// As a sanity check, the result should be valid LZW.
if err := checkLZW(data); err != nil {
return err
if *histogramFlag {
which := "first frame"
if *allframesFlag {
which = "all frames"
}
fmt.Printf("\n%s (%s)\n", arg, which)
histogramTotal = 0
for i := range histogram {
histogram[i] = 0
}
}
b := a[:len(a)-len(filepath.Ext(a))]
if err := ioutil.WriteFile(b+".indexes.giflzw", data, 0644); err != nil {
return err
for frameIndex, frame := range frames {
if err := checkLZW(frame); err != nil {
fmt.Printf("Warning: frame #%d: %v\n", frameIndex, err)
}
if *extractFlag {
filename := arg[:len(arg)-len(filepath.Ext(arg))]
if *allframesFlag {
filename = fmt.Sprintf("%s-frame-%03d", filename, argIndex)
}
filename += ".indexes.giflzw"
if err := ioutil.WriteFile(filename, frame, 0644); err != nil {
return err
}
}
if *histogramFlag {
// Ignore any LZW format errors. Decode as much as we can.
_ = buildHistogram(frame)
}
if !*allframesFlag {
break
}
}
if *histogramFlag {
fmt.Printf(" Percent Count Emission_Length_Bucket\n")
for i, h := range histogram {
percent := float64(100*h) / float64(histogramTotal)
fmt.Printf(" %6.2f %6d %s\n", percent, h, bucketNames[i])
}
}
}
return nil
}
func decode(filename string) (ret []byte, err error) {
func extractLZWFrames(filename string) (ret [][]byte, err error) {
src, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
@@ -79,7 +139,6 @@ func decode(filename string) (ret []byte, err error) {
return nil, err
}
seenID := false
for len(src) > 0 {
switch src[0] {
case sExtension:
@@ -91,11 +150,7 @@ func decode(filename string) (ret []byte, err error) {
if src, compressed, err = readImageDescriptor(src[1:]); err != nil {
return nil, err
}
// For an animated GIF, take only the first frame.
if !seenID {
seenID = true
ret = compressed
}
ret = append(ret, compressed)
case sTrailer:
if len(src) != 1 {
return nil, fmt.Errorf("extraneous data after GIF trailer section")
@@ -176,11 +231,11 @@ func readBlockData(src []byte, ret []byte) (src1 []byte, ret1 []byte, err error)
}
}
func checkLZW(x []byte) error {
if len(x) == 0 {
func checkLZW(data []byte) error {
if len(data) == 0 {
return fmt.Errorf("missing GIF literal width")
}
rc := lzw.NewReader(bytes.NewReader(x[1:]), lzw.LSB, int(x[0]))
rc := lzw.NewReader(bytes.NewReader(data[1:]), lzw.LSB, int(data[0]))
defer rc.Close()
_, err := ioutil.ReadAll(rc)
if err != nil {
@@ -189,6 +244,126 @@ func checkLZW(x []byte) error {
return nil
}
const numBuckets = 14
var (
histogramTotal = uint32(0)
histogram = [numBuckets]uint32{}
bucketNames = [numBuckets]string{
" 1 (Literal Code)",
" 2",
" 3",
" 4",
" 5",
" 6",
" 7",
" 8",
" 9 .. 16",
" 17 .. 32",
" 33 .. 64",
" 65 .. 128",
"127 .. 256",
"256+ ",
}
)
func bucket(emissionLength uint32) uint32 {
switch {
case emissionLength <= 0:
panic("bad emissionLength")
case emissionLength <= 8:
return emissionLength - 1
case emissionLength <= 16:
return 8
case emissionLength <= 32:
return 9
case emissionLength <= 64:
return 10
case emissionLength <= 128:
return 11
case emissionLength <= 256:
return 12
default:
return 13
}
}
func buildHistogram(data []byte) error {
if len(data) == 0 {
return fmt.Errorf("missing GIF literal width")
}
literalWidth, data := uint32(data[0]), data[1:]
clearCode := uint32(1) << literalWidth
endCode := clearCode + 1
saveCode := endCode
prevCode := uint32(0)
width := literalWidth + 1
prefixes := [4096]uint32{}
emissionLengths := [4096]uint32{}
for i := uint32(0); i < clearCode; i++ {
emissionLengths[i] = 1
}
bits := uint32(0)
nBits := uint32(0)
for {
for nBits < width {
if len(data) == 0 {
return fmt.Errorf("bad LZW data: short read")
}
bits |= uint32(data[0]) << nBits
data = data[1:]
nBits += 8
}
code := bits & ((uint32(1) << width) - 1)
bits >>= width
nBits -= width
if code < clearCode {
// Literal, emitting exactly 1 byte.
} else if code == clearCode {
saveCode = endCode
prevCode = 0
width = literalWidth + 1
continue
} else if code == endCode {
break
} else if code <= saveCode {
// Copy, emitting 2 or more bytes.
} else {
return fmt.Errorf("bad LZW data: code out of range")
}
n := emissionLengths[code]
if code == saveCode {
n = emissionLengths[prevCode] + 1
}
histogram[bucket(n)]++
histogramTotal++
if saveCode <= 4095 {
prefixes[saveCode] = prevCode
emissionLengths[saveCode] = emissionLengths[prevCode] + 1
saveCode++
if (saveCode == (1 << width)) && (width < 12) {
width++
}
prevCode = code
}
}
return nil
}
// The constants below are intrinsic to the GIF file format. The spec is
// http://www.w3.org/Graphics/GIF/spec-gif89a.txt
const (