diff --git a/script/extract-giflzw.go b/script/extract-giflzw.go index cc81d2c6..1f64c354 100644 --- a/script/extract-giflzw.go +++ b/script/extract-giflzw.go @@ -16,20 +16,37 @@ package main -// extract-giflzw.go extracts the LZW-compressed block data in a GIF image. The -// initial byte in each written file is the LZW literal width. +// extract-giflzw.go extracts or analyzes the LZW-compressed block data in a +// GIF image. // -// Usage: go run extract-giflzw.go foo.gif bar.gif +// - Add the "-extract" flag to write the LZW data to a file. The initial +// byte in each written file is the LZW literal width. +// +// - Add the "-histogram" flag to print emission length histograms instead of +// writing out *.indexes.giflzw files. +// +// Add the optional "-allframes" flag to process all frames of an animated GIF, +// not just the first frame. It applies to both -extract and -histogram. +// +// Usage: go run extract-giflzw.go -extract foo.gif bar.gif +// go run extract-giflzw.go -histogram -allframes foo.gif bar.gif import ( "bytes" "compress/lzw" + "flag" "fmt" "io/ioutil" "os" "path/filepath" ) +var ( + allframesFlag = flag.Bool("allframes", false, "process all frames, not just the first") + extractFlag = flag.Bool("extract", false, "write LZW data to a file") + histogramFlag = flag.Bool("histogram", false, "print emission length histograms") +) + func main() { if err := main1(); err != nil { os.Stderr.WriteString(err.Error() + "\n") @@ -38,24 +55,67 @@ func main() { } func main1() error { - for _, a := range os.Args[1:] { - data, err := decode(a) + flag.Parse() + if !*extractFlag && !*histogramFlag { + return fmt.Errorf("none of -extract or -histogram given") + } + + for argIndex, arg := range flag.Args() { + frames, err := extractLZWFrames(arg) if err != nil { return err } - // As a sanity check, the result should be valid LZW. - if err := checkLZW(data); err != nil { - return err + + if *histogramFlag { + which := "first frame" + if *allframesFlag { + which = "all frames" + } + fmt.Printf("\n%s (%s)\n", arg, which) + + histogramTotal = 0 + for i := range histogram { + histogram[i] = 0 + } } - b := a[:len(a)-len(filepath.Ext(a))] - if err := ioutil.WriteFile(b+".indexes.giflzw", data, 0644); err != nil { - return err + + for frameIndex, frame := range frames { + if err := checkLZW(frame); err != nil { + fmt.Printf("Warning: frame #%d: %v\n", frameIndex, err) + } + + if *extractFlag { + filename := arg[:len(arg)-len(filepath.Ext(arg))] + if *allframesFlag { + filename = fmt.Sprintf("%s-frame-%03d", filename, argIndex) + } + filename += ".indexes.giflzw" + if err := ioutil.WriteFile(filename, frame, 0644); err != nil { + return err + } + } + if *histogramFlag { + // Ignore any LZW format errors. Decode as much as we can. + _ = buildHistogram(frame) + } + + if !*allframesFlag { + break + } + } + + if *histogramFlag { + fmt.Printf(" Percent Count Emission_Length_Bucket\n") + for i, h := range histogram { + percent := float64(100*h) / float64(histogramTotal) + fmt.Printf(" %6.2f %6d %s\n", percent, h, bucketNames[i]) + } } } return nil } -func decode(filename string) (ret []byte, err error) { +func extractLZWFrames(filename string) (ret [][]byte, err error) { src, err := ioutil.ReadFile(filename) if err != nil { return nil, err @@ -79,7 +139,6 @@ func decode(filename string) (ret []byte, err error) { return nil, err } - seenID := false for len(src) > 0 { switch src[0] { case sExtension: @@ -91,11 +150,7 @@ func decode(filename string) (ret []byte, err error) { if src, compressed, err = readImageDescriptor(src[1:]); err != nil { return nil, err } - // For an animated GIF, take only the first frame. - if !seenID { - seenID = true - ret = compressed - } + ret = append(ret, compressed) case sTrailer: if len(src) != 1 { return nil, fmt.Errorf("extraneous data after GIF trailer section") @@ -176,11 +231,11 @@ func readBlockData(src []byte, ret []byte) (src1 []byte, ret1 []byte, err error) } } -func checkLZW(x []byte) error { - if len(x) == 0 { +func checkLZW(data []byte) error { + if len(data) == 0 { return fmt.Errorf("missing GIF literal width") } - rc := lzw.NewReader(bytes.NewReader(x[1:]), lzw.LSB, int(x[0])) + rc := lzw.NewReader(bytes.NewReader(data[1:]), lzw.LSB, int(data[0])) defer rc.Close() _, err := ioutil.ReadAll(rc) if err != nil { @@ -189,6 +244,126 @@ func checkLZW(x []byte) error { return nil } +const numBuckets = 14 + +var ( + histogramTotal = uint32(0) + histogram = [numBuckets]uint32{} + bucketNames = [numBuckets]string{ + " 1 (Literal Code)", + " 2", + " 3", + " 4", + " 5", + " 6", + " 7", + " 8", + " 9 .. 16", + " 17 .. 32", + " 33 .. 64", + " 65 .. 128", + "127 .. 256", + "256+ ", + } +) + +func bucket(emissionLength uint32) uint32 { + switch { + case emissionLength <= 0: + panic("bad emissionLength") + case emissionLength <= 8: + return emissionLength - 1 + case emissionLength <= 16: + return 8 + case emissionLength <= 32: + return 9 + case emissionLength <= 64: + return 10 + case emissionLength <= 128: + return 11 + case emissionLength <= 256: + return 12 + default: + return 13 + } +} + +func buildHistogram(data []byte) error { + if len(data) == 0 { + return fmt.Errorf("missing GIF literal width") + } + + literalWidth, data := uint32(data[0]), data[1:] + clearCode := uint32(1) << literalWidth + endCode := clearCode + 1 + + saveCode := endCode + prevCode := uint32(0) + width := literalWidth + 1 + + prefixes := [4096]uint32{} + emissionLengths := [4096]uint32{} + for i := uint32(0); i < clearCode; i++ { + emissionLengths[i] = 1 + } + + bits := uint32(0) + nBits := uint32(0) + + for { + for nBits < width { + if len(data) == 0 { + return fmt.Errorf("bad LZW data: short read") + } + bits |= uint32(data[0]) << nBits + data = data[1:] + nBits += 8 + } + + code := bits & ((uint32(1) << width) - 1) + bits >>= width + nBits -= width + + if code < clearCode { + // Literal, emitting exactly 1 byte. + + } else if code == clearCode { + saveCode = endCode + prevCode = 0 + width = literalWidth + 1 + continue + + } else if code == endCode { + break + + } else if code <= saveCode { + // Copy, emitting 2 or more bytes. + + } else { + return fmt.Errorf("bad LZW data: code out of range") + } + + n := emissionLengths[code] + if code == saveCode { + n = emissionLengths[prevCode] + 1 + } + histogram[bucket(n)]++ + histogramTotal++ + + if saveCode <= 4095 { + prefixes[saveCode] = prevCode + emissionLengths[saveCode] = emissionLengths[prevCode] + 1 + saveCode++ + if (saveCode == (1 << width)) && (width < 12) { + width++ + } + prevCode = code + } + } + + return nil +} + // The constants below are intrinsic to the GIF file format. The spec is // http://www.w3.org/Graphics/GIF/spec-gif89a.txt const ( diff --git a/test/c/std/gif.c b/test/c/std/gif.c index 417e0130..6765dfe5 100644 --- a/test/c/std/gif.c +++ b/test/c/std/gif.c @@ -1177,9 +1177,14 @@ const char* test_mimic_gif_decode_hat() { return do_test_mimic_gif_decode("../../data/hat.gif"); } -const char* test_mimic_gif_decode_hibiscus() { +const char* test_mimic_gif_decode_hibiscus_primitive() { CHECK_FOCUS(__func__); - return do_test_mimic_gif_decode("../../data/hibiscus.gif"); + return do_test_mimic_gif_decode("../../data/hibiscus.primitive.gif"); +} + +const char* test_mimic_gif_decode_hibiscus_regular() { + CHECK_FOCUS(__func__); + return do_test_mimic_gif_decode("../../data/hibiscus.regular.gif"); } const char* test_mimic_gif_decode_hippopotamus_interlaced() { @@ -1256,9 +1261,16 @@ const char* bench_wuffs_gif_decode_10k() { return do_bench_gif_decode(wuffs_gif_decode, "../../data/hat.gif", 100); } -const char* bench_wuffs_gif_decode_100k() { +const char* bench_wuffs_gif_decode_100k_artificial() { CHECK_FOCUS(__func__); - return do_bench_gif_decode(wuffs_gif_decode, "../../data/hibiscus.gif", 10); + return do_bench_gif_decode(wuffs_gif_decode, + "../../data/hibiscus.primitive.gif", 15); +} + +const char* bench_wuffs_gif_decode_100k_realistic() { + CHECK_FOCUS(__func__); + return do_bench_gif_decode(wuffs_gif_decode, + "../../data/hibiscus.regular.gif", 10); } const char* bench_wuffs_gif_decode_1000k() { @@ -1266,6 +1278,12 @@ const char* bench_wuffs_gif_decode_1000k() { return do_bench_gif_decode(wuffs_gif_decode, "../../data/harvesters.gif", 1); } +const char* bench_wuffs_gif_decode_anim_screencap() { + CHECK_FOCUS(__func__); + return do_bench_gif_decode(wuffs_gif_decode, + "../../data/gifplayer-muybridge.gif", 1); +} + // ---------------- Mimic Benches #ifdef WUFFS_MIMIC @@ -1287,9 +1305,16 @@ const char* bench_mimic_gif_decode_10k() { return do_bench_gif_decode(mimic_gif_decode, "../../data/hat.gif", 100); } -const char* bench_mimic_gif_decode_100k() { +const char* bench_mimic_gif_decode_100k_artificial() { CHECK_FOCUS(__func__); - return do_bench_gif_decode(mimic_gif_decode, "../../data/hibiscus.gif", 10); + return do_bench_gif_decode(mimic_gif_decode, + "../../data/hibiscus.primitive.gif", 15); +} + +const char* bench_mimic_gif_decode_100k_realistic() { + CHECK_FOCUS(__func__); + return do_bench_gif_decode(mimic_gif_decode, + "../../data/hibiscus.regular.gif", 10); } const char* bench_mimic_gif_decode_1000k() { @@ -1297,6 +1322,12 @@ const char* bench_mimic_gif_decode_1000k() { return do_bench_gif_decode(mimic_gif_decode, "../../data/harvesters.gif", 1); } +const char* bench_mimic_gif_decode_anim_screencap() { + CHECK_FOCUS(__func__); + return do_bench_gif_decode(mimic_gif_decode, + "../../data/gifplayer-muybridge.gif", 1); +} + #endif // WUFFS_MIMIC // ---------------- Manifest @@ -1341,7 +1372,8 @@ proc tests[] = { test_mimic_gif_decode_gifplayer_muybridge, // test_mimic_gif_decode_harvesters, // test_mimic_gif_decode_hat, // - test_mimic_gif_decode_hibiscus, // + test_mimic_gif_decode_hibiscus_primitive, // + test_mimic_gif_decode_hibiscus_regular, // test_mimic_gif_decode_hippopotamus_interlaced, // test_mimic_gif_decode_hippopotamus_regular, // test_mimic_gif_decode_muybridge, // @@ -1355,19 +1387,23 @@ proc tests[] = { // The empty comments forces clang-format to place one element per line. proc benches[] = { - bench_wuffs_gif_decode_1k_bw, // - bench_wuffs_gif_decode_1k_color, // - bench_wuffs_gif_decode_10k, // - bench_wuffs_gif_decode_100k, // - bench_wuffs_gif_decode_1000k, // + bench_wuffs_gif_decode_1k_bw, // + bench_wuffs_gif_decode_1k_color, // + bench_wuffs_gif_decode_10k, // + bench_wuffs_gif_decode_100k_artificial, // + bench_wuffs_gif_decode_100k_realistic, // + bench_wuffs_gif_decode_1000k, // + bench_wuffs_gif_decode_anim_screencap, // #ifdef WUFFS_MIMIC - bench_mimic_gif_decode_1k_bw, // - bench_mimic_gif_decode_1k_color, // - bench_mimic_gif_decode_10k, // - bench_mimic_gif_decode_100k, // - bench_mimic_gif_decode_1000k, // + bench_mimic_gif_decode_1k_bw, // + bench_mimic_gif_decode_1k_color, // + bench_mimic_gif_decode_10k, // + bench_mimic_gif_decode_100k_artificial, // + bench_mimic_gif_decode_100k_realistic, // + bench_mimic_gif_decode_1000k, // + bench_mimic_gif_decode_anim_screencap, // #endif // WUFFS_MIMIC diff --git a/test/data/README.md b/test/data/README.md index c9da33db..3487b649 100644 --- a/test/data/README.md +++ b/test/data/README.md @@ -41,11 +41,17 @@ hat.\* are various encodings of a photo of "Self-Portrait with a Straw Hat Museum of art. `http://www.metmuseum.org/art/collection/search/436532` lists that image as in the public domain. -hibiscus.\* are various encodings of a photo of "Hibiscus and Parrots" by Louis -Comfort Tiffany, held by the Metropolitan Museum of Art. +hibiscus.regular.\* are various encodings of a photo of "Hibiscus and Parrots" +by Louis Comfort Tiffany, held by the Metropolitan Museum of Art. `http://www.metmuseum.org/art/collection/search/13503` lists that image as in the public domain. +hibiscus.primitive.\* is the result of running the hibiscus.regular.png image +through `github.com/fogleman/primitive` and other programs: + - `primitive -i hibiscus.regular.png -o hibiscus.primitive.svg -s 442 -n 32` + - `inkscape -z -e hibiscus.primitive.png hibiscus.primitive.svg` + - Various invocations of `convert` and `cwebp` + hippopotamus.\* are various encodings of a cropping of a photo of "Hippopotamus (William)", held by the Metropolitan Museum of Art. `http://www.metmuseum.org/art/collection/search/544227` lists that image as in diff --git a/test/data/analysis/giflzw-emission-lengths.txt b/test/data/analysis/giflzw-emission-lengths.txt new file mode 100644 index 00000000..1eddb263 --- /dev/null +++ b/test/data/analysis/giflzw-emission-lengths.txt @@ -0,0 +1,227 @@ +# To reproduce this analysis: +# +# cd .. +# go run ../../script/extract-giflzw.go -histogram -allframes *.gif > analysis/giflzw-emission-lengths.txt + +animated-red-blue.gif (all frames) + Percent Count Emission_Length_Bucket + 49.23 542 1 (Literal Code) + 5.27 58 2 + 6.36 70 3 + 5.99 66 4 + 4.18 46 5 + 3.54 39 6 + 2.45 27 7 + 1.73 19 8 + 9.36 103 9 .. 16 + 7.18 79 17 .. 32 + 4.72 52 33 .. 64 + 0.00 0 65 .. 128 + 0.00 0 127 .. 256 + 0.00 0 256+ + +bricks-dither.gif (all frames) + Percent Count Emission_Length_Bucket + 45.58 4853 1 (Literal Code) + 35.89 3821 2 + 13.23 1409 3 + 3.82 407 4 + 1.04 111 5 + 0.31 33 6 + 0.08 9 7 + 0.02 2 8 + 0.02 2 9 .. 16 + 0.00 0 17 .. 32 + 0.00 0 33 .. 64 + 0.00 0 65 .. 128 + 0.00 0 127 .. 256 + 0.00 0 256+ + +bricks-gray.gif (all frames) + Percent Count Emission_Length_Bucket + 47.34 4980 1 (Literal Code) + 31.91 3357 2 + 14.30 1504 3 + 4.53 476 4 + 1.34 141 5 + 0.42 44 6 + 0.14 15 7 + 0.02 2 8 + 0.00 0 9 .. 16 + 0.00 0 17 .. 32 + 0.00 0 33 .. 64 + 0.00 0 65 .. 128 + 0.00 0 127 .. 256 + 0.00 0 256+ + +bricks-nodither.gif (all frames) + Percent Count Emission_Length_Bucket + 41.39 3982 1 (Literal Code) + 32.93 3168 2 + 16.13 1552 3 + 5.93 570 4 + 2.21 213 5 + 0.95 91 6 + 0.31 30 7 + 0.09 9 8 + 0.05 5 9 .. 16 + 0.00 0 17 .. 32 + 0.00 0 33 .. 64 + 0.00 0 65 .. 128 + 0.00 0 127 .. 256 + 0.00 0 256+ + +gifplayer-muybridge.gif (all frames) +Warning: frame #60: block data is not valid LZW: unexpected EOF +Warning: frame #89: block data is not valid LZW: unexpected EOF + Percent Count Emission_Length_Bucket + 12.78 34658 1 (Literal Code) + 14.05 38106 2 + 13.25 35937 3 + 10.59 28707 4 + 7.85 21286 5 + 6.08 16481 6 + 4.85 13156 7 + 3.60 9764 8 + 9.34 25318 9 .. 16 + 4.89 13257 17 .. 32 + 4.78 12954 33 .. 64 + 4.72 12806 65 .. 128 + 3.02 8190 127 .. 256 + 0.20 550 256+ + +harvesters.gif (all frames) + Percent Count Emission_Length_Bucket + 46.73 269145 1 (Literal Code) + 39.51 227566 2 + 10.13 58353 3 + 2.39 13766 4 + 0.71 4106 5 + 0.26 1509 6 + 0.11 622 7 + 0.05 291 8 + 0.08 485 9 .. 16 + 0.02 100 17 .. 32 + 0.01 57 33 .. 64 + 0.00 1 65 .. 128 + 0.00 0 127 .. 256 + 0.00 0 256+ + +hat.gif (all frames) + Percent Count Emission_Length_Bucket + 80.52 6758 1 (Literal Code) + 19.03 1597 2 + 0.39 33 3 + 0.01 1 4 + 0.01 1 5 + 0.01 1 6 + 0.02 2 7 + 0.00 0 8 + 0.00 0 9 .. 16 + 0.00 0 17 .. 32 + 0.00 0 33 .. 64 + 0.00 0 65 .. 128 + 0.00 0 127 .. 256 + 0.00 0 256+ + +hibiscus.primitive.gif (all frames) + Percent Count Emission_Length_Bucket + 25.86 5566 1 (Literal Code) + 19.05 4100 2 + 11.30 2433 3 + 7.30 1571 4 + 5.43 1168 5 + 4.34 934 6 + 3.23 695 7 + 2.55 548 8 + 11.13 2396 9 .. 16 + 6.81 1466 17 .. 32 + 2.89 623 33 .. 64 + 0.10 22 65 .. 128 + 0.00 0 127 .. 256 + 0.00 0 256+ + +hibiscus.regular.gif (all frames) + Percent Count Emission_Length_Bucket + 47.48 37363 1 (Literal Code) + 36.43 28663 2 + 11.91 9370 3 + 2.94 2311 4 + 0.81 637 5 + 0.22 177 6 + 0.09 74 7 + 0.04 31 8 + 0.06 50 9 .. 16 + 0.02 14 17 .. 32 + 0.00 0 33 .. 64 + 0.00 0 65 .. 128 + 0.00 0 127 .. 256 + 0.00 0 256+ + +hippopotamus.interlaced.gif (all frames) + Percent Count Emission_Length_Bucket + 82.56 672 1 (Literal Code) + 13.02 106 2 + 3.19 26 3 + 0.74 6 4 + 0.25 2 5 + 0.25 2 6 + 0.00 0 7 + 0.00 0 8 + 0.00 0 9 .. 16 + 0.00 0 17 .. 32 + 0.00 0 33 .. 64 + 0.00 0 65 .. 128 + 0.00 0 127 .. 256 + 0.00 0 256+ + +hippopotamus.regular.gif (all frames) + Percent Count Emission_Length_Bucket + 82.66 672 1 (Literal Code) + 13.04 106 2 + 2.83 23 3 + 0.86 7 4 + 0.37 3 5 + 0.25 2 6 + 0.00 0 7 + 0.00 0 8 + 0.00 0 9 .. 16 + 0.00 0 17 .. 32 + 0.00 0 33 .. 64 + 0.00 0 65 .. 128 + 0.00 0 127 .. 256 + 0.00 0 256+ + +muybridge.gif (all frames) + Percent Count Emission_Length_Bucket + 80.23 5863 1 (Literal Code) + 16.80 1228 2 + 2.63 192 3 + 0.27 20 4 + 0.07 5 5 + 0.00 0 6 + 0.00 0 7 + 0.00 0 8 + 0.00 0 9 .. 16 + 0.00 0 17 .. 32 + 0.00 0 33 .. 64 + 0.00 0 65 .. 128 + 0.00 0 127 .. 256 + 0.00 0 256+ + +pjw-thumbnail.gif (all frames) + Percent Count Emission_Length_Bucket + 2.80 4 1 (Literal Code) + 5.59 8 2 + 8.39 12 3 + 11.19 16 4 + 10.49 15 5 + 9.09 13 6 + 9.79 14 7 + 8.39 12 8 + 32.17 46 9 .. 16 + 2.10 3 17 .. 32 + 0.00 0 33 .. 64 + 0.00 0 65 .. 128 + 0.00 0 127 .. 256 + 0.00 0 256+ diff --git a/test/data/hibiscus.primitive.bmp b/test/data/hibiscus.primitive.bmp new file mode 100644 index 00000000..771fbb4a Binary files /dev/null and b/test/data/hibiscus.primitive.bmp differ diff --git a/test/data/hibiscus.primitive.gif b/test/data/hibiscus.primitive.gif new file mode 100644 index 00000000..fe035097 Binary files /dev/null and b/test/data/hibiscus.primitive.gif differ diff --git a/test/data/hibiscus.primitive.jpeg b/test/data/hibiscus.primitive.jpeg new file mode 100644 index 00000000..65ca12b8 Binary files /dev/null and b/test/data/hibiscus.primitive.jpeg differ diff --git a/test/data/hibiscus.primitive.lossless.webp b/test/data/hibiscus.primitive.lossless.webp new file mode 100644 index 00000000..702400d2 Binary files /dev/null and b/test/data/hibiscus.primitive.lossless.webp differ diff --git a/test/data/hibiscus.primitive.lossy.webp b/test/data/hibiscus.primitive.lossy.webp new file mode 100644 index 00000000..ad45209b Binary files /dev/null and b/test/data/hibiscus.primitive.lossy.webp differ diff --git a/test/data/hibiscus.primitive.png b/test/data/hibiscus.primitive.png new file mode 100644 index 00000000..676d476f Binary files /dev/null and b/test/data/hibiscus.primitive.png differ diff --git a/test/data/hibiscus.primitive.svg b/test/data/hibiscus.primitive.svg new file mode 100644 index 00000000..4c728668 --- /dev/null +++ b/test/data/hibiscus.primitive.svg @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/data/hibiscus.primitive.tiff b/test/data/hibiscus.primitive.tiff new file mode 100644 index 00000000..67c89911 Binary files /dev/null and b/test/data/hibiscus.primitive.tiff differ diff --git a/test/data/hibiscus.bmp b/test/data/hibiscus.regular.bmp similarity index 100% rename from test/data/hibiscus.bmp rename to test/data/hibiscus.regular.bmp diff --git a/test/data/hibiscus.gif b/test/data/hibiscus.regular.gif similarity index 100% rename from test/data/hibiscus.gif rename to test/data/hibiscus.regular.gif diff --git a/test/data/hibiscus.jpeg b/test/data/hibiscus.regular.jpeg similarity index 100% rename from test/data/hibiscus.jpeg rename to test/data/hibiscus.regular.jpeg diff --git a/test/data/hibiscus.lossless.webp b/test/data/hibiscus.regular.lossless.webp similarity index 100% rename from test/data/hibiscus.lossless.webp rename to test/data/hibiscus.regular.lossless.webp diff --git a/test/data/hibiscus.lossy.webp b/test/data/hibiscus.regular.lossy.webp similarity index 100% rename from test/data/hibiscus.lossy.webp rename to test/data/hibiscus.regular.lossy.webp diff --git a/test/data/hibiscus.png b/test/data/hibiscus.regular.png similarity index 100% rename from test/data/hibiscus.png rename to test/data/hibiscus.regular.png diff --git a/test/data/hibiscus.tiff b/test/data/hibiscus.regular.tiff similarity index 100% rename from test/data/hibiscus.tiff rename to test/data/hibiscus.regular.tiff