Add script/print-json-token-debug-format.c

This commit is contained in:
Nigel Tao committed 2020-02-26 22:47:31 +11:00
1 parent 550ef90114
commit 193527bd2d
6 files changed
+346 -80

No files matched your search

+210
View File
@@ -0,0 +1,210 @@
// Copyright 2020 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------
// print-json-token-debug-format.c parses JSON from stdin and prints the
// resulting token stream, eliding any non-essential (e.g. whitespace) tokens.
//
// The output format is only for debugging, and certainly not for long term
// storage. It isn't guaranteed to be stable between versions of this program
// and of the Wuffs standard library.
//
// It prints 16 bytes (4 big-endian uint32's) per token: the token's position
// (total length of all previous tokens), length, major value and minor value.
// Together with the hexadecimal WUFFS_BASE__TOKEN__ETC constants defined in
// token-public.h, this format is somewhat human-readable when piped through a
// hex-dump program (e.g. /usr/bin/hd).
//
// If the input or output is larger than the program's buffers (64 MiB and
// 131072 tokens by default), there may be multiple valid tokenizations of any
// given input. For example, if a source string "abcde" straddles an I/O
// boundary, it may be tokenized as a 5-length complete string or as a 3-length
// incomplete string followed by a 2-length complete string.
//
// A Wuffs token stream, in general, can support inputs more than (1 << 32)
// bytes long, but this program can not, as it tracks the tokens' cumulative
// position as a uint32.
#include <inttypes.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
// Wuffs ships as a "single file C library" or "header file library" as per
// https://github.com/nothings/stb/blob/master/docs/stb_howto.txt
//
// To use that single file as a "foo.c"-like implementation, instead of a
// "foo.h"-like header, #define WUFFS_IMPLEMENTATION before #include'ing or
// compiling it.
#define WUFFS_IMPLEMENTATION
// Defining the WUFFS_CONFIG__MODULE* macros are optional, but it lets users of
// release/c/etc.c whitelist which parts of Wuffs to build. That file contains
// the entire Wuffs standard library, implementing a variety of codecs and file
// formats. Without this macro definition, an optimizing compiler or linker may
// very well discard Wuffs code for unused codecs, but listing the Wuffs
// modules we use makes that process explicit. Preprocessing means that such
// code simply isn't compiled.
#define WUFFS_CONFIG__MODULES
#define WUFFS_CONFIG__MODULE__BASE
#define WUFFS_CONFIG__MODULE__JSON
// If building this program in an environment that doesn't easily accommodate
// relative includes, you can use the script/inline-c-relative-includes.go
// program to generate a stand-alone C++ file.
#include "../release/c/wuffs-unsupported-snapshot.c"
#ifndef SRC_BUFFER_SIZE
#define SRC_BUFFER_SIZE (64 * 1024 * 1024)
#endif
#ifndef TOKEN_BUFFER_SIZE
#define TOKEN_BUFFER_SIZE (128 * 1024)
#endif
uint8_t src_buffer[SRC_BUFFER_SIZE];
wuffs_base__token tok_buffer[TOKEN_BUFFER_SIZE];
wuffs_base__io_buffer src;
wuffs_base__token_buffer tok;
wuffs_json__decoder dec;
wuffs_base__status dec_status;
static inline void //
store_u32be(uint8_t* p, uint32_t x) {
p[0] = (uint8_t)(x >> 24);
p[1] = (uint8_t)(x >> 16);
p[2] = (uint8_t)(x >> 8);
p[3] = (uint8_t)(x >> 0);
}
#define TRY(error_msg) \
do { \
const char* z = error_msg; \
if (z) { \
return z; \
} \
} while (false)
const char* //
read_src() {
if (src.meta.closed) {
return "main: internal error: read requested on a closed source";
}
wuffs_base__io_buffer__compact(&src);
if (src.meta.wi >= src.data.len) {
return "main: src buffer is full";
}
size_t n = fread(src.data.ptr + src.meta.wi, sizeof(uint8_t),
src.data.len - src.meta.wi, stdin);
src.meta.wi += n;
src.meta.closed = feof(stdin);
if ((n == 0) && !src.meta.closed) {
return "main: read error";
}
return NULL;
}
// ----
const char* //
main1(int argc, char** argv) {
src = wuffs_base__make_io_buffer(
wuffs_base__make_slice_u8(src_buffer, SRC_BUFFER_SIZE),
wuffs_base__empty_io_buffer_meta());
tok = wuffs_base__make_token_buffer(
wuffs_base__make_slice_token(tok_buffer, TOKEN_BUFFER_SIZE),
wuffs_base__empty_token_buffer_meta());
wuffs_base__status init_status = wuffs_json__decoder__initialize(
&dec, sizeof__wuffs_json__decoder(), WUFFS_VERSION, 0);
if (!wuffs_base__status__is_ok(&init_status)) {
return wuffs_base__status__message(&init_status);
}
uint64_t pos = 0;
while (true) {
wuffs_base__status status =
wuffs_json__decoder__decode_tokens(&dec, &tok, &src);
while (tok.meta.ri < tok.meta.wi) {
wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++];
uint64_t len = wuffs_base__token__length(t);
if (wuffs_base__token__value(t) != 0) {
uint64_t major = wuffs_base__token__value_major(t);
uint64_t minor = wuffs_base__token__value_minor(t);
uint8_t buf[16];
store_u32be(&buf[0 * 4], (uint32_t)(pos));
store_u32be(&buf[1 * 4], (uint32_t)(len));
store_u32be(&buf[2 * 4], (uint32_t)(major));
store_u32be(&buf[3 * 4], (uint32_t)(minor));
const int stdout_fd = 1;
write(stdout_fd, &buf[0], 16);
}
pos += len;
if (pos > 0xFFFFFFFF) {
return "main: input is too long";
}
}
if (status.repr == NULL) {
return NULL;
} else if (status.repr == wuffs_base__suspension__short_read) {
TRY(read_src());
} else if (status.repr == wuffs_base__suspension__short_write) {
wuffs_base__token_buffer__compact(&tok);
} else {
return wuffs_base__status__message(&status);
}
}
}
// ----
int //
compute_exit_code(const char* status_msg) {
if (!status_msg) {
return 0;
}
size_t n = strnlen(status_msg, 2047);
if (n >= 2047) {
status_msg = "main: internal error: error message is too long";
n = strnlen(status_msg, 2047);
}
fprintf(stderr, "%s\n", status_msg);
// Return an exit code of 1 for regular (forseen) errors, e.g. badly
// formatted or unsupported input.
//
// Return an exit code of 2 for internal (exceptional) errors, e.g. defensive
// run-time checks found that an internal invariant did not hold.
//
// Automated testing, including badly formatted inputs, can therefore
// discriminate between expected failure (exit code 1) and unexpected failure
// (other non-zero exit codes). Specifically, exit code 2 for internal
// invariant violation, exit code 139 (which is 128 + SIGSEGV on x86_64
// linux) for a segmentation fault (e.g. null pointer dereference).
return strstr(status_msg, "internal error:") ? 2 : 1;
}
int //
main(int argc, char** argv) {
const char* z = main1(argc, argv);
int exit_code = compute_exit_code(z);
return exit_code;
}
+29 -17
View File
@@ -201,7 +201,8 @@ test_strconv_parse_number_u64() {
// ---------------- Golden Tests
golden_test json_australian_abc_gt = {
.src_filename = "test/data/australian-abc-local-stations.json", //
.want_filename = "test/data/australian-abc-local-stations.tokens", //
.src_filename = "test/data/australian-abc-local-stations.json", //
};
golden_test json_file_sizes_gt = {
@@ -212,6 +213,11 @@ golden_test json_github_tags_gt = {
.src_filename = "test/data/github-tags.json", //
};
golden_test json_json_things_unformatted_gt = {
.want_filename = "test/data/json-things.unformatted.tokens", //
.src_filename = "test/data/json-things.unformatted.json", //
};
golden_test json_nobel_prizes_gt = {
.src_filename = "test/data/nobel-prizes.json", //
};
@@ -221,24 +227,30 @@ golden_test json_nobel_prizes_gt = {
const char* //
test_wuffs_json_decode_interface() {
CHECK_FOCUS(__func__);
wuffs_json__decoder dec;
CHECK_STATUS("initialize",
wuffs_json__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
const uint64_t vbc = WUFFS_BASE__TOKEN__VBC__STRUCTURE;
const uint64_t vbd = WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP |
WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST |
WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE;
const uint64_t len = 1;
{
wuffs_json__decoder dec;
CHECK_STATUS("initialize",
wuffs_json__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
CHECK_STRING(do_test__wuffs_base__token_decoder(
wuffs_json__decoder__upcast_as__wuffs_base__token_decoder(&dec),
&json_json_things_unformatted_gt));
}
return do_test__wuffs_base__token_decoder(
wuffs_json__decoder__upcast_as__wuffs_base__token_decoder(&dec),
"test/data/github-tags.json", 0, SIZE_MAX,
(vbc << WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) |
(vbd << WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) |
(len << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
{
wuffs_json__decoder dec;
CHECK_STATUS("initialize",
wuffs_json__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
CHECK_STRING(do_test__wuffs_base__token_decoder(
wuffs_json__decoder__upcast_as__wuffs_base__token_decoder(&dec),
&json_australian_abc_gt));
}
return NULL;
}
const char* //
+53 -13
View File
@@ -1061,23 +1061,63 @@ do_test__wuffs_base__io_transformer(wuffs_base__io_transformer* b,
const char* //
do_test__wuffs_base__token_decoder(wuffs_base__token_decoder* b,
const char* src_filename,
size_t src_ri,
size_t src_wi,
uint64_t want_final_token_repr) {
wuffs_base__token_buffer have = ((wuffs_base__token_buffer){
golden_test* gt) {
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = global_have_slice,
});
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
.data = global_want_slice,
});
wuffs_base__token_buffer tok = ((wuffs_base__token_buffer){
.data = global_have_token_slice,
});
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = global_src_slice,
});
CHECK_STRING(read_file_fragment(&src, src_filename, src_ri, src_wi));
CHECK_STATUS("decode_tokens",
wuffs_base__token_decoder__decode_tokens(b, &have, &src));
if ((have.meta.wi > 0) &&
(have.data.ptr[have.meta.wi - 1].repr != want_final_token_repr)) {
RETURN_FAIL("final token: have 0x%" PRIX64 ", want 0x%" PRIX64,
have.data.ptr[have.meta.wi - 1].repr, want_final_token_repr);
if (gt->src_filename) {
CHECK_STRING(
read_file_fragment(&src, gt->src_filename, gt->src_offset0,
gt->src_offset1 ? gt->src_offset1 : UINT64_MAX));
} else {
src.meta.closed = true;
}
return NULL;
CHECK_STATUS("decode_tokens",
wuffs_base__token_decoder__decode_tokens(b, &tok, &src));
uint64_t pos = 0;
while (tok.meta.ri < tok.meta.wi) {
wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++];
uint64_t len = wuffs_base__token__length(t);
if (wuffs_base__token__value(t) != 0) {
uint64_t major = wuffs_base__token__value_major(t);
uint64_t minor = wuffs_base__token__value_minor(t);
if ((have.data.len - have.meta.wi) < 16) {
return "testlib: output is too long";
}
// This 16-bytes-per-token debug format is the same one used by
// `script/print-json-token-debug-format.c`.
uint8_t* ptr = have.data.ptr + have.meta.wi;
wuffs_base__store_u32be(ptr + (0 * 4), (uint32_t)(pos));
wuffs_base__store_u32be(ptr + (1 * 4), (uint32_t)(len));
wuffs_base__store_u32be(ptr + (2 * 4), (uint32_t)(major));
wuffs_base__store_u32be(ptr + (3 * 4), (uint32_t)(minor));
have.meta.wi += 16;
}
pos += len;
if (pos > 0xFFFFFFFF) {
return "testlib: input is too long";
}
}
if (gt->want_filename) {
CHECK_STRING(read_file(&want, gt->want_filename));
} else {
want.meta.closed = true;
}
return check_io_buffers_equal("", &have, &want);
}
+54 -50
View File
@@ -1,13 +1,14 @@
Unless otherwise noted, the \*.gz files were generated by the gzip command line
tool and the \*.deflate and \*.zlib versions were then generated by
script/extract-deflate-offsets.go. Similarly, the \*.giflzw files were
generated by script/extract-giflzw.go and the \*.palette and \*.indexes files
were generated by script/extract-palette-indexes.go
Unless otherwise noted, the `*.gz`files were generated by the `gzip` command
line tool and the `*.deflate` and `*.zlib` versions were then generated by
`script/extract-deflate-offsets.go`. Similarly, the `*.giflzw` files were
generated by `script/extract-giflzw.go` and the `*.palette` and `*.indexes`
files were generated by `script/extract-palette-indexes.go`. The `*.tokens`
files were generated by `script/print-json-token-debug-format.c`.
The \*.jpeg files are usually the canonical versions of the test/data images,
and other versions (\*.bmp, \*.gif, \*.png, \*.tiff) were generated by
ImageMagick's convert command line tool. The \*.wbmp versions were generated by
the `script/convert-png-to-wbmp.go` command line tool. The \*.webp versions
The `*.jpeg` files are usually the canonical versions of the test/data images,
and other versions (`*.bmp`, `*.gif`, `*.png`, `*.tiff`) were generated by
ImageMagick's `convert` command line tool. The `*.wbmp` versions were generated
by the `script/convert-png-to-wbmp.go` command line tool. The `*.webp` versions
were generated by the cwebp command line tool.
---
@@ -15,7 +16,7 @@ were generated by the cwebp command line tool.
The artificial directory holds artificially generated test data, often to
explicitly test corner cases of various file formats. The files there usually
come in families whose names have a common prefix, such as three files
"foo.bar", "foo.bar.qux", "foo.bar.commentary.txt" all prefixed by "foo.bar".
`foo.bar`, `foo.bar.qux`, `foo.bar.commentary.txt` all prefixed by `foo.bar`.
The file whose name is that prefix is usually the canonical (typically
hand-crafted) file, and the other files are then derived from that. Outside of
the artificial directory, the other files in this directory are typically real
@@ -23,81 +24,84 @@ world examples of various file formats, or deriviations of them.
---
animated-red-blue.gif is an original animation by Nigel Tao
`animated-red-blue.gif` is an original animation by Nigel Tao
<nigeltao@golang.org>.
australian-abc-local-stations.json was crawled from
`http://data.gov.au/geoserver/abc-local-stations/wfs?request=GetFeature&typeName=ckan_d534c0e9_a9bf_487b_ac8f_b7877a09d162&outputFormat=json`
`australian-abc-local-stations.json` was crawled from
[data.gov.au](http://data.gov.au/geoserver/abc-local-stations/wfs?request=GetFeature&typeName=ckan_d534c0e9_a9bf_487b_ac8f_b7877a09d162&outputFormat=json).
bricks-\* are various encodings of an original photo by Nigel Tao
`bricks-*` are various encodings of an original photo by Nigel Tao
<nigeltao@golang.org>.
crude-flag.\* is an original animation by Nigel Tao
`crude-flag.*` is an original animation by Nigel Tao
<nigeltao@golang.org>. See the `lib/nie` documentation.
file-sizes.json was created by running `script/print-file-sizes-json.go` in
`file-sizes.json` was created by running `script/print-file-sizes-json.go` in
this repository's root directory.
gifplayer-muybridge.gif is an original animation by Nigel Tao
`gifplayer-muybridge.gif` is an original animation by Nigel Tao
<nigeltao@golang.org>.
harvesters.\* are various encodings of a photo of "The Harvesters" by Pieter
`harvesters.*` are various encodings of a photo of "The Harvesters" by Pieter
Bruegel the Elder, held by the Metropolitan Museum of Art.
`http://www.metmuseum.org/art/collection/search/435809` lists that image as in
the public domain.
[www.metmuseum.org](http://www.metmuseum.org/art/collection/search/435809)
lists that image as in the public domain.
github-tags.json was crawled from
`https://api.github.com/repos/google/wuffs/tags`
`github-tags.json` was crawled from
[api.github.com](https://api.github.com/repos/google/wuffs/tags).
hat.\* are various encodings of a photo of "Self-Portrait with a Straw Hat
`hat.*` are various encodings of a photo of "Self-Portrait with a Straw Hat
(obverse: The Potato Peeler)" by Vincent van Gogh, held by the Metropolitan
Museum of art. `http://www.metmuseum.org/art/collection/search/436532` lists
Museum of art.
[www.metmuseum.org](http://www.metmuseum.org/art/collection/search/436532)
lists that image as in the public domain.
`hibiscus.regular.*` are various encodings of a photo of "Hibiscus and Parrots"
by Louis Comfort Tiffany, held by the Metropolitan Museum of Art.
[www.metmuseum.org](http://www.metmuseum.org/art/collection/search/13503) lists
that image as in the public domain.
hibiscus.regular.\* are various encodings of a photo of "Hibiscus and Parrots"
by Louis Comfort Tiffany, held by the Metropolitan Museum of Art.
`http://www.metmuseum.org/art/collection/search/13503` lists that image as in
the public domain.
hibiscus.primitive.\* is the result of running the hibiscus.regular.png image
through `github.com/fogleman/primitive` and other programs:
`hibiscus.primitive.*` is the result of running the `hibiscus.regular.png`
image through `github.com/fogleman/primitive` and other programs:
- `primitive -i hibiscus.regular.png -o hibiscus.primitive.svg -s 442 -n 32`
- `inkscape -z -e hibiscus.primitive.png hibiscus.primitive.svg`
- Various invocations of `convert` and `cwebp`
hippopotamus.\* are various encodings of a cropping of a photo of "Hippopotamus
(William)", held by the Metropolitan Museum of Art.
`http://www.metmuseum.org/art/collection/search/544227` lists that image as in
the public domain.
`hippopotamus.*` are various encodings of a cropping of a photo of
"Hippopotamus (William)", held by the Metropolitan Museum of Art.
[www.metmuseum.org](http://www.metmuseum.org/art/collection/search/544227)
lists that image as in the public domain.
`http://www.metmuseum.org/about-the-met/policies-and-documents/image-resources`
[www.metmuseum.org](http://www.metmuseum.org/about-the-met/policies-and-documents/image-resources)
says that "You are welcome to use images of artworks in The Met collection that
the Museum believes to be in the public domain, or those to which the Museum
waives any copyright it might have, for any purpose, including commercial and
noncommercial use, free of charge and without requiring permission from the
Museum."
json-things.\* are original JSON objects by Nigel Tao <nigeltao@golang.org>.
`json-things.*` are original JSON objects by Nigel Tao <nigeltao@golang.org>.
midsummer.txt is an excerpt of Shakespeare's "A Midsummer Night's Dream",
copied from `http://shakespeare.mit.edu/midsummer/midsummer.1.1.html`
`midsummer.txt` is an excerpt of Shakespeare's "A Midsummer Night's Dream",
copied from
[shakespeare.mit.edu](http://shakespeare.mit.edu/midsummer/midsummer.1.1.html).
muybridge.gif is derived from
`https://en.wikipedia.org/wiki/File:Muybridge_race_horse_animated.gif` which is
in the public domain.
`muybridge.gif` is derived from
[en.wikipedia.org](https://en.wikipedia.org/wiki/File:Muybridge_race_horse_animated.gif)
which is in the public domain.
nobel-prizes.json was crawled from `http://api.nobelprize.org/v1/prize.json`
`nobel-prizes.json` was crawled from
[api.nobelprize.org](http://api.nobelprize.org/v1/prize.json).
pi.txt contains the digits of pi.
`pi.txt` contains the digits of pi.
pjw-thumbnail.\* are various encodings of an image derived from an iconic,
`pjw-thumbnail.*` are various encodings of an image derived from an iconic,
original photo of Peter J. Weinberger by Rob Pike <r@golang.org>.
romeo.txt is an excerpt of Shakespeare's "Romeo and Juliet", copied from
`http://shakespeare.mit.edu/romeo_juliet/romeo_juliet.2.2.html`
`romeo.txt` is an excerpt of Shakespeare's "Romeo and Juliet", copied from
[shakespeare.mit.edu](http://shakespeare.mit.edu/romeo_juliet/romeo_juliet.2.2.html).
romeo.txt.fixed-huff.deflate was derived from romeo.txt by a custom program to
use fixed (not dynamic) Huffman tables for the deflate encoding.
`romeo.txt.fixed-huff.deflate` was derived from `romeo.txt` by a custom program
to use fixed (not dynamic) Huffman tables for the deflate encoding.
sheep-more.rac is a RAC-compression of original text by Nigel Tao
`sheep-more.rac` is a RAC-compression of original text by Nigel Tao
<nigeltao@golang.org>.
Binary file not shown.
Binary file not shown.