Add script/print-json-token-debug-format.c

This commit is contained in:
Nigel Tao committed 2020-02-26 22:47:31 +11:00
1 parent 550ef90114
commit 193527bd2d
6 files changed
+346 -80

No files matched your search

+29 -17
View File
@@ -201,7 +201,8 @@ test_strconv_parse_number_u64() {
// ---------------- Golden Tests
golden_test json_australian_abc_gt = {
.src_filename = "test/data/australian-abc-local-stations.json", //
.want_filename = "test/data/australian-abc-local-stations.tokens", //
.src_filename = "test/data/australian-abc-local-stations.json", //
};
golden_test json_file_sizes_gt = {
@@ -212,6 +213,11 @@ golden_test json_github_tags_gt = {
.src_filename = "test/data/github-tags.json", //
};
golden_test json_json_things_unformatted_gt = {
.want_filename = "test/data/json-things.unformatted.tokens", //
.src_filename = "test/data/json-things.unformatted.json", //
};
golden_test json_nobel_prizes_gt = {
.src_filename = "test/data/nobel-prizes.json", //
};
@@ -221,24 +227,30 @@ golden_test json_nobel_prizes_gt = {
const char* //
test_wuffs_json_decode_interface() {
CHECK_FOCUS(__func__);
wuffs_json__decoder dec;
CHECK_STATUS("initialize",
wuffs_json__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
const uint64_t vbc = WUFFS_BASE__TOKEN__VBC__STRUCTURE;
const uint64_t vbd = WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP |
WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST |
WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE;
const uint64_t len = 1;
{
wuffs_json__decoder dec;
CHECK_STATUS("initialize",
wuffs_json__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
CHECK_STRING(do_test__wuffs_base__token_decoder(
wuffs_json__decoder__upcast_as__wuffs_base__token_decoder(&dec),
&json_json_things_unformatted_gt));
}
return do_test__wuffs_base__token_decoder(
wuffs_json__decoder__upcast_as__wuffs_base__token_decoder(&dec),
"test/data/github-tags.json", 0, SIZE_MAX,
(vbc << WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) |
(vbd << WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) |
(len << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
{
wuffs_json__decoder dec;
CHECK_STATUS("initialize",
wuffs_json__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
CHECK_STRING(do_test__wuffs_base__token_decoder(
wuffs_json__decoder__upcast_as__wuffs_base__token_decoder(&dec),
&json_australian_abc_gt));
}
return NULL;
}
const char* //
+53 -13
View File
@@ -1061,23 +1061,63 @@ do_test__wuffs_base__io_transformer(wuffs_base__io_transformer* b,
const char* //
do_test__wuffs_base__token_decoder(wuffs_base__token_decoder* b,
const char* src_filename,
size_t src_ri,
size_t src_wi,
uint64_t want_final_token_repr) {
wuffs_base__token_buffer have = ((wuffs_base__token_buffer){
golden_test* gt) {
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = global_have_slice,
});
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
.data = global_want_slice,
});
wuffs_base__token_buffer tok = ((wuffs_base__token_buffer){
.data = global_have_token_slice,
});
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = global_src_slice,
});
CHECK_STRING(read_file_fragment(&src, src_filename, src_ri, src_wi));
CHECK_STATUS("decode_tokens",
wuffs_base__token_decoder__decode_tokens(b, &have, &src));
if ((have.meta.wi > 0) &&
(have.data.ptr[have.meta.wi - 1].repr != want_final_token_repr)) {
RETURN_FAIL("final token: have 0x%" PRIX64 ", want 0x%" PRIX64,
have.data.ptr[have.meta.wi - 1].repr, want_final_token_repr);
if (gt->src_filename) {
CHECK_STRING(
read_file_fragment(&src, gt->src_filename, gt->src_offset0,
gt->src_offset1 ? gt->src_offset1 : UINT64_MAX));
} else {
src.meta.closed = true;
}
return NULL;
CHECK_STATUS("decode_tokens",
wuffs_base__token_decoder__decode_tokens(b, &tok, &src));
uint64_t pos = 0;
while (tok.meta.ri < tok.meta.wi) {
wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++];
uint64_t len = wuffs_base__token__length(t);
if (wuffs_base__token__value(t) != 0) {
uint64_t major = wuffs_base__token__value_major(t);
uint64_t minor = wuffs_base__token__value_minor(t);
if ((have.data.len - have.meta.wi) < 16) {
return "testlib: output is too long";
}
// This 16-bytes-per-token debug format is the same one used by
// `script/print-json-token-debug-format.c`.
uint8_t* ptr = have.data.ptr + have.meta.wi;
wuffs_base__store_u32be(ptr + (0 * 4), (uint32_t)(pos));
wuffs_base__store_u32be(ptr + (1 * 4), (uint32_t)(len));
wuffs_base__store_u32be(ptr + (2 * 4), (uint32_t)(major));
wuffs_base__store_u32be(ptr + (3 * 4), (uint32_t)(minor));
have.meta.wi += 16;
}
pos += len;
if (pos > 0xFFFFFFFF) {
return "testlib: input is too long";
}
}
if (gt->want_filename) {
CHECK_STRING(read_file(&want, gt->want_filename));
} else {
want.meta.closed = true;
}
return check_io_buffers_equal("", &have, &want);
}
+54 -50
View File
@@ -1,13 +1,14 @@
Unless otherwise noted, the \*.gz files were generated by the gzip command line
tool and the \*.deflate and \*.zlib versions were then generated by
script/extract-deflate-offsets.go. Similarly, the \*.giflzw files were
generated by script/extract-giflzw.go and the \*.palette and \*.indexes files
were generated by script/extract-palette-indexes.go
Unless otherwise noted, the `*.gz`files were generated by the `gzip` command
line tool and the `*.deflate` and `*.zlib` versions were then generated by
`script/extract-deflate-offsets.go`. Similarly, the `*.giflzw` files were
generated by `script/extract-giflzw.go` and the `*.palette` and `*.indexes`
files were generated by `script/extract-palette-indexes.go`. The `*.tokens`
files were generated by `script/print-json-token-debug-format.c`.
The \*.jpeg files are usually the canonical versions of the test/data images,
and other versions (\*.bmp, \*.gif, \*.png, \*.tiff) were generated by
ImageMagick's convert command line tool. The \*.wbmp versions were generated by
the `script/convert-png-to-wbmp.go` command line tool. The \*.webp versions
The `*.jpeg` files are usually the canonical versions of the test/data images,
and other versions (`*.bmp`, `*.gif`, `*.png`, `*.tiff`) were generated by
ImageMagick's `convert` command line tool. The `*.wbmp` versions were generated
by the `script/convert-png-to-wbmp.go` command line tool. The `*.webp` versions
were generated by the cwebp command line tool.
---
@@ -15,7 +16,7 @@ were generated by the cwebp command line tool.
The artificial directory holds artificially generated test data, often to
explicitly test corner cases of various file formats. The files there usually
come in families whose names have a common prefix, such as three files
"foo.bar", "foo.bar.qux", "foo.bar.commentary.txt" all prefixed by "foo.bar".
`foo.bar`, `foo.bar.qux`, `foo.bar.commentary.txt` all prefixed by `foo.bar`.
The file whose name is that prefix is usually the canonical (typically
hand-crafted) file, and the other files are then derived from that. Outside of
the artificial directory, the other files in this directory are typically real
@@ -23,81 +24,84 @@ world examples of various file formats, or deriviations of them.
---
animated-red-blue.gif is an original animation by Nigel Tao
`animated-red-blue.gif` is an original animation by Nigel Tao
<nigeltao@golang.org>.
australian-abc-local-stations.json was crawled from
`http://data.gov.au/geoserver/abc-local-stations/wfs?request=GetFeature&typeName=ckan_d534c0e9_a9bf_487b_ac8f_b7877a09d162&outputFormat=json`
`australian-abc-local-stations.json` was crawled from
[data.gov.au](http://data.gov.au/geoserver/abc-local-stations/wfs?request=GetFeature&typeName=ckan_d534c0e9_a9bf_487b_ac8f_b7877a09d162&outputFormat=json).
bricks-\* are various encodings of an original photo by Nigel Tao
`bricks-*` are various encodings of an original photo by Nigel Tao
<nigeltao@golang.org>.
crude-flag.\* is an original animation by Nigel Tao
`crude-flag.*` is an original animation by Nigel Tao
<nigeltao@golang.org>. See the `lib/nie` documentation.
file-sizes.json was created by running `script/print-file-sizes-json.go` in
`file-sizes.json` was created by running `script/print-file-sizes-json.go` in
this repository's root directory.
gifplayer-muybridge.gif is an original animation by Nigel Tao
`gifplayer-muybridge.gif` is an original animation by Nigel Tao
<nigeltao@golang.org>.
harvesters.\* are various encodings of a photo of "The Harvesters" by Pieter
`harvesters.*` are various encodings of a photo of "The Harvesters" by Pieter
Bruegel the Elder, held by the Metropolitan Museum of Art.
`http://www.metmuseum.org/art/collection/search/435809` lists that image as in
the public domain.
[www.metmuseum.org](http://www.metmuseum.org/art/collection/search/435809)
lists that image as in the public domain.
github-tags.json was crawled from
`https://api.github.com/repos/google/wuffs/tags`
`github-tags.json` was crawled from
[api.github.com](https://api.github.com/repos/google/wuffs/tags).
hat.\* are various encodings of a photo of "Self-Portrait with a Straw Hat
`hat.*` are various encodings of a photo of "Self-Portrait with a Straw Hat
(obverse: The Potato Peeler)" by Vincent van Gogh, held by the Metropolitan
Museum of art. `http://www.metmuseum.org/art/collection/search/436532` lists
Museum of art.
[www.metmuseum.org](http://www.metmuseum.org/art/collection/search/436532)
lists that image as in the public domain.
`hibiscus.regular.*` are various encodings of a photo of "Hibiscus and Parrots"
by Louis Comfort Tiffany, held by the Metropolitan Museum of Art.
[www.metmuseum.org](http://www.metmuseum.org/art/collection/search/13503) lists
that image as in the public domain.
hibiscus.regular.\* are various encodings of a photo of "Hibiscus and Parrots"
by Louis Comfort Tiffany, held by the Metropolitan Museum of Art.
`http://www.metmuseum.org/art/collection/search/13503` lists that image as in
the public domain.
hibiscus.primitive.\* is the result of running the hibiscus.regular.png image
through `github.com/fogleman/primitive` and other programs:
`hibiscus.primitive.*` is the result of running the `hibiscus.regular.png`
image through `github.com/fogleman/primitive` and other programs:
- `primitive -i hibiscus.regular.png -o hibiscus.primitive.svg -s 442 -n 32`
- `inkscape -z -e hibiscus.primitive.png hibiscus.primitive.svg`
- Various invocations of `convert` and `cwebp`
hippopotamus.\* are various encodings of a cropping of a photo of "Hippopotamus
(William)", held by the Metropolitan Museum of Art.
`http://www.metmuseum.org/art/collection/search/544227` lists that image as in
the public domain.
`hippopotamus.*` are various encodings of a cropping of a photo of
"Hippopotamus (William)", held by the Metropolitan Museum of Art.
[www.metmuseum.org](http://www.metmuseum.org/art/collection/search/544227)
lists that image as in the public domain.
`http://www.metmuseum.org/about-the-met/policies-and-documents/image-resources`
[www.metmuseum.org](http://www.metmuseum.org/about-the-met/policies-and-documents/image-resources)
says that "You are welcome to use images of artworks in The Met collection that
the Museum believes to be in the public domain, or those to which the Museum
waives any copyright it might have, for any purpose, including commercial and
noncommercial use, free of charge and without requiring permission from the
Museum."
json-things.\* are original JSON objects by Nigel Tao <nigeltao@golang.org>.
`json-things.*` are original JSON objects by Nigel Tao <nigeltao@golang.org>.
midsummer.txt is an excerpt of Shakespeare's "A Midsummer Night's Dream",
copied from `http://shakespeare.mit.edu/midsummer/midsummer.1.1.html`
`midsummer.txt` is an excerpt of Shakespeare's "A Midsummer Night's Dream",
copied from
[shakespeare.mit.edu](http://shakespeare.mit.edu/midsummer/midsummer.1.1.html).
muybridge.gif is derived from
`https://en.wikipedia.org/wiki/File:Muybridge_race_horse_animated.gif` which is
in the public domain.
`muybridge.gif` is derived from
[en.wikipedia.org](https://en.wikipedia.org/wiki/File:Muybridge_race_horse_animated.gif)
which is in the public domain.
nobel-prizes.json was crawled from `http://api.nobelprize.org/v1/prize.json`
`nobel-prizes.json` was crawled from
[api.nobelprize.org](http://api.nobelprize.org/v1/prize.json).
pi.txt contains the digits of pi.
`pi.txt` contains the digits of pi.
pjw-thumbnail.\* are various encodings of an image derived from an iconic,
`pjw-thumbnail.*` are various encodings of an image derived from an iconic,
original photo of Peter J. Weinberger by Rob Pike <r@golang.org>.
romeo.txt is an excerpt of Shakespeare's "Romeo and Juliet", copied from
`http://shakespeare.mit.edu/romeo_juliet/romeo_juliet.2.2.html`
`romeo.txt` is an excerpt of Shakespeare's "Romeo and Juliet", copied from
[shakespeare.mit.edu](http://shakespeare.mit.edu/romeo_juliet/romeo_juliet.2.2.html).
romeo.txt.fixed-huff.deflate was derived from romeo.txt by a custom program to
use fixed (not dynamic) Huffman tables for the deflate encoding.
`romeo.txt.fixed-huff.deflate` was derived from `romeo.txt` by a custom program
to use fixed (not dynamic) Huffman tables for the deflate encoding.
sheep-more.rac is a RAC-compression of original text by Nigel Tao
`sheep-more.rac` is a RAC-compression of original text by Nigel Tao
<nigeltao@golang.org>.
Binary file not shown.
Binary file not shown.