mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 16:32:11 +01:00
Add script/print-json-token-debug-format.c
This commit is contained in:
1 parent
550ef90114
commit
193527bd2d
6 files changed
+346
-80
No files matched your search
+29
-17
@@ -201,7 +201,8 @@ test_strconv_parse_number_u64() {
|
||||
// ---------------- Golden Tests
|
||||
|
||||
golden_test json_australian_abc_gt = {
|
||||
.src_filename = "test/data/australian-abc-local-stations.json", //
|
||||
.want_filename = "test/data/australian-abc-local-stations.tokens", //
|
||||
.src_filename = "test/data/australian-abc-local-stations.json", //
|
||||
};
|
||||
|
||||
golden_test json_file_sizes_gt = {
|
||||
@@ -212,6 +213,11 @@ golden_test json_github_tags_gt = {
|
||||
.src_filename = "test/data/github-tags.json", //
|
||||
};
|
||||
|
||||
golden_test json_json_things_unformatted_gt = {
|
||||
.want_filename = "test/data/json-things.unformatted.tokens", //
|
||||
.src_filename = "test/data/json-things.unformatted.json", //
|
||||
};
|
||||
|
||||
golden_test json_nobel_prizes_gt = {
|
||||
.src_filename = "test/data/nobel-prizes.json", //
|
||||
};
|
||||
@@ -221,24 +227,30 @@ golden_test json_nobel_prizes_gt = {
|
||||
const char* //
|
||||
test_wuffs_json_decode_interface() {
|
||||
CHECK_FOCUS(__func__);
|
||||
wuffs_json__decoder dec;
|
||||
CHECK_STATUS("initialize",
|
||||
wuffs_json__decoder__initialize(
|
||||
&dec, sizeof dec, WUFFS_VERSION,
|
||||
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
|
||||
|
||||
const uint64_t vbc = WUFFS_BASE__TOKEN__VBC__STRUCTURE;
|
||||
const uint64_t vbd = WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP |
|
||||
WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST |
|
||||
WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE;
|
||||
const uint64_t len = 1;
|
||||
{
|
||||
wuffs_json__decoder dec;
|
||||
CHECK_STATUS("initialize",
|
||||
wuffs_json__decoder__initialize(
|
||||
&dec, sizeof dec, WUFFS_VERSION,
|
||||
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
|
||||
CHECK_STRING(do_test__wuffs_base__token_decoder(
|
||||
wuffs_json__decoder__upcast_as__wuffs_base__token_decoder(&dec),
|
||||
&json_json_things_unformatted_gt));
|
||||
}
|
||||
|
||||
return do_test__wuffs_base__token_decoder(
|
||||
wuffs_json__decoder__upcast_as__wuffs_base__token_decoder(&dec),
|
||||
"test/data/github-tags.json", 0, SIZE_MAX,
|
||||
(vbc << WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) |
|
||||
(vbd << WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) |
|
||||
(len << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
{
|
||||
wuffs_json__decoder dec;
|
||||
CHECK_STATUS("initialize",
|
||||
wuffs_json__decoder__initialize(
|
||||
&dec, sizeof dec, WUFFS_VERSION,
|
||||
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
|
||||
CHECK_STRING(do_test__wuffs_base__token_decoder(
|
||||
wuffs_json__decoder__upcast_as__wuffs_base__token_decoder(&dec),
|
||||
&json_australian_abc_gt));
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
|
||||
+53
-13
@@ -1061,23 +1061,63 @@ do_test__wuffs_base__io_transformer(wuffs_base__io_transformer* b,
|
||||
|
||||
const char* //
|
||||
do_test__wuffs_base__token_decoder(wuffs_base__token_decoder* b,
|
||||
const char* src_filename,
|
||||
size_t src_ri,
|
||||
size_t src_wi,
|
||||
uint64_t want_final_token_repr) {
|
||||
wuffs_base__token_buffer have = ((wuffs_base__token_buffer){
|
||||
golden_test* gt) {
|
||||
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
|
||||
.data = global_have_slice,
|
||||
});
|
||||
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
|
||||
.data = global_want_slice,
|
||||
});
|
||||
wuffs_base__token_buffer tok = ((wuffs_base__token_buffer){
|
||||
.data = global_have_token_slice,
|
||||
});
|
||||
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
|
||||
.data = global_src_slice,
|
||||
});
|
||||
CHECK_STRING(read_file_fragment(&src, src_filename, src_ri, src_wi));
|
||||
CHECK_STATUS("decode_tokens",
|
||||
wuffs_base__token_decoder__decode_tokens(b, &have, &src));
|
||||
if ((have.meta.wi > 0) &&
|
||||
(have.data.ptr[have.meta.wi - 1].repr != want_final_token_repr)) {
|
||||
RETURN_FAIL("final token: have 0x%" PRIX64 ", want 0x%" PRIX64,
|
||||
have.data.ptr[have.meta.wi - 1].repr, want_final_token_repr);
|
||||
|
||||
if (gt->src_filename) {
|
||||
CHECK_STRING(
|
||||
read_file_fragment(&src, gt->src_filename, gt->src_offset0,
|
||||
gt->src_offset1 ? gt->src_offset1 : UINT64_MAX));
|
||||
} else {
|
||||
src.meta.closed = true;
|
||||
}
|
||||
return NULL;
|
||||
|
||||
CHECK_STATUS("decode_tokens",
|
||||
wuffs_base__token_decoder__decode_tokens(b, &tok, &src));
|
||||
|
||||
uint64_t pos = 0;
|
||||
while (tok.meta.ri < tok.meta.wi) {
|
||||
wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++];
|
||||
uint64_t len = wuffs_base__token__length(t);
|
||||
|
||||
if (wuffs_base__token__value(t) != 0) {
|
||||
uint64_t major = wuffs_base__token__value_major(t);
|
||||
uint64_t minor = wuffs_base__token__value_minor(t);
|
||||
|
||||
if ((have.data.len - have.meta.wi) < 16) {
|
||||
return "testlib: output is too long";
|
||||
}
|
||||
// This 16-bytes-per-token debug format is the same one used by
|
||||
// `script/print-json-token-debug-format.c`.
|
||||
uint8_t* ptr = have.data.ptr + have.meta.wi;
|
||||
wuffs_base__store_u32be(ptr + (0 * 4), (uint32_t)(pos));
|
||||
wuffs_base__store_u32be(ptr + (1 * 4), (uint32_t)(len));
|
||||
wuffs_base__store_u32be(ptr + (2 * 4), (uint32_t)(major));
|
||||
wuffs_base__store_u32be(ptr + (3 * 4), (uint32_t)(minor));
|
||||
have.meta.wi += 16;
|
||||
}
|
||||
|
||||
pos += len;
|
||||
if (pos > 0xFFFFFFFF) {
|
||||
return "testlib: input is too long";
|
||||
}
|
||||
}
|
||||
|
||||
if (gt->want_filename) {
|
||||
CHECK_STRING(read_file(&want, gt->want_filename));
|
||||
} else {
|
||||
want.meta.closed = true;
|
||||
}
|
||||
return check_io_buffers_equal("", &have, &want);
|
||||
}
|
||||
+54
-50
@@ -1,13 +1,14 @@
|
||||
Unless otherwise noted, the \*.gz files were generated by the gzip command line
|
||||
tool and the \*.deflate and \*.zlib versions were then generated by
|
||||
script/extract-deflate-offsets.go. Similarly, the \*.giflzw files were
|
||||
generated by script/extract-giflzw.go and the \*.palette and \*.indexes files
|
||||
were generated by script/extract-palette-indexes.go
|
||||
Unless otherwise noted, the `*.gz`files were generated by the `gzip` command
|
||||
line tool and the `*.deflate` and `*.zlib` versions were then generated by
|
||||
`script/extract-deflate-offsets.go`. Similarly, the `*.giflzw` files were
|
||||
generated by `script/extract-giflzw.go` and the `*.palette` and `*.indexes`
|
||||
files were generated by `script/extract-palette-indexes.go`. The `*.tokens`
|
||||
files were generated by `script/print-json-token-debug-format.c`.
|
||||
|
||||
The \*.jpeg files are usually the canonical versions of the test/data images,
|
||||
and other versions (\*.bmp, \*.gif, \*.png, \*.tiff) were generated by
|
||||
ImageMagick's convert command line tool. The \*.wbmp versions were generated by
|
||||
the `script/convert-png-to-wbmp.go` command line tool. The \*.webp versions
|
||||
The `*.jpeg` files are usually the canonical versions of the test/data images,
|
||||
and other versions (`*.bmp`, `*.gif`, `*.png`, `*.tiff`) were generated by
|
||||
ImageMagick's `convert` command line tool. The `*.wbmp` versions were generated
|
||||
by the `script/convert-png-to-wbmp.go` command line tool. The `*.webp` versions
|
||||
were generated by the cwebp command line tool.
|
||||
|
||||
---
|
||||
@@ -15,7 +16,7 @@ were generated by the cwebp command line tool.
|
||||
The artificial directory holds artificially generated test data, often to
|
||||
explicitly test corner cases of various file formats. The files there usually
|
||||
come in families whose names have a common prefix, such as three files
|
||||
"foo.bar", "foo.bar.qux", "foo.bar.commentary.txt" all prefixed by "foo.bar".
|
||||
`foo.bar`, `foo.bar.qux`, `foo.bar.commentary.txt` all prefixed by `foo.bar`.
|
||||
The file whose name is that prefix is usually the canonical (typically
|
||||
hand-crafted) file, and the other files are then derived from that. Outside of
|
||||
the artificial directory, the other files in this directory are typically real
|
||||
@@ -23,81 +24,84 @@ world examples of various file formats, or deriviations of them.
|
||||
|
||||
---
|
||||
|
||||
animated-red-blue.gif is an original animation by Nigel Tao
|
||||
`animated-red-blue.gif` is an original animation by Nigel Tao
|
||||
<nigeltao@golang.org>.
|
||||
|
||||
australian-abc-local-stations.json was crawled from
|
||||
`http://data.gov.au/geoserver/abc-local-stations/wfs?request=GetFeature&typeName=ckan_d534c0e9_a9bf_487b_ac8f_b7877a09d162&outputFormat=json`
|
||||
`australian-abc-local-stations.json` was crawled from
|
||||
[data.gov.au](http://data.gov.au/geoserver/abc-local-stations/wfs?request=GetFeature&typeName=ckan_d534c0e9_a9bf_487b_ac8f_b7877a09d162&outputFormat=json).
|
||||
|
||||
bricks-\* are various encodings of an original photo by Nigel Tao
|
||||
`bricks-*` are various encodings of an original photo by Nigel Tao
|
||||
<nigeltao@golang.org>.
|
||||
|
||||
crude-flag.\* is an original animation by Nigel Tao
|
||||
`crude-flag.*` is an original animation by Nigel Tao
|
||||
<nigeltao@golang.org>. See the `lib/nie` documentation.
|
||||
|
||||
file-sizes.json was created by running `script/print-file-sizes-json.go` in
|
||||
`file-sizes.json` was created by running `script/print-file-sizes-json.go` in
|
||||
this repository's root directory.
|
||||
|
||||
gifplayer-muybridge.gif is an original animation by Nigel Tao
|
||||
`gifplayer-muybridge.gif` is an original animation by Nigel Tao
|
||||
<nigeltao@golang.org>.
|
||||
|
||||
harvesters.\* are various encodings of a photo of "The Harvesters" by Pieter
|
||||
`harvesters.*` are various encodings of a photo of "The Harvesters" by Pieter
|
||||
Bruegel the Elder, held by the Metropolitan Museum of Art.
|
||||
`http://www.metmuseum.org/art/collection/search/435809` lists that image as in
|
||||
the public domain.
|
||||
[www.metmuseum.org](http://www.metmuseum.org/art/collection/search/435809)
|
||||
lists that image as in the public domain.
|
||||
|
||||
github-tags.json was crawled from
|
||||
`https://api.github.com/repos/google/wuffs/tags`
|
||||
`github-tags.json` was crawled from
|
||||
[api.github.com](https://api.github.com/repos/google/wuffs/tags).
|
||||
|
||||
hat.\* are various encodings of a photo of "Self-Portrait with a Straw Hat
|
||||
`hat.*` are various encodings of a photo of "Self-Portrait with a Straw Hat
|
||||
(obverse: The Potato Peeler)" by Vincent van Gogh, held by the Metropolitan
|
||||
Museum of art. `http://www.metmuseum.org/art/collection/search/436532` lists
|
||||
Museum of art.
|
||||
[www.metmuseum.org](http://www.metmuseum.org/art/collection/search/436532)
|
||||
lists that image as in the public domain.
|
||||
|
||||
`hibiscus.regular.*` are various encodings of a photo of "Hibiscus and Parrots"
|
||||
by Louis Comfort Tiffany, held by the Metropolitan Museum of Art.
|
||||
[www.metmuseum.org](http://www.metmuseum.org/art/collection/search/13503) lists
|
||||
that image as in the public domain.
|
||||
|
||||
hibiscus.regular.\* are various encodings of a photo of "Hibiscus and Parrots"
|
||||
by Louis Comfort Tiffany, held by the Metropolitan Museum of Art.
|
||||
`http://www.metmuseum.org/art/collection/search/13503` lists that image as in
|
||||
the public domain.
|
||||
|
||||
hibiscus.primitive.\* is the result of running the hibiscus.regular.png image
|
||||
through `github.com/fogleman/primitive` and other programs:
|
||||
`hibiscus.primitive.*` is the result of running the `hibiscus.regular.png`
|
||||
image through `github.com/fogleman/primitive` and other programs:
|
||||
- `primitive -i hibiscus.regular.png -o hibiscus.primitive.svg -s 442 -n 32`
|
||||
- `inkscape -z -e hibiscus.primitive.png hibiscus.primitive.svg`
|
||||
- Various invocations of `convert` and `cwebp`
|
||||
|
||||
hippopotamus.\* are various encodings of a cropping of a photo of "Hippopotamus
|
||||
(William)", held by the Metropolitan Museum of Art.
|
||||
`http://www.metmuseum.org/art/collection/search/544227` lists that image as in
|
||||
the public domain.
|
||||
`hippopotamus.*` are various encodings of a cropping of a photo of
|
||||
"Hippopotamus (William)", held by the Metropolitan Museum of Art.
|
||||
[www.metmuseum.org](http://www.metmuseum.org/art/collection/search/544227)
|
||||
lists that image as in the public domain.
|
||||
|
||||
`http://www.metmuseum.org/about-the-met/policies-and-documents/image-resources`
|
||||
[www.metmuseum.org](http://www.metmuseum.org/about-the-met/policies-and-documents/image-resources)
|
||||
says that "You are welcome to use images of artworks in The Met collection that
|
||||
the Museum believes to be in the public domain, or those to which the Museum
|
||||
waives any copyright it might have, for any purpose, including commercial and
|
||||
noncommercial use, free of charge and without requiring permission from the
|
||||
Museum."
|
||||
|
||||
json-things.\* are original JSON objects by Nigel Tao <nigeltao@golang.org>.
|
||||
`json-things.*` are original JSON objects by Nigel Tao <nigeltao@golang.org>.
|
||||
|
||||
midsummer.txt is an excerpt of Shakespeare's "A Midsummer Night's Dream",
|
||||
copied from `http://shakespeare.mit.edu/midsummer/midsummer.1.1.html`
|
||||
`midsummer.txt` is an excerpt of Shakespeare's "A Midsummer Night's Dream",
|
||||
copied from
|
||||
[shakespeare.mit.edu](http://shakespeare.mit.edu/midsummer/midsummer.1.1.html).
|
||||
|
||||
muybridge.gif is derived from
|
||||
`https://en.wikipedia.org/wiki/File:Muybridge_race_horse_animated.gif` which is
|
||||
in the public domain.
|
||||
`muybridge.gif` is derived from
|
||||
[en.wikipedia.org](https://en.wikipedia.org/wiki/File:Muybridge_race_horse_animated.gif)
|
||||
which is in the public domain.
|
||||
|
||||
nobel-prizes.json was crawled from `http://api.nobelprize.org/v1/prize.json`
|
||||
`nobel-prizes.json` was crawled from
|
||||
[api.nobelprize.org](http://api.nobelprize.org/v1/prize.json).
|
||||
|
||||
pi.txt contains the digits of pi.
|
||||
`pi.txt` contains the digits of pi.
|
||||
|
||||
pjw-thumbnail.\* are various encodings of an image derived from an iconic,
|
||||
`pjw-thumbnail.*` are various encodings of an image derived from an iconic,
|
||||
original photo of Peter J. Weinberger by Rob Pike <r@golang.org>.
|
||||
|
||||
romeo.txt is an excerpt of Shakespeare's "Romeo and Juliet", copied from
|
||||
`http://shakespeare.mit.edu/romeo_juliet/romeo_juliet.2.2.html`
|
||||
`romeo.txt` is an excerpt of Shakespeare's "Romeo and Juliet", copied from
|
||||
[shakespeare.mit.edu](http://shakespeare.mit.edu/romeo_juliet/romeo_juliet.2.2.html).
|
||||
|
||||
romeo.txt.fixed-huff.deflate was derived from romeo.txt by a custom program to
|
||||
use fixed (not dynamic) Huffman tables for the deflate encoding.
|
||||
`romeo.txt.fixed-huff.deflate` was derived from `romeo.txt` by a custom program
|
||||
to use fixed (not dynamic) Huffman tables for the deflate encoding.
|
||||
|
||||
sheep-more.rac is a RAC-compression of original text by Nigel Tao
|
||||
`sheep-more.rac` is a RAC-compression of original text by Nigel Tao
|
||||
<nigeltao@golang.org>.
|
||||
Binary file not shown.
Binary file not shown.
Reference in new issue
Block a user