From a3827275c05ec29db6e0976f38cc4e0dd81b1fba Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Thu, 12 Mar 2020 14:33:27 +1100 Subject: [PATCH] Add std/json quirks --- doc/note/quirks.md | 21 +++- release/c/wuffs-unsupported-snapshot.c | 32 +++++ script/base38-encode.go | 61 ++++++++++ std/gif/decode_quirks.wuffs | 19 +-- std/json/decode_quirks.wuffs | 158 +++++++++++++++++++++++++ 5 files changed, 279 insertions(+), 12 deletions(-) create mode 100644 script/base38-encode.go create mode 100644 std/json/decode_quirks.wuffs diff --git a/doc/note/quirks.md b/doc/note/quirks.md index b18e4865..fdeb021b 100644 --- a/doc/note/quirks.md +++ b/doc/note/quirks.md @@ -14,6 +14,13 @@ For example, in an animated GIF image, the first frame might not fill out the entire image, and different implementations choose a different default color for the outside pixels: opaque black, transparent black, or something else. +Other file formats have unofficial extensions. For example, a strict reading of +the [JSON specification](https://json.org/) rejects an initial `"\x1E"` ASCII +Record Separator, `"// comments"` or commas after the final array element or +object key-value pair (e.g. `"[1,2,3,]"`), but +[some](https://www.ietf.org/rfc/rfc7464.txt) [extensions](https://json5.org) to +the format allow these and other changes. + Wuffs, out of the box, makes particular choices (typically mimicking the de facto canonical implementation), but enabling various quirks results in different choices. In particular, quirks are useful in regression testing that @@ -28,12 +35,18 @@ convention](/doc/note/base38-and-fourcc.md). Decoders and encoders can have a `set_quirk_enabled!(quirk base.u32, enabled base.bool)` method whose first argument is this `uint32_t` value. -For example, the base38 encoding of `"gif "` is `0xF8586`, so that the -GIF-specific quirks have a `uint32_t` value of `((0xF8586 << 10) | n)`, for -some small integer `n`. The generated `C` language file defines human-readable -names for those constant values. +For example, the base38 encoding of `"gif "` and `"json"` is `0x0F8586` and +`0x124265` respectively, so that the GIF-specific quirks have a `uint32_t` +value of `((0x0F8586 << 10) | g)`, and the JSON-specific quirks have a +`uint32_t` value of `((0x124265 << 10) | j)`, for some small integers `g` and +`j`. The high bits are a namespace. The overall quirk values are different even +if `g` and `j` re-use the same 10-bit integer. + +The generated `C` language file defines human-readable names for those constant +values, such as `WUFFS_JSON__QUIRK_ALLOW_LEADING_UNICODE_BYTE_ORDER_MARK`. ## Listing - [GIF image decoder quirks](/std/gif/decode_quirks.wuffs) +- [JSON decoder quirks](/std/json/decode_quirks.wuffs) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 67bf4450..c7559ddc 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -5797,6 +5797,38 @@ extern const char* wuffs_json__error__unsupported_recursion_depth; #define WUFFS_JSON__DECODER_SRC_IO_BUFFER_LENGTH_MIN_INCL 100 +#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_A 1225364480 + +#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_CAPITAL_U 1225364481 + +#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_E 1225364482 + +#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_QUESTION_MARK 1225364483 + +#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_SINGLE_QUOTE 1225364484 + +#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_V 1225364485 + +#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X 1225364486 + +#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_ZERO 1225364487 + +#define WUFFS_JSON__QUIRK_ALLOW_COMMENT_BLOCK 1225364488 + +#define WUFFS_JSON__QUIRK_ALLOW_COMMENT_LINE 1225364489 + +#define WUFFS_JSON__QUIRK_ALLOW_FINAL_COMMA 1225364490 + +#define WUFFS_JSON__QUIRK_ALLOW_INF_NAN_NUMBERS 1225364491 + +#define WUFFS_JSON__QUIRK_ALLOW_LEADING_ASCII_RECORD_SEPARATOR 1225364492 + +#define WUFFS_JSON__QUIRK_ALLOW_LEADING_UNICODE_BYTE_ORDER_MARK 1225364493 + +#define WUFFS_JSON__QUIRK_ALLOW_TRAILING_NEW_LINE 1225364494 + +#define WUFFS_JSON__QUIRK_REPLACE_INVALID_UTF_8 1225364495 + // ---------------- Struct Declarations typedef struct wuffs_json__decoder__struct wuffs_json__decoder; diff --git a/script/base38-encode.go b/script/base38-encode.go new file mode 100644 index 00000000..f2501292 --- /dev/null +++ b/script/base38-encode.go @@ -0,0 +1,61 @@ +// Copyright 2020 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build ignore + +package main + +// base38-encode.go prints the base38 encoding of each argument. +// +// Usage: go run base38-encode.go gif json + +import ( + "fmt" + "os" + "strings" + + "github.com/google/wuffs/lib/base38" +) + +func main() { + if err := main1(); err != nil { + os.Stderr.WriteString(err.Error() + "\n") + os.Exit(1) + } +} + +func main1() error { + args := os.Args + if len(args) > 1 { + for _, arg := range args[1:] { + arg := (strings.ToLower(arg) + " ")[:4] + if code, ok := base38.Encode(arg); ok { + code0 := fmt.Sprintf("0x%06X", code) + code1 := fmt.Sprintf("0x%08X", code<<10) + fmt.Printf("%s code: %s %s code<<10: %s %s\n", + arg, code0, underscore(code0), code1, underscore(code1)) + } else { + fmt.Printf("%s -\n", arg) + } + } + } + return nil +} + +func underscore(s string) string { + if n := len(s) - 4; n > 0 { + return s[:n] + "_" + s[n:] + } + return s +} diff --git a/std/gif/decode_quirks.wuffs b/std/gif/decode_quirks.wuffs index d0c9123a..1e874187 100644 --- a/std/gif/decode_quirks.wuffs +++ b/std/gif/decode_quirks.wuffs @@ -16,7 +16,10 @@ // Quirks are discussed in (/doc/note/quirks.md). // -// The base38 encoding of "gif " is 0xF_8586. +// The base38 encoding of "gif " is 0x0F_8586. Left shifting by 10 gives +// 0x3E16_1800. + +// -------- // When this quirk is enabled, when skipping over frames, the number of frames // visited isn't incremented when the last byte of the N'th frame is seen. @@ -32,7 +35,7 @@ // see the N+1'th frame's header (or the end-of-animation terminator), so that // e.g. the API for visiting the N'th frame can also return whether it's the // final frame. Enabling this quirk allows for matching that behavior. -pub const quirk_delay_num_decoded_frames base.u32 = (0xF_8586 << 10) | 0 +pub const quirk_delay_num_decoded_frames base.u32 = 0x3E16_1800 | 0x00 // When this quirk is enabled, the background color of the first frame is set // to black whenever that first frame has a local (frame-specific) palette. @@ -44,7 +47,7 @@ pub const quirk_delay_num_decoded_frames base.u32 = (0xF_8586 << 10) | 0 // // There isn't really much of a rationale for this, other than it matches the // behavior of another GIF implementation. -pub const quirk_first_frame_local_palette_means_black_background base.u32 = (0xF_8586 << 10) | 1 +pub const quirk_first_frame_local_palette_means_black_background base.u32 = 0x3E16_1800 | 0x01 // When this quirk is enabled, the background color is taken from the GIF // instead of always being transparent black. If the background color index in @@ -60,12 +63,12 @@ pub const quirk_first_frame_local_palette_means_black_background base.u32 = (0xF // processing WUFFS_BASE__ANIMATION_DISPOSAL__RESTORE_BACKGROUND. In both // cases, the caller of Wuffs, not Wuffs itself, is responsible for filling the // pixel buffer with that color. -pub const quirk_honor_background_color base.u32 = (0xF_8586 << 10) | 2 +pub const quirk_honor_background_color base.u32 = 0x3E16_1800 | 0x02 // When this quirk is enabled, silently ignore e.g. a frame that reports a // width and height of 6 pixels each, followed by 50 pixel values. In that // case, we process the first 36 pixel values and discard the excess 14. -pub const quirk_ignore_too_much_pixel_data base.u32 = (0xF_8586 << 10) | 3 +pub const quirk_ignore_too_much_pixel_data base.u32 = 0x3E16_1800 | 0x03 // When this quirk is enabled, if the initial frame bounds extends beyond the // image bounds, then the image bounds stay unchanged. By default (with this @@ -74,12 +77,12 @@ pub const quirk_ignore_too_much_pixel_data base.u32 = (0xF_8586 << 10) | 3 // // For more discussion, see // https://github.com/google/wuffs/blob/master/test/data/artificial/gif-frame-out-of-bounds.gif.make-artificial.txt -pub const quirk_image_bounds_are_strict base.u32 = (0xF_8586 << 10) | 4 +pub const quirk_image_bounds_are_strict base.u32 = 0x3E16_1800 | 0x04 // When this quirk is enabled, a frame with zero width or height is rejected // during decode_frame (but accepted during decode_frame_config). -pub const quirk_reject_empty_frame base.u32 = (0xF_8586 << 10) | 5 +pub const quirk_reject_empty_frame base.u32 = 0x3E16_1800 | 0x05 // When this quirk is enabled, a frame with no explicit palette is rejected, // instead of implicitly having a palette with every entry being opaque black. -pub const quirk_reject_empty_palette base.u32 = (0xF_8586 << 10) | 6 +pub const quirk_reject_empty_palette base.u32 = 0x3E16_1800 | 0x06 diff --git a/std/json/decode_quirks.wuffs b/std/json/decode_quirks.wuffs new file mode 100644 index 00000000..c354505b --- /dev/null +++ b/std/json/decode_quirks.wuffs @@ -0,0 +1,158 @@ +// Copyright 2020 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// -------- + +// Quirks are discussed in (/doc/note/quirks.md). +// +// The base38 encoding of "json" is 0x12_4265. Left shifting by 10 gives +// 0x4909_9400. + +// -------- + +// When this quirk is enabled, e.g. "abc\az" is accepted as a JSON string, +// equivalent to "abc\u0007z", containing an ASCII Bell control character. +pub const quirk_allow_backslash_a base.u32 = 0x4909_9400 | 0x00 + +// When this quirk is enabled, e.g. "abc\U0001F4A9z" is accepted as a JSON +// string, equivalent to "abc\uD83D\uDCA9z", containing the U+0001F4A9 PILE OF +// POO Unicode code point. There are exactly 8 encoded bytes after each "\U". +pub const quirk_allow_backslash_capital_u base.u32 = 0x4909_9400 | 0x01 + +// When this quirk is enabled, e.g. "abc\ez" is accepted as a JSON string, +// equivalent to "abc\u001Bz", containing an ASCII Escape control character. +pub const quirk_allow_backslash_e base.u32 = 0x4909_9400 | 0x02 + +// When this quirk is enabled, e.g. "abc\?z" is accepted as a JSON string, +// equivalent to "abc?z". +pub const quirk_allow_backslash_question_mark base.u32 = 0x4909_9400 | 0x03 + +// When this quirk is enabled, e.g. "abc\'z" is accepted as a JSON string, +// equivalent to "abc'z". +pub const quirk_allow_backslash_single_quote base.u32 = 0x4909_9400 | 0x04 + +// When this quirk is enabled, e.g. "abc\vz" is accepted as a JSON string, +// equivalent to "abc\u000Bz", containing an ASCII Vertical Tab control +// character. +pub const quirk_allow_backslash_v base.u32 = 0x4909_9400 | 0x05 + +// When this quirk is enabled, e.g. "abc\xeFz" is accepted as a JSON string, +// decoding to 5 bytes: 0x61, 0x62, 0x63, 0xEF and 0x7A. There are exactly 2 +// encoded bytes after each "\x". +// +// Decoded strings are byte strings, no longer guaranteed to be valid UTF-8. +// +// "\x", "\x9", "\x9$" and "\X99" are all still rejected. +pub const quirk_allow_backslash_x base.u32 = 0x4909_9400 | 0x06 + +// When this quirk is enabled, e.g. "abc\0z" is accepted as a JSON string, +// equivalent to "abc\u0000z", containing an ASCII NUL control character. +pub const quirk_allow_backslash_zero base.u32 = 0x4909_9400 | 0x07 + +// When this quirk is enabled, "/* C/C++ style block comments */" are accepted +// anywhere whitespace would be, although see the quirk_allow_trailing_new_line +// comment for additional interaction when combining multiple quirks. +// +// They produce WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK tokens. +pub const quirk_allow_comment_block base.u32 = 0x4909_9400 | 0x08 + +// When this quirk is enabled, "// C/C++ style line comments\n" are accepted +// anywhere whitespace would be, although see the quirk_allow_trailing_new_line +// comment for additional interaction when combining multiple quirks. +// +// A line comment may omit the trailing '\n' if there is no input afterwards +// (i.e. the line comment ends with the end-of-file), but note the +// quirk_allow_trailing_new_line interaction already mentioned. +// +// They produce WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE tokens. +pub const quirk_allow_comment_line base.u32 = 0x4909_9400 | 0x09 + +// When this quirk is enabled, there may be a comma after the final array +// element or object key-value pair and before the closing "]" or "}". A comma +// can therefore be a terminator (not just a separator) for elements / pairs, +// which can simplify incremental edits and line oriented diffs. +// +// For example, `[1,]`, `[1,2,3,]` and `{"k":"v",}` all become acceptable, but +// `[,]`, `{,}` and `{"k",:"v"}` are still rejected. +pub const quirk_allow_final_comma base.u32 = 0x4909_9400 | 0x0A + +// When this quirk is enabled, "inf", "Infinity", "NAN" and their +// case-insensitive variants, optionally preceded immediately by "-" or "+", +// are accepted anywhere a JSON number would be. +pub const quirk_allow_inf_nan_numbers base.u32 = 0x4909_9400 | 0x0B + +// When this quirk is enabled, the input byte stream may optionally start with +// "\x1E" (the ASCII Record Separator control character). That byte is skipped +// and decoding proceeds normally. +// +// When combined with quirk_allow_leading_unicode_byte_order_mark, either mark +// may come first in the byte stream. +// +// When combined with quirk_allow_trailing_new_line, this format is also known +// as RFC 7464, Json Text Sequences and MIME type "application/json-seq". +pub const quirk_allow_leading_ascii_record_separator base.u32 = 0x4909_9400 | 0x0C + +// When this quirk is enabled, the input byte stream may optionally start with +// "\xEF\xBB\xBF", the UTF-8 encoding of the Unicode BOM (Byte Order Mark). +// Those 3 bytes are skipped and decoding proceeds normally. +// +// When combined with quirk_allow_leading_ascii_record_separator, either mark +// may come first in the byte stream. +pub const quirk_allow_leading_unicode_byte_order_mark base.u32 = 0x4909_9400 | 0x0D + +// When this quirk is enabled, following a successful decoding of a top-level +// JSON value, any trailing whitespace (ASCII characters 0x09, 0x0A, 0x0D or +// 0x20) is also consumed (and WUFFS_BASE__TOKEN__VBC__FILLER tokens emitted) +// up to the end-of-file or up to and including a single new line (ASCII 0x0A, +// also known as '\n'), whichever comes first. This trailing whitespace is not +// mandatory, but it is consumed if present. +// +// When enabled, trailing non-whitespace (before a '\n') is an error. For +// example, with "007" input, decoding with this quirk disabled (the default +// case) will consume just 1 byte ("0") and leave the rest ("07") unread +// without error, as "0" is a perfectly valid JSON value (but "00" is not). +// Decoding "007" (or "007\n") with this quirk enabled will return an error. +// +// When enabled, the decoder will not consume more than one trailing '\n', nor +// will it consume any other whitespace immediately after a trailing '\n'. +// +// If a JSON encoder avoids emitting (optional) '\n' bytes, other than a single +// '\n' after each top-level value in a multi-JSON-value stream, this format is +// also known as Line-delimited JSON (LDJSON), newline-delimited JSON (NDJSON) +// and JSON lines (JSONL, http://jsonlines.org/). With or without this quirk +// enabled, this decoder always accepts '\n' bytes before the end of a +// top-level JSON value, anywhere whitespace is valid per the JSON spec, but +// such bytes may confuse other line oriented Unix tools that assume exactly +// one JSON value per line. +// +// When combined with quirk_allow_leading_ascii_record_separator, this format +// is also known as RFC 7464, Json Text Sequences and MIME type +// "application/json-seq". +// +// When combined with quirk_allow_comment_block or quirk_allow_comment_line, it +// is an error for a comment to occur in this trailing whitespace, before an +// end-of-file or '\n' is encountered. Treating this as an error avoids any +// ambiguity in accounting for new lines within a block comment or ending a +// line comment. +pub const quirk_allow_trailing_new_line base.u32 = 0x4909_9400 | 0x0E + +// When this quirk is enabled, invalid UTF-8 inside a JSON string is accepted. +// Each byte of invalid UTF-8 is equivalent to "\uFFFD", the Unicode +// Replacement Character. Invalid UTF-8 outside a JSON string remains an error. +// +// Similarly, for backslash-u escapes featuring incorrectly paired Unicode +// surrogates, each backslash-u 6-byte unit is replaced. For example, +// "abc\uDC00z" and "ijk\uD800\uDBFFz" are equivalent to "abc\uFFFDz" and +// "ijk\uFFFD\uFFFDz". +pub const quirk_replace_invalid_utf_8 base.u32 = 0x4909_9400 | 0x0F