mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 00:32:11 +01:00
Add std/json quirks
This commit is contained in:
1 parent
d6fdfb133f
commit
a3827275c0
5 files changed
+279
-12
No files matched your search
+17
-4
@@ -14,6 +14,13 @@ For example, in an animated GIF image, the first frame might not fill out the
|
||||
entire image, and different implementations choose a different default color
|
||||
for the outside pixels: opaque black, transparent black, or something else.
|
||||
|
||||
Other file formats have unofficial extensions. For example, a strict reading of
|
||||
the [JSON specification](https://json.org/) rejects an initial `"\x1E"` ASCII
|
||||
Record Separator, `"// comments"` or commas after the final array element or
|
||||
object key-value pair (e.g. `"[1,2,3,]"`), but
|
||||
[some](https://www.ietf.org/rfc/rfc7464.txt) [extensions](https://json5.org) to
|
||||
the format allow these and other changes.
|
||||
|
||||
Wuffs, out of the box, makes particular choices (typically mimicking the de
|
||||
facto canonical implementation), but enabling various quirks results in
|
||||
different choices. In particular, quirks are useful in regression testing that
|
||||
@@ -28,12 +35,18 @@ convention](/doc/note/base38-and-fourcc.md). Decoders and encoders can have a
|
||||
`set_quirk_enabled!(quirk base.u32, enabled base.bool)` method whose first
|
||||
argument is this `uint32_t` value.
|
||||
|
||||
For example, the base38 encoding of `"gif "` is `0xF8586`, so that the
|
||||
GIF-specific quirks have a `uint32_t` value of `((0xF8586 << 10) | n)`, for
|
||||
some small integer `n`. The generated `C` language file defines human-readable
|
||||
names for those constant values.
|
||||
For example, the base38 encoding of `"gif "` and `"json"` is `0x0F8586` and
|
||||
`0x124265` respectively, so that the GIF-specific quirks have a `uint32_t`
|
||||
value of `((0x0F8586 << 10) | g)`, and the JSON-specific quirks have a
|
||||
`uint32_t` value of `((0x124265 << 10) | j)`, for some small integers `g` and
|
||||
`j`. The high bits are a namespace. The overall quirk values are different even
|
||||
if `g` and `j` re-use the same 10-bit integer.
|
||||
|
||||
The generated `C` language file defines human-readable names for those constant
|
||||
values, such as `WUFFS_JSON__QUIRK_ALLOW_LEADING_UNICODE_BYTE_ORDER_MARK`.
|
||||
|
||||
|
||||
## Listing
|
||||
|
||||
- [GIF image decoder quirks](/std/gif/decode_quirks.wuffs)
|
||||
- [JSON decoder quirks](/std/json/decode_quirks.wuffs)
|
||||
@@ -5797,6 +5797,38 @@ extern const char* wuffs_json__error__unsupported_recursion_depth;
|
||||
|
||||
#define WUFFS_JSON__DECODER_SRC_IO_BUFFER_LENGTH_MIN_INCL 100
|
||||
|
||||
#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_A 1225364480
|
||||
|
||||
#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_CAPITAL_U 1225364481
|
||||
|
||||
#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_E 1225364482
|
||||
|
||||
#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_QUESTION_MARK 1225364483
|
||||
|
||||
#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_SINGLE_QUOTE 1225364484
|
||||
|
||||
#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_V 1225364485
|
||||
|
||||
#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X 1225364486
|
||||
|
||||
#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_ZERO 1225364487
|
||||
|
||||
#define WUFFS_JSON__QUIRK_ALLOW_COMMENT_BLOCK 1225364488
|
||||
|
||||
#define WUFFS_JSON__QUIRK_ALLOW_COMMENT_LINE 1225364489
|
||||
|
||||
#define WUFFS_JSON__QUIRK_ALLOW_FINAL_COMMA 1225364490
|
||||
|
||||
#define WUFFS_JSON__QUIRK_ALLOW_INF_NAN_NUMBERS 1225364491
|
||||
|
||||
#define WUFFS_JSON__QUIRK_ALLOW_LEADING_ASCII_RECORD_SEPARATOR 1225364492
|
||||
|
||||
#define WUFFS_JSON__QUIRK_ALLOW_LEADING_UNICODE_BYTE_ORDER_MARK 1225364493
|
||||
|
||||
#define WUFFS_JSON__QUIRK_ALLOW_TRAILING_NEW_LINE 1225364494
|
||||
|
||||
#define WUFFS_JSON__QUIRK_REPLACE_INVALID_UTF_8 1225364495
|
||||
|
||||
// ---------------- Struct Declarations
|
||||
|
||||
typedef struct wuffs_json__decoder__struct wuffs_json__decoder;
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
// Copyright 2020 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build ignore
|
||||
|
||||
package main
|
||||
|
||||
// base38-encode.go prints the base38 encoding of each argument.
|
||||
//
|
||||
// Usage: go run base38-encode.go gif json
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/google/wuffs/lib/base38"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := main1(); err != nil {
|
||||
os.Stderr.WriteString(err.Error() + "\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func main1() error {
|
||||
args := os.Args
|
||||
if len(args) > 1 {
|
||||
for _, arg := range args[1:] {
|
||||
arg := (strings.ToLower(arg) + " ")[:4]
|
||||
if code, ok := base38.Encode(arg); ok {
|
||||
code0 := fmt.Sprintf("0x%06X", code)
|
||||
code1 := fmt.Sprintf("0x%08X", code<<10)
|
||||
fmt.Printf("%s code: %s %s code<<10: %s %s\n",
|
||||
arg, code0, underscore(code0), code1, underscore(code1))
|
||||
} else {
|
||||
fmt.Printf("%s -\n", arg)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func underscore(s string) string {
|
||||
if n := len(s) - 4; n > 0 {
|
||||
return s[:n] + "_" + s[n:]
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -16,7 +16,10 @@
|
||||
|
||||
// Quirks are discussed in (/doc/note/quirks.md).
|
||||
//
|
||||
// The base38 encoding of "gif " is 0xF_8586.
|
||||
// The base38 encoding of "gif " is 0x0F_8586. Left shifting by 10 gives
|
||||
// 0x3E16_1800.
|
||||
|
||||
// --------
|
||||
|
||||
// When this quirk is enabled, when skipping over frames, the number of frames
|
||||
// visited isn't incremented when the last byte of the N'th frame is seen.
|
||||
@@ -32,7 +35,7 @@
|
||||
// see the N+1'th frame's header (or the end-of-animation terminator), so that
|
||||
// e.g. the API for visiting the N'th frame can also return whether it's the
|
||||
// final frame. Enabling this quirk allows for matching that behavior.
|
||||
pub const quirk_delay_num_decoded_frames base.u32 = (0xF_8586 << 10) | 0
|
||||
pub const quirk_delay_num_decoded_frames base.u32 = 0x3E16_1800 | 0x00
|
||||
|
||||
// When this quirk is enabled, the background color of the first frame is set
|
||||
// to black whenever that first frame has a local (frame-specific) palette.
|
||||
@@ -44,7 +47,7 @@ pub const quirk_delay_num_decoded_frames base.u32 = (0xF_8586 << 10) | 0
|
||||
//
|
||||
// There isn't really much of a rationale for this, other than it matches the
|
||||
// behavior of another GIF implementation.
|
||||
pub const quirk_first_frame_local_palette_means_black_background base.u32 = (0xF_8586 << 10) | 1
|
||||
pub const quirk_first_frame_local_palette_means_black_background base.u32 = 0x3E16_1800 | 0x01
|
||||
|
||||
// When this quirk is enabled, the background color is taken from the GIF
|
||||
// instead of always being transparent black. If the background color index in
|
||||
@@ -60,12 +63,12 @@ pub const quirk_first_frame_local_palette_means_black_background base.u32 = (0xF
|
||||
// processing WUFFS_BASE__ANIMATION_DISPOSAL__RESTORE_BACKGROUND. In both
|
||||
// cases, the caller of Wuffs, not Wuffs itself, is responsible for filling the
|
||||
// pixel buffer with that color.
|
||||
pub const quirk_honor_background_color base.u32 = (0xF_8586 << 10) | 2
|
||||
pub const quirk_honor_background_color base.u32 = 0x3E16_1800 | 0x02
|
||||
|
||||
// When this quirk is enabled, silently ignore e.g. a frame that reports a
|
||||
// width and height of 6 pixels each, followed by 50 pixel values. In that
|
||||
// case, we process the first 36 pixel values and discard the excess 14.
|
||||
pub const quirk_ignore_too_much_pixel_data base.u32 = (0xF_8586 << 10) | 3
|
||||
pub const quirk_ignore_too_much_pixel_data base.u32 = 0x3E16_1800 | 0x03
|
||||
|
||||
// When this quirk is enabled, if the initial frame bounds extends beyond the
|
||||
// image bounds, then the image bounds stay unchanged. By default (with this
|
||||
@@ -74,12 +77,12 @@ pub const quirk_ignore_too_much_pixel_data base.u32 = (0xF_8586 << 10) | 3
|
||||
//
|
||||
// For more discussion, see
|
||||
// https://github.com/google/wuffs/blob/master/test/data/artificial/gif-frame-out-of-bounds.gif.make-artificial.txt
|
||||
pub const quirk_image_bounds_are_strict base.u32 = (0xF_8586 << 10) | 4
|
||||
pub const quirk_image_bounds_are_strict base.u32 = 0x3E16_1800 | 0x04
|
||||
|
||||
// When this quirk is enabled, a frame with zero width or height is rejected
|
||||
// during decode_frame (but accepted during decode_frame_config).
|
||||
pub const quirk_reject_empty_frame base.u32 = (0xF_8586 << 10) | 5
|
||||
pub const quirk_reject_empty_frame base.u32 = 0x3E16_1800 | 0x05
|
||||
|
||||
// When this quirk is enabled, a frame with no explicit palette is rejected,
|
||||
// instead of implicitly having a palette with every entry being opaque black.
|
||||
pub const quirk_reject_empty_palette base.u32 = (0xF_8586 << 10) | 6
|
||||
pub const quirk_reject_empty_palette base.u32 = 0x3E16_1800 | 0x06
|
||||
@@ -0,0 +1,158 @@
|
||||
// Copyright 2020 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// --------
|
||||
|
||||
// Quirks are discussed in (/doc/note/quirks.md).
|
||||
//
|
||||
// The base38 encoding of "json" is 0x12_4265. Left shifting by 10 gives
|
||||
// 0x4909_9400.
|
||||
|
||||
// --------
|
||||
|
||||
// When this quirk is enabled, e.g. "abc\az" is accepted as a JSON string,
|
||||
// equivalent to "abc\u0007z", containing an ASCII Bell control character.
|
||||
pub const quirk_allow_backslash_a base.u32 = 0x4909_9400 | 0x00
|
||||
|
||||
// When this quirk is enabled, e.g. "abc\U0001F4A9z" is accepted as a JSON
|
||||
// string, equivalent to "abc\uD83D\uDCA9z", containing the U+0001F4A9 PILE OF
|
||||
// POO Unicode code point. There are exactly 8 encoded bytes after each "\U".
|
||||
pub const quirk_allow_backslash_capital_u base.u32 = 0x4909_9400 | 0x01
|
||||
|
||||
// When this quirk is enabled, e.g. "abc\ez" is accepted as a JSON string,
|
||||
// equivalent to "abc\u001Bz", containing an ASCII Escape control character.
|
||||
pub const quirk_allow_backslash_e base.u32 = 0x4909_9400 | 0x02
|
||||
|
||||
// When this quirk is enabled, e.g. "abc\?z" is accepted as a JSON string,
|
||||
// equivalent to "abc?z".
|
||||
pub const quirk_allow_backslash_question_mark base.u32 = 0x4909_9400 | 0x03
|
||||
|
||||
// When this quirk is enabled, e.g. "abc\'z" is accepted as a JSON string,
|
||||
// equivalent to "abc'z".
|
||||
pub const quirk_allow_backslash_single_quote base.u32 = 0x4909_9400 | 0x04
|
||||
|
||||
// When this quirk is enabled, e.g. "abc\vz" is accepted as a JSON string,
|
||||
// equivalent to "abc\u000Bz", containing an ASCII Vertical Tab control
|
||||
// character.
|
||||
pub const quirk_allow_backslash_v base.u32 = 0x4909_9400 | 0x05
|
||||
|
||||
// When this quirk is enabled, e.g. "abc\xeFz" is accepted as a JSON string,
|
||||
// decoding to 5 bytes: 0x61, 0x62, 0x63, 0xEF and 0x7A. There are exactly 2
|
||||
// encoded bytes after each "\x".
|
||||
//
|
||||
// Decoded strings are byte strings, no longer guaranteed to be valid UTF-8.
|
||||
//
|
||||
// "\x", "\x9", "\x9$" and "\X99" are all still rejected.
|
||||
pub const quirk_allow_backslash_x base.u32 = 0x4909_9400 | 0x06
|
||||
|
||||
// When this quirk is enabled, e.g. "abc\0z" is accepted as a JSON string,
|
||||
// equivalent to "abc\u0000z", containing an ASCII NUL control character.
|
||||
pub const quirk_allow_backslash_zero base.u32 = 0x4909_9400 | 0x07
|
||||
|
||||
// When this quirk is enabled, "/* C/C++ style block comments */" are accepted
|
||||
// anywhere whitespace would be, although see the quirk_allow_trailing_new_line
|
||||
// comment for additional interaction when combining multiple quirks.
|
||||
//
|
||||
// They produce WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK tokens.
|
||||
pub const quirk_allow_comment_block base.u32 = 0x4909_9400 | 0x08
|
||||
|
||||
// When this quirk is enabled, "// C/C++ style line comments\n" are accepted
|
||||
// anywhere whitespace would be, although see the quirk_allow_trailing_new_line
|
||||
// comment for additional interaction when combining multiple quirks.
|
||||
//
|
||||
// A line comment may omit the trailing '\n' if there is no input afterwards
|
||||
// (i.e. the line comment ends with the end-of-file), but note the
|
||||
// quirk_allow_trailing_new_line interaction already mentioned.
|
||||
//
|
||||
// They produce WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE tokens.
|
||||
pub const quirk_allow_comment_line base.u32 = 0x4909_9400 | 0x09
|
||||
|
||||
// When this quirk is enabled, there may be a comma after the final array
|
||||
// element or object key-value pair and before the closing "]" or "}". A comma
|
||||
// can therefore be a terminator (not just a separator) for elements / pairs,
|
||||
// which can simplify incremental edits and line oriented diffs.
|
||||
//
|
||||
// For example, `[1,]`, `[1,2,3,]` and `{"k":"v",}` all become acceptable, but
|
||||
// `[,]`, `{,}` and `{"k",:"v"}` are still rejected.
|
||||
pub const quirk_allow_final_comma base.u32 = 0x4909_9400 | 0x0A
|
||||
|
||||
// When this quirk is enabled, "inf", "Infinity", "NAN" and their
|
||||
// case-insensitive variants, optionally preceded immediately by "-" or "+",
|
||||
// are accepted anywhere a JSON number would be.
|
||||
pub const quirk_allow_inf_nan_numbers base.u32 = 0x4909_9400 | 0x0B
|
||||
|
||||
// When this quirk is enabled, the input byte stream may optionally start with
|
||||
// "\x1E" (the ASCII Record Separator control character). That byte is skipped
|
||||
// and decoding proceeds normally.
|
||||
//
|
||||
// When combined with quirk_allow_leading_unicode_byte_order_mark, either mark
|
||||
// may come first in the byte stream.
|
||||
//
|
||||
// When combined with quirk_allow_trailing_new_line, this format is also known
|
||||
// as RFC 7464, Json Text Sequences and MIME type "application/json-seq".
|
||||
pub const quirk_allow_leading_ascii_record_separator base.u32 = 0x4909_9400 | 0x0C
|
||||
|
||||
// When this quirk is enabled, the input byte stream may optionally start with
|
||||
// "\xEF\xBB\xBF", the UTF-8 encoding of the Unicode BOM (Byte Order Mark).
|
||||
// Those 3 bytes are skipped and decoding proceeds normally.
|
||||
//
|
||||
// When combined with quirk_allow_leading_ascii_record_separator, either mark
|
||||
// may come first in the byte stream.
|
||||
pub const quirk_allow_leading_unicode_byte_order_mark base.u32 = 0x4909_9400 | 0x0D
|
||||
|
||||
// When this quirk is enabled, following a successful decoding of a top-level
|
||||
// JSON value, any trailing whitespace (ASCII characters 0x09, 0x0A, 0x0D or
|
||||
// 0x20) is also consumed (and WUFFS_BASE__TOKEN__VBC__FILLER tokens emitted)
|
||||
// up to the end-of-file or up to and including a single new line (ASCII 0x0A,
|
||||
// also known as '\n'), whichever comes first. This trailing whitespace is not
|
||||
// mandatory, but it is consumed if present.
|
||||
//
|
||||
// When enabled, trailing non-whitespace (before a '\n') is an error. For
|
||||
// example, with "007" input, decoding with this quirk disabled (the default
|
||||
// case) will consume just 1 byte ("0") and leave the rest ("07") unread
|
||||
// without error, as "0" is a perfectly valid JSON value (but "00" is not).
|
||||
// Decoding "007" (or "007\n") with this quirk enabled will return an error.
|
||||
//
|
||||
// When enabled, the decoder will not consume more than one trailing '\n', nor
|
||||
// will it consume any other whitespace immediately after a trailing '\n'.
|
||||
//
|
||||
// If a JSON encoder avoids emitting (optional) '\n' bytes, other than a single
|
||||
// '\n' after each top-level value in a multi-JSON-value stream, this format is
|
||||
// also known as Line-delimited JSON (LDJSON), newline-delimited JSON (NDJSON)
|
||||
// and JSON lines (JSONL, http://jsonlines.org/). With or without this quirk
|
||||
// enabled, this decoder always accepts '\n' bytes before the end of a
|
||||
// top-level JSON value, anywhere whitespace is valid per the JSON spec, but
|
||||
// such bytes may confuse other line oriented Unix tools that assume exactly
|
||||
// one JSON value per line.
|
||||
//
|
||||
// When combined with quirk_allow_leading_ascii_record_separator, this format
|
||||
// is also known as RFC 7464, Json Text Sequences and MIME type
|
||||
// "application/json-seq".
|
||||
//
|
||||
// When combined with quirk_allow_comment_block or quirk_allow_comment_line, it
|
||||
// is an error for a comment to occur in this trailing whitespace, before an
|
||||
// end-of-file or '\n' is encountered. Treating this as an error avoids any
|
||||
// ambiguity in accounting for new lines within a block comment or ending a
|
||||
// line comment.
|
||||
pub const quirk_allow_trailing_new_line base.u32 = 0x4909_9400 | 0x0E
|
||||
|
||||
// When this quirk is enabled, invalid UTF-8 inside a JSON string is accepted.
|
||||
// Each byte of invalid UTF-8 is equivalent to "\uFFFD", the Unicode
|
||||
// Replacement Character. Invalid UTF-8 outside a JSON string remains an error.
|
||||
//
|
||||
// Similarly, for backslash-u escapes featuring incorrectly paired Unicode
|
||||
// surrogates, each backslash-u 6-byte unit is replaced. For example,
|
||||
// "abc\uDC00z" and "ijk\uD800\uDBFFz" are equivalent to "abc\uFFFDz" and
|
||||
// "ijk\uFFFD\uFFFDz".
|
||||
pub const quirk_replace_invalid_utf_8 base.u32 = 0x4909_9400 | 0x0F
|
||||
Reference in new issue
Block a user