diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 4b84d8a7..73cb5e25 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -19145,6 +19145,8 @@ const char* wuffs_json__error__internal_error_inconsistent_i_o = // ---------------- Private Consts +#define WUFFS_JSON__DECODER_NUMBER_LENGTH_MAX_INCL 99 + static const uint8_t // wuffs_json__lut_backslashes[256] // WUFFS_BASE__POTENTIALLY_UNUSED = { @@ -20253,7 +20255,7 @@ wuffs_json__decoder__decode_number(wuffs_json__decoder* self, if (a_src) { iop_a_src = a_src->data.ptr + a_src->meta.ri; } - if (v_n > 253) { + if (v_n > 99) { goto label__goto_done__break; } } @@ -20266,9 +20268,13 @@ wuffs_json__decoder__decode_number(wuffs_json__decoder* self, v_c = wuffs_base__load_u8be__no_bounds_check(iop_a_src); if (v_c != 46) { } else { - v_floating_point = 128; + if (v_n >= 99) { + v_n |= 512; + goto label__goto_done__break; + } v_n += 1; (iop_a_src += 1, wuffs_base__make_empty_struct()); + v_floating_point = 128; if (a_src) { a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); } @@ -20276,7 +20282,7 @@ wuffs_json__decoder__decode_number(wuffs_json__decoder* self, if (a_src) { iop_a_src = a_src->data.ptr + a_src->meta.ri; } - if (v_n > 253) { + if (v_n > 99) { goto label__goto_done__break; } if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { @@ -20290,9 +20296,13 @@ wuffs_json__decoder__decode_number(wuffs_json__decoder* self, if ((v_c != 69) && (v_c != 101)) { goto label__goto_done__break; } - v_floating_point = 128; + if (v_n >= 99) { + v_n |= 512; + goto label__goto_done__break; + } v_n += 1; (iop_a_src += 1, wuffs_base__make_empty_struct()); + v_floating_point = 128; if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { if (!(a_src && a_src->meta.closed)) { v_n |= 768; @@ -20303,6 +20313,10 @@ wuffs_json__decoder__decode_number(wuffs_json__decoder* self, v_c = wuffs_base__load_u8be__no_bounds_check(iop_a_src); if ((v_c != 43) && (v_c != 45)) { } else { + if (v_n >= 99) { + v_n |= 512; + goto label__goto_done__break; + } v_n += 1; (iop_a_src += 1, wuffs_base__make_empty_struct()); } diff --git a/std/json/common_consts.wuffs b/std/json/common_consts.wuffs index 3cf4041e..553768d0 100644 --- a/std/json/common_consts.wuffs +++ b/std/json/common_consts.wuffs @@ -42,6 +42,18 @@ pub const decoder_dst_token_buffer_length_min_incl base.u64 = 1 // as that content is placed in an io_buffer whose data.len is at least 100. pub const decoder_src_io_buffer_length_min_incl base.u64 = 100 +// decoder_number_length_max_incl is the longest supported byte length for a +// JSON number. Unlike JSON strings, this package's tokenizer never splits a +// single JSON number into multiple tokens, as this simplifies the callers. +// They can then call e.g. wuffs_base__parse_number_i64 without having to +// reconstitute a contiguous string representation. +// +// The JSON spec itself does not define a limit, but allows implementations to +// set their own limits. +pri const decoder_number_length_max_incl base.u64 = 99 + +// -------- + // Look-Up Tables (LUTs). pri const lut_backslashes array[256] base.u8 = [ diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index 5364618d..00bec3b3 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -880,13 +880,13 @@ pri func decoder.decode_number!(src: base.io_reader) base.u32[..= 0x3FF] { if c == 0x30 { // 0x30 is '0'. n += 1 args.src.skip32_fast!(actual: 1, worst_case: 1) - assert n <= 0xFD + assert n <= 99 } else { n = this.decode_digits!(src: args.src, n: n) - if n > 0xFD { + if n > 99 { break.goto_done } - assert n <= 0xFD + assert n <= 99 } // Peek. @@ -901,14 +901,18 @@ pri func decoder.decode_number!(src: base.io_reader) base.u32[..= 0x3FF] { // Scan the optional fraction. if c <> 0x2E { // 0x2E is '.'. assert args.src.available() > 0 - assert n <= 0xFD + assert n <= 99 } else { - floating_point = 0x80 + if n >= 99 { + n |= 0x200 + break.goto_done + } n += 1 args.src.skip32_fast!(actual: 1, worst_case: 1) + floating_point = 0x80 n = this.decode_digits!(src: args.src, n: n) - if n > 0xFD { + if n > 99 { break.goto_done } @@ -922,17 +926,21 @@ pri func decoder.decode_number!(src: base.io_reader) base.u32[..= 0x3FF] { c = args.src.peek_u8() assert args.src.available() > 0 - assert n <= 0xFD + assert n <= 99 } // Scan the optional 'E' or 'e'. if (c <> 0x45) and (c <> 0x65) { // 0x45 and 0x65 are 'E' and 'e'. break.goto_done } - floating_point = 0x80 + if n >= 99 { + n |= 0x200 + break.goto_done + } n += 1 args.src.skip32_fast!(actual: 1, worst_case: 1) - assert n <= 0xFE + floating_point = 0x80 + assert n <= 99 // Peek. if args.src.available() <= 0 { @@ -946,11 +954,15 @@ pri func decoder.decode_number!(src: base.io_reader) base.u32[..= 0x3FF] { // Scan the optional '+' or '-'. if (c <> 0x2B) and (c <> 0x2D) { // 0x2B and 0x2D are '+' and '-'. - assert n <= 255 + assert n <= 99 } else { + if n >= 99 { + n |= 0x200 + break.goto_done + } n += 1 args.src.skip32_fast!(actual: 1, worst_case: 1) - assert n <= 255 + assert n <= 99 } // Scan the exponent digits. @@ -961,7 +973,7 @@ pri func decoder.decode_number!(src: base.io_reader) base.u32[..= 0x3FF] { return n | floating_point } -pri func decoder.decode_digits!(src: base.io_reader, n: base.u32[..= 0xFF]) base.u32[..= 0x3FF] { +pri func decoder.decode_digits!(src: base.io_reader, n: base.u32[..= 99]) base.u32[..= 0x3FF] { var c : base.u8 var n : base.u32[..= 0x3FF] @@ -977,9 +989,9 @@ pri func decoder.decode_digits!(src: base.io_reader, n: base.u32[..= 0xFF]) base if 0x00 == lut_decimal_digits[c] { break } - // Cap the maximum supported number length at an arbitrary value, 99. - // The caller's src.data.len should therefore be at least 100, also - // known as decoder_src_io_buffer_length_min_incl. + // Cap decoder_number_length_max_incl at an arbitrary value, 99. The + // caller's src.data.len should therefore be at least 100, also known + // as decoder_src_io_buffer_length_min_incl. // // An example of a JSON number that is 81 bytes long is: // https://github.com/nst/JSONTestSuite/blob/master/test_parsing/y_number_double_close_to_zero.json diff --git a/test/c/std/json.c b/test/c/std/json.c index 451d8a47..2f60b52b 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c @@ -400,6 +400,140 @@ wuffs_json_decode(wuffs_base__token_buffer* tok, } } +const char* // +test_wuffs_json_decode_long_numbers() { + CHECK_FOCUS(__func__); + + // Spelling "false" with four letters helps clang-format align the test + // cases, when viewed in a fixed width font. + const bool fals = false; + + // Each test case produces multiple test strings: the test_cases[tc].suffix + // field is prefixed with N '9's, for multiple values of N, so that the test + // string's total length is near WUFFS_JSON__DECODER_NUMBER_LENGTH_MAX_INCL. + // For example, a ".2e4" suffix means an overall string of "999etc999.2e4". + // + // The test_cases[tc].valid field holds the whether overall string is a valid + // JSON number. + struct { + bool valid; + const char* suffix; + } test_cases[] = { + {.valid = true, .suffix = ""}, // + {.valid = true, .suffix = " "}, // + {.valid = fals, .suffix = "."}, // + {.valid = fals, .suffix = ". "}, // + {.valid = fals, .suffix = "E"}, // + {.valid = fals, .suffix = "E "}, // + {.valid = fals, .suffix = "E-"}, // + {.valid = fals, .suffix = "E- "}, // + {.valid = true, .suffix = "e2"}, // + {.valid = true, .suffix = "e2 "}, // + {.valid = true, .suffix = "e+34"}, // + {.valid = true, .suffix = "e+34 "}, // + {.valid = true, .suffix = ".2"}, // + {.valid = true, .suffix = ".2 "}, // + {.valid = fals, .suffix = ".2e"}, // + {.valid = fals, .suffix = ".2e "}, // + {.valid = fals, .suffix = ".2e+"}, // + {.valid = fals, .suffix = ".2e+ "}, // + {.valid = true, .suffix = ".2e4"}, // + {.valid = true, .suffix = ".2e4 "}, // + {.valid = true, .suffix = ".2E+5"}, // + {.valid = true, .suffix = ".2E+5 "}, // + {.valid = true, .suffix = ".2e-5678"}, // + {.valid = true, .suffix = ".2e-5678 "}, // + }; + + // src_array holds the overall test string. 119 is arbitrary but long enough. + // See the "if (suffix_length > etc)" check below. 102 is also arbitrary but + // larger than WUFFS_JSON__DECODER_NUMBER_LENGTH_MAX_INCL. + // + // See also test_wuffs_json_decode_src_io_buffer_length. + uint8_t src_array[119]; + memset(&src_array[0], '9', 102); + if (102 <= WUFFS_JSON__DECODER_NUMBER_LENGTH_MAX_INCL) { + RETURN_FAIL("insufficient number_length test case coverage"); + } + + wuffs_json__decoder dec; + + int tc; + for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) { + size_t suffix_length = strlen(test_cases[tc].suffix); + if ((suffix_length + 1) > (119 - 102)) { // +1 for the terminal NUL. + RETURN_FAIL("tc=%d: src_array is too short", tc); + } + bool ends_with_space = (suffix_length > 0) && + (test_cases[tc].suffix[suffix_length - 1] == ' '); + + // Copying the terminal NUL isn't necessary for Wuffs' slices (which are a + // pointer-length pair), but this backstop can help debugging with printf + // where "%s" takes a C string (a bare pointer). + memcpy(&src_array[102], test_cases[tc].suffix, suffix_length + 1); + + size_t nines_length; + for (nines_length = 90; nines_length < 102; nines_length++) { + wuffs_base__slice_u8 src_data = ((wuffs_base__slice_u8){ + .ptr = &src_array[102 - nines_length], + .len = nines_length + suffix_length, + }); + size_t number_length = src_data.len - (ends_with_space ? 1 : 0); + + int closed; + for (closed = 0; closed < 2; closed++) { + CHECK_STATUS( + "initialize", + wuffs_json__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED)); + + wuffs_base__token_buffer tok = ((wuffs_base__token_buffer){ + .data = global_have_token_slice, + }); + + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = src_data, + .meta = wuffs_base__make_io_buffer_meta(src_data.len, 0, 0, + closed != 0), + }); + + const char* have = + wuffs_json__decoder__decode_tokens(&dec, &tok, &src).repr; + + size_t total_length = 0; + while (tok.meta.ri < tok.meta.wi) { + wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++]; + total_length += wuffs_base__token__length(t); + } + if (total_length != src.meta.ri) { + RETURN_FAIL( + "tc=%d, nines_length=%zu, closed=%d: total_length: have %zu, " + "want %zu", + tc, nines_length, closed, total_length, src.meta.ri); + } + + const char* want; + if (number_length > WUFFS_JSON__DECODER_NUMBER_LENGTH_MAX_INCL) { + want = wuffs_json__error__unsupported_number_length; + } else if (closed || ends_with_space) { + want = test_cases[tc].valid ? NULL : wuffs_json__error__bad_input; + } else { + want = wuffs_base__suspension__short_read; + } + + if (have != want) { + RETURN_FAIL( + "tc=%d, nines_length=%zu, closed=%d: have \"%s\", want \"%s\"", + tc, nines_length, closed, have, want); + } + } + } + } + + return NULL; +} + // test_wuffs_json_decode_prior_valid_utf_8 tests that when encountering // invalid or incomplete UTF-8, or a backslash-escape, any prior valid UTF-8 is // still output. The decoder batches output so that, ignoring the quotation @@ -502,11 +636,8 @@ test_wuffs_json_decode_prior_valid_utf_8() { int closed; for (closed = 0; closed < 2; closed++) { - CHECK_STATUS( - "initialize", - wuffs_json__decoder__initialize( - &dec, sizeof dec, WUFFS_VERSION, - WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED)); + CHECK_STATUS("initialize", wuffs_json__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, 0)); wuffs_base__token_buffer tok = ((wuffs_base__token_buffer){ .data = global_have_token_slice, @@ -661,6 +792,82 @@ test_wuffs_json_decode_unicode4_escapes() { return NULL; } +// test_wuffs_json_decode_src_io_buffer_length tests that given a sufficient +// amount of source data (WUFFS_JSON__DECODER_SRC_IO_BUFFER_LENGTH_MIN_INCL or +// more), decoding will always return a conclusive result, not a suspension +// such as "$short read". +// +// The JSON specification doesn't give a maximum byte length for a number, but +// implementations are permitted to impose one. Wuffs' implementation imposes +// WUFFS_JSON__DECODER_NUMBER_LENGTH_MAX_INCL. +const char* // +test_wuffs_json_decode_src_io_buffer_length() { + CHECK_FOCUS(__func__); + + if (WUFFS_JSON__DECODER_NUMBER_LENGTH_MAX_INCL >= + WUFFS_JSON__DECODER_SRC_IO_BUFFER_LENGTH_MIN_INCL) { + RETURN_FAIL( + "inconsistent WUFFS_JSON__DECODER_NUMBER_LENGTH_MAX_INCL vs " + "WUFFS_JSON__DECODER_SRC_IO_BUFFER_LENGTH_MIN_INCL"); + } + + // src_array holds the test string of repeated '7's. 107 is arbitrary but + // long enough for the loop below. + uint8_t src_array[107]; + memset(&src_array[0], '7', 107); + + wuffs_json__decoder dec; + + int i; + for (i = WUFFS_JSON__DECODER_NUMBER_LENGTH_MAX_INCL - 2; + i <= WUFFS_JSON__DECODER_NUMBER_LENGTH_MAX_INCL + 2; i++) { + if (i < 0) { + RETURN_FAIL("invalid test case: i=%d", i); + } else if (i > 107) { + RETURN_FAIL("invalid test case: i=%d", i); + } + + wuffs_base__slice_u8 src_data = ((wuffs_base__slice_u8){ + .ptr = &src_array[0], + .len = i, + }); + + int closed; + for (closed = 0; closed < 2; closed++) { + wuffs_base__token_buffer tok = ((wuffs_base__token_buffer){ + .data = global_have_token_slice, + }); + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = src_data, + .meta = + wuffs_base__make_io_buffer_meta(src_data.len, 0, 0, closed != 0), + }); + CHECK_STATUS("initialize", + wuffs_json__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED)); + + wuffs_base__status have = + wuffs_json__decoder__decode_tokens(&dec, &tok, &src); + const char* want = + (i > WUFFS_JSON__DECODER_NUMBER_LENGTH_MAX_INCL) + ? wuffs_json__error__unsupported_number_length + : (closed ? NULL : wuffs_base__suspension__short_read); + if (have.repr != want) { + RETURN_FAIL("i=%d, closed=%d: have \"%s\", want \"%s\"", i, closed, + have.repr, want); + } + + if ((i >= WUFFS_JSON__DECODER_SRC_IO_BUFFER_LENGTH_MIN_INCL) && + wuffs_base__status__is_suspension(&have)) { + RETURN_FAIL("i=%d, closed=%d: have a suspension", i, closed); + } + } + } + + return NULL; +} + const char* // test_wuffs_json_decode_string() { CHECK_FOCUS(__func__); @@ -830,10 +1037,12 @@ proc tests[] = { test_strconv_parse_number_u64, // test_strconv_utf_8_next, // - test_wuffs_json_decode_interface, // - test_wuffs_json_decode_prior_valid_utf_8, // - test_wuffs_json_decode_string, // - test_wuffs_json_decode_unicode4_escapes, // + test_wuffs_json_decode_interface, // + test_wuffs_json_decode_long_numbers, // + test_wuffs_json_decode_prior_valid_utf_8, // + test_wuffs_json_decode_src_io_buffer_length, // + test_wuffs_json_decode_string, // + test_wuffs_json_decode_unicode4_escapes, // #ifdef WUFFS_MIMIC