diff --git a/example/jsonptr/jsonptr.cc b/example/jsonptr/jsonptr.cc index 12b66d33..a8209615 100644 --- a/example/jsonptr/jsonptr.cc +++ b/example/jsonptr/jsonptr.cc @@ -670,8 +670,16 @@ initialize_globals(int argc, char** argv) { suppress_write_dst = query.next_fragment() ? 1 : 0; wrote_to_dst = false; - return dec.initialize(sizeof__wuffs_json__decoder(), WUFFS_VERSION, 0) - .message(); + TRY(dec.initialize(sizeof__wuffs_json__decoder(), WUFFS_VERSION, 0) + .message()); + + // Consume an optional whitespace trailer. This isn't part of the JSON spec, + // but it works better with line oriented Unix tools (such as "echo 123 | + // jsonptr" where it's "echo", not "echo -n") or hand-edited JSON files which + // can accidentally contain trailing whitespace. + dec.set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_TRAILING_NEW_LINE, true); + + return nullptr; } // ---- @@ -1051,30 +1059,6 @@ end_of_data: return nullptr; } - // Consume an optional whitespace trailer. This isn't part of the JSON spec, - // but it works better with line oriented Unix tools (such as "echo 123 | - // jsonptr" where it's "echo", not "echo -n") or hand-edited JSON files which - // can accidentally contain trailing whitespace. - // - // A whitespace trailer is zero or more ' ' and then zero or one '\n'. - while (true) { - if (src.meta.ri < src.meta.wi) { - uint8_t c = src.data.ptr[src.meta.ri]; - if (c == ' ') { - src.meta.ri++; - continue; - } else if (c == '\n') { - src.meta.ri++; - break; - } - // The "exhausted the input" check below will fail. - break; - } else if (src.meta.closed) { - break; - } - TRY(read_src()); - } - // Check that we've exhausted the input. if ((src.meta.ri == src.meta.wi) && !src.meta.closed) { TRY(read_src()); @@ -1084,9 +1068,9 @@ end_of_data: } // Check that we've used all of the decoded tokens, other than trailing - // filler tokens. For example, a bare `"foo"` string is valid JSON, but even - // without a trailing '\n', the Wuffs JSON parser emits a filler token for - // the final '\"'. + // filler tokens. For example, "true\n" is valid JSON (and fully consumed + // with WUFFS_JSON__QUIRK_ALLOW_TRAILING_NEW_LINE enabled) with a trailing + // filler token for the "\n". for (; tok.meta.ri < tok.meta.wi; tok.meta.ri++) { if (tok.data.ptr[tok.meta.ri].value_base_category() != WUFFS_BASE__TOKEN__VBC__FILLER) { diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index aa6a9a2a..3bc35c96 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -5999,6 +5999,7 @@ struct wuffs_json__decoder__struct { uint32_t p_decode_tokens[1]; uint32_t p_decode_leading[1]; + uint32_t p_decode_trailing_new_line[1]; } private_impl; struct { @@ -20296,6 +20297,11 @@ wuffs_json__decoder__decode_leading(wuffs_json__decoder* self, wuffs_base__token_buffer* a_dst, wuffs_base__io_buffer* a_src); +static wuffs_base__status // +wuffs_json__decoder__decode_trailing_new_line(wuffs_json__decoder* self, + wuffs_base__token_buffer* a_dst, + wuffs_base__io_buffer* a_src); + // ---------------- VTables const wuffs_base__token_decoder__func_ptrs @@ -21278,6 +21284,26 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, v_expect = v_expect_after_value; } label__outer__break:; + if (self->private_impl.f_quirk_enabled_allow_trailing_new_line) { + if (a_dst) { + a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr)); + } + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(19); + status = + wuffs_json__decoder__decode_trailing_new_line(self, a_dst, a_src); + if (a_dst) { + iop_a_dst = a_dst->data.ptr + a_dst->meta.wi; + } + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + } self->private_impl.f_end_of_data = true; goto ok; @@ -21611,6 +21637,124 @@ exit: return status; } +// -------- func json.decoder.decode_trailing_new_line + +static wuffs_base__status // +wuffs_json__decoder__decode_trailing_new_line(wuffs_json__decoder* self, + wuffs_base__token_buffer* a_dst, + wuffs_base__io_buffer* a_src) { + wuffs_base__status status = wuffs_base__make_status(NULL); + + uint8_t v_c = 0; + uint32_t v_whitespace_length = 0; + + wuffs_base__token* iop_a_dst = NULL; + wuffs_base__token* io0_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + wuffs_base__token* io1_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + wuffs_base__token* io2_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_dst) { + io0_a_dst = a_dst->data.ptr; + io1_a_dst = io0_a_dst + a_dst->meta.wi; + iop_a_dst = io1_a_dst; + io2_a_dst = io0_a_dst + a_dst->data.len; + if (a_dst->meta.closed) { + io2_a_dst = iop_a_dst; + } + } + uint8_t* iop_a_src = NULL; + uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } + + uint32_t coro_susp_point = self->private_impl.p_decode_trailing_new_line[0]; + if (coro_susp_point) { + } + switch (coro_susp_point) { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; + + label__outer__continue:; + while (true) { + if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { + status = wuffs_base__make_status(wuffs_base__suspension__short_write); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(1); + v_whitespace_length = 0; + goto label__outer__continue; + } + while (true) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { + if (v_whitespace_length > 0) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(v_whitespace_length)) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_whitespace_length = 0; + } + if (a_src && a_src->meta.closed) { + goto label__outer__break; + } + status = wuffs_base__make_status(wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(2); + v_whitespace_length = 0; + goto label__outer__continue; + } + v_c = wuffs_base__load_u8be__no_bounds_check(iop_a_src); + if (wuffs_json__lut_classes[v_c] != 0) { + if (v_whitespace_length > 0) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(v_whitespace_length)) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_whitespace_length = 0; + } + status = wuffs_base__make_status(wuffs_json__error__bad_input); + goto exit; + } + (iop_a_src += 1, wuffs_base__make_empty_struct()); + if ((v_whitespace_length >= 65534) || (v_c == 10)) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((v_whitespace_length + 1))) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_whitespace_length = 0; + if (v_c == 10) { + goto label__outer__break; + } + goto label__outer__continue; + } + v_whitespace_length += 1; + } + } + label__outer__break:; + + goto ok; + ok: + self->private_impl.p_decode_trailing_new_line[0] = 0; + goto exit; + } + + goto suspend; +suspend: + self->private_impl.p_decode_trailing_new_line[0] = + wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; + + goto exit; +exit: + if (a_dst) { + a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr)); + } + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + + return status; +} + #endif // !defined(WUFFS_CONFIG__MODULES) || // defined(WUFFS_CONFIG__MODULE__JSON) diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index 44d16785..b8152941 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -902,6 +902,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { expect = expect_after_value } endwhile.outer + if this.quirk_enabled_allow_trailing_new_line { + this.decode_trailing_new_line?(dst: args.dst, src: args.src) + } + this.end_of_data = true } @@ -1124,3 +1128,57 @@ pri func decoder.decode_leading?(dst: base.token_writer, src: base.io_reader) { break } } + +pri func decoder.decode_trailing_new_line?(dst: base.token_writer, src: base.io_reader) { + var c : base.u8 + var whitespace_length : base.u32[..= 0xFFFE] + + while.outer true { + if args.dst.available() <= 0 { + yield? base."$short write" + whitespace_length = 0 + continue.outer + } + + while.inner true, + pre args.dst.available() > 0, + { + if args.src.available() <= 0 { + if whitespace_length > 0 { + args.dst.write_fast_token!( + value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length) + whitespace_length = 0 + } + if args.src.is_closed() { + break.outer + } + yield? base."$short read" + whitespace_length = 0 + continue.outer + } + + c = args.src.peek_u8() + if lut_classes[c] <> 0x00 { // 0x00 is CLASS_WHITESPACE. + if whitespace_length > 0 { + args.dst.write_fast_token!( + value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length) + whitespace_length = 0 + } + return "#bad input" + } + + args.src.skip32_fast!(actual: 1, worst_case: 1) + if (whitespace_length >= 0xFFFE) or (c == 0x0A) { + args.dst.write_fast_token!( + value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length + 1) + whitespace_length = 0 + if c == 0x0A { + break.outer + } + continue.outer + } + whitespace_length += 1 + } endwhile.inner + } endwhile.outer + +} diff --git a/test/c/std/json.c b/test/c/std/json.c index 2c1d7127..b9fac03f 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c @@ -1113,21 +1113,22 @@ test_wuffs_json_decode_quirk_allow_leading_etc() { // want has 4 bytes, one for each possible q: // - q&1 sets WUFFS_JSON__QUIRK_ALLOW_LEADING_ASCII_RECORD_SEPARATOR. // - q&2 sets WUFFS_JSON__QUIRK_ALLOW_LEADING_UNICODE_BYTE_ORDER_MARK. - // A '+' or '-' means that decoding should succeed or fail. + // An 'X', '+' or '-' means that decoding should succeed (and consume the + // entire input), succeed (without consuming the entire input) or fail. const char* want; const char* str; } test_cases[] = { - {.want = "-+-+", .str = "\x1Etrue"}, - {.want = "--++", .str = "\xEF\xBB\xBFtrue"}, - {.want = "---+", .str = "\x1E\xEF\xBB\xBFtrue"}, - {.want = "---+", .str = "\xEF\xBB\xBF\x1Etrue"}, - {.want = "----", .str = " \x1Etrue"}, - {.want = "----", .str = "\x1E \xEF\xBB\xBFtrue"}, - {.want = "----", .str = "\x1E\x1Etrue"}, - {.want = "----", .str = "\xEF\xBB"}, - {.want = "----", .str = "\xEF\xBB\xBF"}, - {.want = "----", .str = "\xEF\xBB\xBF$"}, - {.want = "----", .str = "\xEFtrue"}, + {.want = "-X-X", .str = "\x1Etrue"}, // + {.want = "--XX", .str = "\xEF\xBB\xBFtrue"}, // + {.want = "---X", .str = "\x1E\xEF\xBB\xBFtrue"}, // + {.want = "---X", .str = "\xEF\xBB\xBF\x1Etrue"}, // + {.want = "----", .str = " \x1Etrue"}, // + {.want = "----", .str = "\x1E \xEF\xBB\xBFtrue"}, // + {.want = "----", .str = "\x1E\x1Etrue"}, // + {.want = "----", .str = "\xEF\xBB"}, // + {.want = "----", .str = "\xEF\xBB\xBF"}, // + {.want = "----", .str = "\xEF\xBB\xBF$"}, // + {.want = "----", .str = "\xEFtrue"}, // }; int tc; @@ -1152,7 +1153,7 @@ test_wuffs_json_decode_quirk_allow_leading_etc() { const char* have = wuffs_json__decoder__decode_tokens(&dec, &tok, &src).repr; const char* want = - (test_cases[tc].want[q] == '+') ? NULL : wuffs_json__error__bad_input; + (test_cases[tc].want[q] != '-') ? NULL : wuffs_json__error__bad_input; if (have != want) { RETURN_FAIL("tc=%d, q=%d: decode_tokens: have \"%s\", want \"%s\"", tc, q, have, want); @@ -1166,6 +1167,95 @@ test_wuffs_json_decode_quirk_allow_leading_etc() { RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want %zu", tc, q, total_length, src.meta.ri); } + if (test_cases[tc].want[q] == 'X') { + if (total_length != src.data.len) { + RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want %zu", tc, q, + total_length, src.data.len); + } + } else if (test_cases[tc].want[q] == '+') { + if (total_length >= src.data.len) { + RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want < %zu", tc, q, + total_length, src.data.len); + } + } + } + } + return NULL; +} + +const char* // +test_wuffs_json_decode_quirk_allow_trailing_etc() { + CHECK_FOCUS(__func__); + + struct { + // want has 2 bytes, one for each possible q: + // - q&1 sets WUFFS_JSON__QUIRK_ALLOW_TRAILING_NEW_LINE. + // An 'X', '+' or '-' means that decoding should succeed (and consume the + // entire input), succeed (without consuming the entire input) or fail. + const char* want; + const char* str; + } test_cases[] = { + {.want = "++", .str = "0 \n "}, // + {.want = "++", .str = "0 \n\n"}, // + {.want = "++", .str = "0\n\n"}, // + {.want = "+-", .str = "0 true \n"}, // + {.want = "+-", .str = "007"}, // + {.want = "+-", .str = "007\n"}, // + {.want = "+-", .str = "0true "}, // + {.want = "+-", .str = "0true"}, // + {.want = "+X", .str = "0 "}, // + {.want = "+X", .str = "0 \n"}, // + {.want = "+X", .str = "0\n"}, // + {.want = "+X", .str = "0\t\r\n"}, // + {.want = "--", .str = "\n"}, // + {.want = "XX", .str = "0"}, // + }; + + int tc; + for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) { + int q; + for (q = 0; q < 2; q++) { + wuffs_json__decoder dec; + CHECK_STATUS("initialize", wuffs_json__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__DEFAULT_OPTIONS)); + wuffs_json__decoder__set_quirk_enabled( + &dec, WUFFS_JSON__QUIRK_ALLOW_TRAILING_NEW_LINE, q & 1); + + wuffs_base__token_buffer tok = + wuffs_base__make_token_buffer_writer(global_have_token_slice); + wuffs_base__io_buffer src = wuffs_base__make_io_buffer_reader( + wuffs_base__make_slice_u8((void*)(test_cases[tc].str), + strlen(test_cases[tc].str)), + true); + const char* have = + wuffs_json__decoder__decode_tokens(&dec, &tok, &src).repr; + const char* want = + (test_cases[tc].want[q] != '-') ? NULL : wuffs_json__error__bad_input; + if (have != want) { + RETURN_FAIL("tc=%d, q=%d: decode_tokens: have \"%s\", want \"%s\"", tc, + q, have, want); + } + + size_t total_length = 0; + while (tok.meta.ri < tok.meta.wi) { + total_length += wuffs_base__token__length(&tok.data.ptr[tok.meta.ri++]); + } + if (total_length != src.meta.ri) { + RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want %zu", tc, q, + total_length, src.meta.ri); + } + if (test_cases[tc].want[q] == 'X') { + if (total_length != src.data.len) { + RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want %zu", tc, q, + total_length, src.data.len); + } + } else if (test_cases[tc].want[q] == '+') { + if (total_length >= src.data.len) { + RETURN_FAIL("tc=%d, q=%d: total_length: have %zu, want < %zu", tc, q, + total_length, src.data.len); + } + } } } return NULL; @@ -1519,14 +1609,15 @@ proc tests[] = { test_strconv_parse_number_u64, // test_strconv_utf_8_next, // - test_wuffs_json_decode_end_of_data, // - test_wuffs_json_decode_interface, // - test_wuffs_json_decode_long_numbers, // - test_wuffs_json_decode_prior_valid_utf_8, // - test_wuffs_json_decode_quirk_allow_leading_etc, // - test_wuffs_json_decode_src_io_buffer_length, // - test_wuffs_json_decode_string, // - test_wuffs_json_decode_unicode4_escapes, // + test_wuffs_json_decode_end_of_data, // + test_wuffs_json_decode_interface, // + test_wuffs_json_decode_long_numbers, // + test_wuffs_json_decode_prior_valid_utf_8, // + test_wuffs_json_decode_quirk_allow_leading_etc, // + test_wuffs_json_decode_quirk_allow_trailing_etc, // + test_wuffs_json_decode_src_io_buffer_length, // + test_wuffs_json_decode_string, // + test_wuffs_json_decode_unicode4_escapes, // #ifdef WUFFS_MIMIC