Have std/cbor parse text strings

This commit is contained in:
Nigel Tao committed 2020-07-19 22:17:48 +10:00
1 parent 3c8589b6e9
commit d191a3fc5f
3 files changed
+198 -5

No files matched your search

+27 -2
View File
@@ -1192,6 +1192,30 @@ write_cbor_output_string(uint8_t* ptr, size_t len, bool finish) {
return nullptr;
}
const char* //
handle_unicode_code_point(uint32_t ucp);
const char* //
write_json_escaped_string(uint8_t* ptr, size_t len) {
restart:
while (true) {
size_t i;
for (i = 0; i < len; i++) {
uint8_t c = ptr[i];
if ((c == '"') || (c == '\\') || (c < 0x20)) {
TRY(write_dst(ptr, i));
TRY(handle_unicode_code_point(c));
ptr += i + 1;
len -= i + 1;
goto restart;
}
}
TRY(write_dst(ptr, len));
break;
}
return nullptr;
}
const char* //
handle_string(uint64_t vbd,
uint64_t len,
@@ -1222,8 +1246,7 @@ handle_string(uint64_t vbd,
if (g_flags.input_format == file_format::json) {
TRY(write_dst(ptr, len));
} else if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8) {
// TODO: escape '\n', '\"', etc.
TRY(write_dst(ptr, len));
TRY(write_json_escaped_string(ptr, len));
} else {
uint8_t as_hex[512];
uint8_t* p = ptr;
@@ -1312,6 +1335,8 @@ handle_unicode_code_point(uint32_t ucp) {
return write_cbor_output_string(&u[0], n, false);
}
// ----
const char* //
handle_token(wuffs_base__token t, bool start_of_token_chain) {
do {
+68
View File
@@ -16610,6 +16610,74 @@ wuffs_cbor__decoder__decode_tokens(
}
goto label__goto_parsed_a_leaf_value__break;
}
} else if (v_c_major == 3) {
if (v_c_minor == 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194579)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__goto_parsed_a_leaf_value__break;
} else if (v_c_minor < 24) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194579)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
} else if (v_c_minor < 28) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194579)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)((1 + (((uint32_t)(1)) << (v_c_minor & 3))))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
} else if (v_c_minor == 31) {
v_indefinite_string_major_type = 3;
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194579)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__outer__continue;
} else {
goto label__goto_fail__break;
}
label__1__continue:;
while (true) {
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
status = wuffs_base__make_status(wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(6);
goto label__1__continue;
}
v_n64 = wuffs_base__u64__min(v_string_length, ((uint64_t)(io2_a_src - iop_a_src)));
v_token_length = ((uint32_t)((v_n64 & 65535)));
if (v_n64 > 65535) {
v_token_length = 65535;
}
if (v_token_length <= 0) {
if (a_src && a_src->meta.closed) {
status = wuffs_base__make_status(wuffs_cbor__error__bad_input);
goto exit;
}
status = wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(7);
goto label__1__continue;
}
if (((uint64_t)(io2_a_src - iop_a_src)) < ((uint64_t)(v_token_length))) {
status = wuffs_base__make_status(wuffs_cbor__error__internal_error_inconsistent_token_length);
goto exit;
}
v_string_length -= ((uint64_t)(v_token_length));
v_continued = 0;
if ((v_string_length > 0) || (v_indefinite_string_major_type > 0)) {
v_continued = 1;
}
(iop_a_src += v_token_length, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194835)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(v_continued)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_token_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
if (v_string_length > 0) {
goto label__1__continue;
} else if (v_indefinite_string_major_type > 0) {
goto label__outer__continue;
}
goto label__goto_parsed_a_leaf_value__break;
}
}
goto label__goto_fail__break;
}
+103 -3
View File
@@ -156,7 +156,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
}} endwhile.goto_have_string_length
}
if c_major == 0 { // Major type 0: an unsigned integer.
if c_major == 0 {
// -------- BEGIN Major type 0: an unsigned integer.
if c_minor < 0x18 {
args.dst.write_simple_token_fast!(
value_major: 0,
@@ -176,8 +177,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
length: 1 + ((1 as base.u32) << (c_minor - 0x18)))
break.goto_parsed_a_leaf_value
}
// -------- END Major type 0: an unsigned integer.
} else if c_major == 1 { // Major type 1: a negative integer.
} else if c_major == 1 {
// -------- END Major type 1: a negative integer.
if c_minor < 0x18 {
args.dst.write_simple_token_fast!(
value_major: 0,
@@ -194,8 +197,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
length: 1 + ((1 as base.u32) << (c_minor - 0x18)))
break.goto_parsed_a_leaf_value
}
// -------- END Major type 1: a negative integer.
} else if c_major == 2 { // Major type 2: a byte string.
} else if c_major == 2 {
// -------- BEGIN Major type 2: a byte string.
if c_minor == 0x00 {
args.dst.write_simple_token_fast!(
value_major: 0,
@@ -269,6 +274,101 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
}
break.goto_parsed_a_leaf_value
} endwhile
// -------- END Major type 2: a byte string.
} else if c_major == 3 {
// -------- BEGIN Major type 3: a text string.
if c_minor == 0x00 {
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__DEFINITELY_ASCII |
base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP,
continued: 0,
length: 1)
break.goto_parsed_a_leaf_value
} else if c_minor < 0x18 {
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__DEFINITELY_ASCII |
base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP,
continued: 1,
length: 1)
} else if c_minor < 0x1C {
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__DEFINITELY_ASCII |
base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP,
continued: 1,
length: 1 + ((1 as base.u32) << (c_minor & 3)))
} else if c_minor == 0x1F {
indefinite_string_major_type = 3
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__DEFINITELY_ASCII |
base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP,
continued: 1,
length: 1)
continue.outer
} else {
break.goto_fail
}
while true {
if args.dst.available() <= 0 {
yield? base."$short write"
continue
}
n64 = string_length.min(a: args.src.available())
token_length = (n64 & 0xFFFF) as base.u32
if n64 > 0xFFFF {
token_length = 0xFFFF
}
// TODO: walk token_length back to a UTF-8 boundary.
if token_length <= 0 {
if args.src.is_closed() {
return "#bad input"
}
yield? base."$short read"
continue
}
if args.src.available() < (token_length as base.u64) {
return "#internal error: inconsistent token length"
}
string_length ~mod-= token_length as base.u64
continued = 0
if (string_length > 0) or (indefinite_string_major_type > 0) {
continued = 1
}
args.src.skip_u32_fast!(actual: token_length, worst_case: token_length)
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__DEFINITELY_ASCII |
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
continued: continued,
length: token_length)
if string_length > 0 {
continue
} else if indefinite_string_major_type > 0 {
continue.outer
}
break.goto_parsed_a_leaf_value
} endwhile
// -------- END Major type 3: a text string.
}
break.goto_fail
}} endwhile.goto_fail