Split ETC__VBC__INLINE_INTEGER into {,un}signed

This commit is contained in:
Nigel Tao committed 2020-07-21 15:31:49 +10:00
1 parent 51a38295b6
commit c9d4e34dfc
7 files changed
+93 -45

No files matched your search

+22 -7
View File
@@ -1107,15 +1107,23 @@ fail:
}
const char* //
write_inline_integer(uint64_t vbd, uint8_t* ptr, size_t len) {
write_inline_integer(uint64_t x, bool x_is_signed, uint8_t* ptr, size_t len) {
if (g_flags.output_format == file_format::cbor) {
return write_dst(ptr, len);
}
uint8_t buf[WUFFS_BASE__I64__BYTE_LENGTH__MAX_INCL];
size_t n = wuffs_base__render_number_i64(
wuffs_base__make_slice_u8(&buf[0], sizeof buf), (int16_t)vbd,
WUFFS_BASE__RENDER_NUMBER_XXX__DEFAULT_OPTIONS);
// Adding the two ETC__BYTE_LENGTH__ETC constants is overkill, but it's
// simpler (for producing a constant-expression array size) than taking the
// maximum of the two.
uint8_t buf[WUFFS_BASE__I64__BYTE_LENGTH__MAX_INCL +
WUFFS_BASE__U64__BYTE_LENGTH__MAX_INCL];
wuffs_base__slice_u8 dst = wuffs_base__make_slice_u8(&buf[0], sizeof buf);
size_t n =
x_is_signed
? wuffs_base__render_number_i64(
dst, (int64_t)x, WUFFS_BASE__RENDER_NUMBER_XXX__DEFAULT_OPTIONS)
: wuffs_base__render_number_u64(
dst, x, WUFFS_BASE__RENDER_NUMBER_XXX__DEFAULT_OPTIONS);
return write_dst(&buf[0], n);
}
@@ -1522,10 +1530,17 @@ handle_token(wuffs_base__token t, bool start_of_token_chain) {
len));
goto after_value;
case WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER:
case WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED:
case WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED: {
bool x_is_signed = vbc == WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED;
uint64_t x = x_is_signed
? ((uint64_t)(t.value_base_detail__sign_extended()))
: vbd;
TRY(write_inline_integer(
vbd, g_src.data.ptr + g_curr_token_end_src_index - len, len));
x, x_is_signed, g_src.data.ptr + g_curr_token_end_src_index - len,
len));
goto after_value;
}
}
if (t.value_major() == WUFFS_CBOR__TOKEN_VALUE_MAJOR) {
+17 -1
View File
@@ -29,6 +29,7 @@ typedef struct {
inline int64_t value_base_category() const;
inline uint64_t value_minor() const;
inline uint64_t value_base_detail() const;
inline int64_t value_base_detail__sign_extended() const;
inline bool continued() const;
inline uint64_t length() const;
#endif // __cplusplus
@@ -63,7 +64,8 @@ wuffs_base__make_token(uint64_t repr) {
#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3
#define WUFFS_BASE__TOKEN__VBC__LITERAL 4
#define WUFFS_BASE__TOKEN__VBC__NUMBER 5
#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER 6
#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED 6
#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED 7
// --------
@@ -205,6 +207,15 @@ wuffs_base__token__value_base_detail(const wuffs_base__token* t) {
return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) & 0x1FFFFF;
}
static inline int64_t //
wuffs_base__token__value_base_detail__sign_extended(
const wuffs_base__token* t) {
// The VBD is 21 bits in the middle of t->repr. Left shift the high (64 - 21
// - ETC__SHIFT) bits off, then right shift (sign-extending) back down.
uint64_t u = t->repr << (43 - WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT);
return ((int64_t)u) >> 43;
}
static inline bool //
wuffs_base__token__continued(const wuffs_base__token* t) {
return t->repr & 0x10000;
@@ -247,6 +258,11 @@ wuffs_base__token::value_base_detail() const {
return wuffs_base__token__value_base_detail(this);
}
inline int64_t //
wuffs_base__token::value_base_detail__sign_extended() const {
return wuffs_base__token__value_base_detail__sign_extended(this);
}
inline bool //
wuffs_base__token::continued() const {
return wuffs_base__token__continued(this);
+13 -12
View File
@@ -68,14 +68,14 @@ const BaseFloatConvSubmoduleCodeC = "" +
" upper_delta = +1;\n } else if (hd != ud) {\n // For example:\n // h = 12345???\n // upper = 12346???\n upper_delta = +0;\n }\n } else if (upper_delta == 0) {\n if ((hd != 9) || (ud != 0)) {\n // For example:\n // h = 1234598?\n // upper = 1234600?\n upper_delta = +1;\n }\n }\n\n // We can round up if upper has a different digit than h and either upper\n // is inclusive or upper is bigger than the result of rounding up.\n bool can_round_up =\n (upper_delta > 0) || //\n ((upper_delta == 0) && //\n (inclusive || ((ui + 1) < ((int32_t)(upper.num_digits)))));\n\n // If we can round either way, round to nearest. If we can round only one\n // way, do it. If we can't round, continue the loop.\n if (can_round_down) {\n if (can_round_up) {\n wuffs_base__private_implementation__high_prec_dec__round_nearest(\n h, hi + 1);\n return;\n } else {\n wuffs_base__private_implementat" +
"ion__high_prec_dec__round_down(h,\n hi + 1);\n return;\n }\n } else {\n if (can_round_up) {\n wuffs_base__private_implementation__high_prec_dec__round_up(h, hi + 1);\n return;\n }\n }\n }\n}\n\n" +
"" +
"// --------\n\n// wuffs_base__private_implementation__parse_number_f64_eisel_lemire produces\n// the IEEE 754 double-precision value for an exact mantissa and base-10\n// exponent. For example:\n// - when parsing \"12345.678e+02\", man is 12345678 and exp10 is -1.\n// - when parsing \"-12\", man is 12 and exp10 is 0. Processing the leading\n// minus sign is the responsibility of the caller, not this function.\n//\n// On success, it returns a non-negative int64_t such that the low 63 bits hold\n// the 11-bit exponent and 52-bit mantissa.\n//\n// On failure, it returns a negative value.\n//\n// The algorithm is based on an original idea by Michael Eisel and refined by\n// Daniel Lemire. See\n// https://lemire.me/blog/2020/03/10/fast-float-parsing-in-practice/\n//\n// Preconditions:\n// - man is non-zero.\n// - exp10 is in the range -326 ..= 310, the same range of the\n// wuffs_base__private_implementation__powers_of_10 array.\nstatic int64_t //\nwuffs_base__private_implementation__parse_number_f64_eisel_lemire(\n uint64_t ma" +
"n,\n int32_t exp10) {\n // Look up the (possibly truncated) base-2 representation of (10 ** exp10).\n // The look-up table was constructed so that it is already normalized: the\n // table entry's mantissa's MSB (most significant bit) is on.\n const uint32_t* po10 =\n &wuffs_base__private_implementation__powers_of_10[5 * (exp10 + 326)];\n\n // Normalize the man argument. The (man != 0) precondition means that a\n // non-zero bit exists.\n uint32_t clz = wuffs_base__count_leading_zeroes_u64(man);\n man <<= clz;\n\n // Calculate the return value's base-2 exponent. We might tweak it by ±1\n // later, but its initial value comes from the look-up table and clz.\n uint64_t ret_exp2 = ((uint64_t)po10[4]) - ((uint64_t)clz);\n\n // Multiply the two mantissas. Normalization means that both mantissas are at\n // least (1<<63), so the 128-bit product must be at least (1<<126). The high\n // 64 bits of the product, x_hi, must therefore be at least (1<<62).\n //\n // As a consequence, x_hi has either 0 or 1 leading zero" +
"es. Shifting x_hi\n // right by either 9 or 10 bits (depending on x_hi's MSB) will therefore\n // leave the top 10 MSBs (bits 54 ..= 63) off and the 11th MSB (bit 53) on.\n#if defined(__SIZEOF_INT128__)\n // See commit 18449ad75d582dd015c236abc85a16f333b796f3 \"Optimize 128-bit muls\n // in parse_number_f64_eisel\" for benchmark numbers.\n __uint128_t x =\n ((__uint128_t)man) * (((uint64_t)po10[2]) | (((uint64_t)po10[3]) << 32));\n uint64_t x_hi = ((uint64_t)(x >> 64));\n uint64_t x_lo = ((uint64_t)(x));\n#else\n wuffs_base__multiply_u64__output x = wuffs_base__multiply_u64(\n man, ((uint64_t)po10[2]) | (((uint64_t)po10[3]) << 32));\n uint64_t x_hi = x.hi;\n uint64_t x_lo = x.lo;\n#endif\n\n // Before we shift right by at least 9 bits, recall that the look-up table\n // entry was possibly truncated. We have so far only calculated a lower bound\n // for the product (man * e), where e is (10 ** exp10). The upper bound would\n // add a further (man * 1) to the 128-bit product, which overflows the lower\n // 64" +
"-bit limb if ((x_lo + man) < man).\n //\n // If overflow occurs, that adds 1 to x_hi. Since we're about to shift right\n // by at least 9 bits, that carried 1 can be ignored unless the higher 64-bit\n // limb's low 9 bits are all on.\n if (((x_hi & 0x1FF) == 0x1FF) && ((x_lo + man) < man)) {\n // Refine our calculation of (man * e). Before, our approximation of e used\n // a \"low resolution\" 64-bit mantissa. Now use a \"high resolution\" 128-bit\n // mantissa. We've already calculated x = (man * bits_0_to_63_incl_of_e).\n // Now calculate y = (man * bits_64_to_127_incl_of_e).\n#if defined(__SIZEOF_INT128__)\n // See commit 18449ad75d582dd015c236abc85a16f333b796f3 \"Optimize 128-bit\n // muls in parse_number_f64_eisel\" for benchmark numbers.\n __uint128_t y = ((__uint128_t)man) *\n (((uint64_t)po10[0]) | (((uint64_t)po10[1]) << 32));\n uint64_t y_hi = ((uint64_t)(y >> 64));\n uint64_t y_lo = ((uint64_t)(y));\n#else\n wuffs_base__multiply_u64__output y = wuffs_base__multiply_u64" +
"(\n man, ((uint64_t)po10[0]) | (((uint64_t)po10[1]) << 32));\n uint64_t y_hi = y.hi;\n uint64_t y_lo = y.lo;\n#endif\n\n // Merge the 128-bit x and 128-bit y, which overlap by 64 bits, to\n // calculate the 192-bit product of the 64-bit man by the 128-bit e.\n // As we exit this if-block, we only care about the high 128 bits\n // (merged_hi and merged_lo) of that 192-bit product.\n uint64_t merged_hi = x_hi;\n uint64_t merged_lo = x_lo + y_hi;\n if (merged_lo < x_lo) {\n merged_hi++; // Carry the overflow bit.\n }\n\n // The \"high resolution\" approximation of e is still a lower bound. Once\n // again, see if the upper bound is large enough to produce a different\n // result. This time, if it does, give up instead of reaching for an even\n // more precise approximation to e.\n //\n // This three-part check is similar to the two-part check that guarded the\n // if block that we're now in, but it has an extra term for the middle 64\n // bits (checking that adding 1 to " +
"merged_lo would overflow).\n if (((merged_hi & 0x1FF) == 0x1FF) && ((merged_lo + 1) == 0) &&\n (y_lo + man < man)) {\n return -1;\n }\n\n // Replace the 128-bit x with merged.\n x_hi = merged_hi;\n x_lo = merged_lo;\n }\n\n // As mentioned above, shifting x_hi right by either 9 or 10 bits will leave\n // the top 10 MSBs (bits 54 ..= 63) off and the 11th MSB (bit 53) on. If the\n // MSB (before shifting) was on, adjust ret_exp2 for the larger shift.\n //\n // Having bit 53 on (and higher bits off) means that ret_mantissa is a 54-bit\n // number.\n uint64_t msb = x_hi >> 63;\n uint64_t ret_mantissa = x_hi >> (msb + 9);\n ret_exp2 -= 1 ^ msb;\n\n // IEEE 754 rounds to-nearest with ties rounded to-even. Rounding to-even can\n // be tricky. If we're half-way between two exactly representable numbers\n // (x's low 73 bits are zero and the next 2 bits that matter are \"01\"), give\n // up instead of trying to pick the winner.\n //\n // Technically, we could tighten the condition by changing \"73\" to \"7" +
"3 or 74,\n // depending on msb\", but a flat \"73\" is simpler.\n if ((x_lo == 0) && ((x_hi & 0x1FF) == 0) && ((ret_mantissa & 3) == 1)) {\n return -1;\n }\n\n // If we're not halfway then it's rounding to-nearest. Starting with a 54-bit\n // number, carry the lowest bit (bit 0) up if it's on. Regardless of whether\n // it was on or off, shifting right by one then produces a 53-bit number. If\n // carrying up overflowed, shift again.\n ret_mantissa += ret_mantissa & 1;\n ret_mantissa >>= 1;\n if ((ret_mantissa >> 53) > 0) {\n ret_mantissa >>= 1;\n ret_exp2++;\n }\n\n // Starting with a 53-bit number, IEEE 754 double-precision normal numbers\n // have an implicit mantissa bit. Mask that away and keep the low 52 bits.\n ret_mantissa &= 0x000FFFFFFFFFFFFF;\n\n // IEEE 754 double-precision floating point has 11 exponent bits. All off (0)\n // means subnormal numbers. All on (2047) means infinity or NaN.\n if ((ret_exp2 <= 0) || (2047 <= ret_exp2)) {\n return -1;\n }\n\n // Pack the bits and return.\n return ((in" +
"t64_t)(ret_mantissa | (ret_exp2 << 52)));\n}\n\n" +
"// --------\n\n// wuffs_base__private_implementation__parse_number_f64_eisel_lemire produces\n// the IEEE 754 double-precision value for an exact mantissa and base-10\n// exponent. For example:\n// - when parsing \"12345.678e+02\", man is 12345678 and exp10 is -1.\n// - when parsing \"-12\", man is 12 and exp10 is 0. Processing the leading\n// minus sign is the responsibility of the caller, not this function.\n//\n// On success, it returns a non-negative int64_t such that the low 63 bits hold\n// the 11-bit exponent and 52-bit mantissa.\n//\n// On failure, it returns a negative value.\n//\n// The algorithm is based on an original idea by Michael Eisel that was refined\n// by Daniel Lemire. See\n// https://lemire.me/blog/2020/03/10/fast-float-parsing-in-practice/\n//\n// Preconditions:\n// - man is non-zero.\n// - exp10 is in the range -326 ..= 310, the same range of the\n// wuffs_base__private_implementation__powers_of_10 array.\nstatic int64_t //\nwuffs_base__private_implementation__parse_number_f64_eisel_lemire(\n uint64" +
"_t man,\n int32_t exp10) {\n // Look up the (possibly truncated) base-2 representation of (10 ** exp10).\n // The look-up table was constructed so that it is already normalized: the\n // table entry's mantissa's MSB (most significant bit) is on.\n const uint32_t* po10 =\n &wuffs_base__private_implementation__powers_of_10[5 * (exp10 + 326)];\n\n // Normalize the man argument. The (man != 0) precondition means that a\n // non-zero bit exists.\n uint32_t clz = wuffs_base__count_leading_zeroes_u64(man);\n man <<= clz;\n\n // Calculate the return value's base-2 exponent. We might tweak it by ±1\n // later, but its initial value comes from the look-up table and clz.\n uint64_t ret_exp2 = ((uint64_t)po10[4]) - ((uint64_t)clz);\n\n // Multiply the two mantissas. Normalization means that both mantissas are at\n // least (1<<63), so the 128-bit product must be at least (1<<126). The high\n // 64 bits of the product, x_hi, must therefore be at least (1<<62).\n //\n // As a consequence, x_hi has either 0 or 1 leading" +
" zeroes. Shifting x_hi\n // right by either 9 or 10 bits (depending on x_hi's MSB) will therefore\n // leave the top 10 MSBs (bits 54 ..= 63) off and the 11th MSB (bit 53) on.\n#if defined(__SIZEOF_INT128__)\n // See commit 18449ad75d582dd015c236abc85a16f333b796f3 \"Optimize 128-bit muls\n // in parse_number_f64_eisel\" for benchmark numbers.\n __uint128_t x =\n ((__uint128_t)man) * (((uint64_t)po10[2]) | (((uint64_t)po10[3]) << 32));\n uint64_t x_hi = ((uint64_t)(x >> 64));\n uint64_t x_lo = ((uint64_t)(x));\n#else\n wuffs_base__multiply_u64__output x = wuffs_base__multiply_u64(\n man, ((uint64_t)po10[2]) | (((uint64_t)po10[3]) << 32));\n uint64_t x_hi = x.hi;\n uint64_t x_lo = x.lo;\n#endif\n\n // Before we shift right by at least 9 bits, recall that the look-up table\n // entry was possibly truncated. We have so far only calculated a lower bound\n // for the product (man * e), where e is (10 ** exp10). The upper bound would\n // add a further (man * 1) to the 128-bit product, which overflows the lower\n " +
"// 64-bit limb if ((x_lo + man) < man).\n //\n // If overflow occurs, that adds 1 to x_hi. Since we're about to shift right\n // by at least 9 bits, that carried 1 can be ignored unless the higher 64-bit\n // limb's low 9 bits are all on.\n if (((x_hi & 0x1FF) == 0x1FF) && ((x_lo + man) < man)) {\n // Refine our calculation of (man * e). Before, our approximation of e used\n // a \"low resolution\" 64-bit mantissa. Now use a \"high resolution\" 128-bit\n // mantissa. We've already calculated x = (man * bits_0_to_63_incl_of_e).\n // Now calculate y = (man * bits_64_to_127_incl_of_e).\n#if defined(__SIZEOF_INT128__)\n // See commit 18449ad75d582dd015c236abc85a16f333b796f3 \"Optimize 128-bit\n // muls in parse_number_f64_eisel\" for benchmark numbers.\n __uint128_t y = ((__uint128_t)man) *\n (((uint64_t)po10[0]) | (((uint64_t)po10[1]) << 32));\n uint64_t y_hi = ((uint64_t)(y >> 64));\n uint64_t y_lo = ((uint64_t)(y));\n#else\n wuffs_base__multiply_u64__output y = wuffs_base__multipl" +
"y_u64(\n man, ((uint64_t)po10[0]) | (((uint64_t)po10[1]) << 32));\n uint64_t y_hi = y.hi;\n uint64_t y_lo = y.lo;\n#endif\n\n // Merge the 128-bit x and 128-bit y, which overlap by 64 bits, to\n // calculate the 192-bit product of the 64-bit man by the 128-bit e.\n // As we exit this if-block, we only care about the high 128 bits\n // (merged_hi and merged_lo) of that 192-bit product.\n uint64_t merged_hi = x_hi;\n uint64_t merged_lo = x_lo + y_hi;\n if (merged_lo < x_lo) {\n merged_hi++; // Carry the overflow bit.\n }\n\n // The \"high resolution\" approximation of e is still a lower bound. Once\n // again, see if the upper bound is large enough to produce a different\n // result. This time, if it does, give up instead of reaching for an even\n // more precise approximation to e.\n //\n // This three-part check is similar to the two-part check that guarded the\n // if block that we're now in, but it has an extra term for the middle 64\n // bits (checking that adding " +
"1 to merged_lo would overflow).\n if (((merged_hi & 0x1FF) == 0x1FF) && ((merged_lo + 1) == 0) &&\n (y_lo + man < man)) {\n return -1;\n }\n\n // Replace the 128-bit x with merged.\n x_hi = merged_hi;\n x_lo = merged_lo;\n }\n\n // As mentioned above, shifting x_hi right by either 9 or 10 bits will leave\n // the top 10 MSBs (bits 54 ..= 63) off and the 11th MSB (bit 53) on. If the\n // MSB (before shifting) was on, adjust ret_exp2 for the larger shift.\n //\n // Having bit 53 on (and higher bits off) means that ret_mantissa is a 54-bit\n // number.\n uint64_t msb = x_hi >> 63;\n uint64_t ret_mantissa = x_hi >> (msb + 9);\n ret_exp2 -= 1 ^ msb;\n\n // IEEE 754 rounds to-nearest with ties rounded to-even. Rounding to-even can\n // be tricky. If we're half-way between two exactly representable numbers\n // (x's low 73 bits are zero and the next 2 bits that matter are \"01\"), give\n // up instead of trying to pick the winner.\n //\n // Technically, we could tighten the condition by changing \"73\" " +
"to \"73 or 74,\n // depending on msb\", but a flat \"73\" is simpler.\n if ((x_lo == 0) && ((x_hi & 0x1FF) == 0) && ((ret_mantissa & 3) == 1)) {\n return -1;\n }\n\n // If we're not halfway then it's rounding to-nearest. Starting with a 54-bit\n // number, carry the lowest bit (bit 0) up if it's on. Regardless of whether\n // it was on or off, shifting right by one then produces a 53-bit number. If\n // carrying up overflowed, shift again.\n ret_mantissa += ret_mantissa & 1;\n ret_mantissa >>= 1;\n if ((ret_mantissa >> 53) > 0) {\n ret_mantissa >>= 1;\n ret_exp2++;\n }\n\n // Starting with a 53-bit number, IEEE 754 double-precision normal numbers\n // have an implicit mantissa bit. Mask that away and keep the low 52 bits.\n ret_mantissa &= 0x000FFFFFFFFFFFFF;\n\n // IEEE 754 double-precision floating point has 11 exponent bits. All off (0)\n // means subnormal numbers. All on (2047) means infinity or NaN.\n if ((ret_exp2 <= 0) || (2047 <= ret_exp2)) {\n return -1;\n }\n\n // Pack the bits and return.\n return" +
" ((int64_t)(ret_mantissa | (ret_exp2 << 52)));\n}\n\n" +
"" +
"// --------\n\nstatic wuffs_base__result_f64 //\nwuffs_base__private_implementation__parse_number_f64_special(\n wuffs_base__slice_u8 s,\n uint32_t options) {\n do {\n if (options & WUFFS_BASE__PARSE_NUMBER_FXX__REJECT_INF_AND_NAN) {\n goto fail;\n }\n\n uint8_t* p = s.ptr;\n uint8_t* q = s.ptr + s.len;\n\n for (; (p < q) && (*p == '_'); p++) {\n }\n if (p >= q) {\n goto fail;\n }\n\n // Parse sign.\n bool negative = false;\n do {\n if (*p == '+') {\n p++;\n } else if (*p == '-') {\n negative = true;\n p++;\n } else {\n break;\n }\n for (; (p < q) && (*p == '_'); p++) {\n }\n } while (0);\n if (p >= q) {\n goto fail;\n }\n\n bool nan = false;\n switch (p[0]) {\n case 'I':\n case 'i':\n if (((q - p) < 3) || //\n ((p[1] != 'N') && (p[1] != 'n')) || //\n ((p[2] != 'F') && (p[2] != 'f'))) {\n goto fail;\n }\n p += 3;\n\n if ((p >= q) || (*p == '_" +
"')) {\n break;\n } else if (((q - p) < 5) || //\n ((p[0] != 'I') && (p[0] != 'i')) || //\n ((p[1] != 'N') && (p[1] != 'n')) || //\n ((p[2] != 'I') && (p[2] != 'i')) || //\n ((p[3] != 'T') && (p[3] != 't')) || //\n ((p[4] != 'Y') && (p[4] != 'y'))) {\n goto fail;\n }\n p += 5;\n\n if ((p >= q) || (*p == '_')) {\n break;\n }\n goto fail;\n\n case 'N':\n case 'n':\n if (((q - p) < 3) || //\n ((p[1] != 'A') && (p[1] != 'a')) || //\n ((p[2] != 'N') && (p[2] != 'n'))) {\n goto fail;\n }\n p += 3;\n\n if ((p >= q) || (*p == '_')) {\n nan = true;\n break;\n }\n goto fail;\n\n default:\n goto fail;\n }\n\n // Finish.\n for (; (p < q) && (*p == '_'); p++) {\n }\n if (p != q) {\n goto fail;\n }\n wuffs_base__result_f64 ret;\n" +
@@ -565,11 +565,11 @@ const BaseTokenPrivateH = "" +
""
const BaseTokenPublicH = "" +
"// ---------------- Tokens\n\n// wuffs_base__token is an element of a byte stream's tokenization.\n//\n// See https://github.com/google/wuffs/blob/master/doc/note/tokens.md\ntypedef struct {\n uint64_t repr;\n\n#ifdef __cplusplus\n inline int64_t value() const;\n inline int64_t value_extension() const;\n inline int64_t value_major() const;\n inline int64_t value_base_category() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_detail() const;\n inline bool continued() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n" +
"// ---------------- Tokens\n\n// wuffs_base__token is an element of a byte stream's tokenization.\n//\n// See https://github.com/google/wuffs/blob/master/doc/note/tokens.md\ntypedef struct {\n uint64_t repr;\n\n#ifdef __cplusplus\n inline int64_t value() const;\n inline int64_t value_extension() const;\n inline int64_t value_major() const;\n inline int64_t value_base_category() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_detail() const;\n inline int64_t value_base_detail__sign_extended() const;\n inline bool continued() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n" +
"" +
"// --------\n\n#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 38\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 17\n#define WUFFS_BASE__TOKEN__CONTINUED__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n" +
"" +
"// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__STRING 2\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3\n#define WUFFS_BASE__TOKEN__VBC__LITERAL 4\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 5\n#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER 6\n\n" +
"// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__STRING 2\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3\n#define WUFFS_BASE__TOKEN__VBC__LITERAL 4\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 5\n#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED 6\n#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED 7\n\n" +
"" +
"// --------\n\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002\n\n" +
"" +
@@ -585,8 +585,9 @@ const BaseTokenPublicH = "" +
"dian, little-endian or text. For binary formats, the\n// token length (after adjusting for FORMAT_IGNORE_ETC) discriminates\n// e.g. u16 little-endian vs u32 little-endian.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_BIG_ENDIAN 0x00100\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_LITTLE_ENDIAN 0x00200\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT 0x00400\n\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_IGNORE_FIRST_BYTE 0x01000\n\n" +
"" +
"// --------\n\n// wuffs_base__token__value returns the token's high 46 bits, sign-extended. A\n// negative value means an extended token, non-negative means a simple token.\nstatic inline int64_t //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE__SHIFT;\n}\n\n// wuffs_base__token__value_extension returns a negative value if the token was\n// not an extended token.\nstatic inline int64_t //\nwuffs_base__token__value_extension(const wuffs_base__token* t) {\n return (~(int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT;\n}\n\n// wuffs_base__token__value_major returns a negative value if the token was not\n// a simple token.\nstatic inline int64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT;\n}\n\n// wuffs_base__token__value_base_category returns a negative value if the token\n// was not a simple token.\nstatic inline int64_t //\nwuffs_base__token__value_base_cat" +
"egory(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0x1FFFFFF;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) & 0x1FFFFF;\n}\n\nstatic inline bool //\nwuffs_base__token__continued(const wuffs_base__token* t) {\n return t->repr & 0x10000;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) & 0xFFFF;\n}\n\n#ifdef __cplusplus\n\ninline int64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_extension() const {\n return wuffs_base__token__value_extension(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_major() const {\n retu" +
"rn wuffs_base__token__value_major(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline bool //\nwuffs_base__token::continued() const {\n return wuffs_base__token__continued(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" +
"egory(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0x1FFFFFF;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) & 0x1FFFFF;\n}\n\nstatic inline int64_t //\nwuffs_base__token__value_base_detail__sign_extended(\n const wuffs_base__token* t) {\n // The VBD is 21 bits in the middle of t->repr. Left shift the high (64 - 21\n // - ETC__SHIFT) bits off, then right shift (sign-extending) back down.\n uint64_t u = t->repr << (43 - WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT);\n return ((int64_t)u) >> 43;\n}\n\nstatic inline bool //\nwuffs_base__token__continued(const wuffs_base__token* t) {\n return t->repr & 0x10000;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token*" +
" t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) & 0xFFFF;\n}\n\n#ifdef __cplusplus\n\ninline int64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_extension() const {\n return wuffs_base__token__value_extension(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_base_detail__sign_extended() const {\n return wuffs_base__token__value_base_detail__sign_extended(this);\n}\n\ninline bool //\nwuffs_base__token::continued() const {\n return wuffs_base__token__continued" +
"(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" +
"" +
"// --------\n\ntypedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;\n\nstatic inline wuffs_base__slice_token //\nwuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {\n wuffs_base__slice_token ret;\n ret.ptr = ptr;\n ret.len = len;\n return ret;\n}\n\n" +
"" +
+2 -1
View File
@@ -81,7 +81,8 @@ var Consts = [...]struct {
{t.IDU32, "3", "TOKEN__VBC__UNICODE_CODE_POINT"},
{t.IDU32, "4", "TOKEN__VBC__LITERAL"},
{t.IDU32, "5", "TOKEN__VBC__NUMBER"},
{t.IDU32, "6", "TOKEN__VBC__INLINE_INTEGER"},
{t.IDU32, "6", "TOKEN__VBC__INLINE_INTEGER_SIGNED"},
{t.IDU32, "7", "TOKEN__VBC__INLINE_INTEGER_UNSIGNED"},
// ----
+20 -4
View File
@@ -2099,6 +2099,7 @@ typedef struct {
inline int64_t value_base_category() const;
inline uint64_t value_minor() const;
inline uint64_t value_base_detail() const;
inline int64_t value_base_detail__sign_extended() const;
inline bool continued() const;
inline uint64_t length() const;
#endif // __cplusplus
@@ -2133,7 +2134,8 @@ wuffs_base__make_token(uint64_t repr) {
#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3
#define WUFFS_BASE__TOKEN__VBC__LITERAL 4
#define WUFFS_BASE__TOKEN__VBC__NUMBER 5
#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER 6
#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED 6
#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED 7
// --------
@@ -2275,6 +2277,15 @@ wuffs_base__token__value_base_detail(const wuffs_base__token* t) {
return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) & 0x1FFFFF;
}
static inline int64_t //
wuffs_base__token__value_base_detail__sign_extended(
const wuffs_base__token* t) {
// The VBD is 21 bits in the middle of t->repr. Left shift the high (64 - 21
// - ETC__SHIFT) bits off, then right shift (sign-extending) back down.
uint64_t u = t->repr << (43 - WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT);
return ((int64_t)u) >> 43;
}
static inline bool //
wuffs_base__token__continued(const wuffs_base__token* t) {
return t->repr & 0x10000;
@@ -2317,6 +2328,11 @@ wuffs_base__token::value_base_detail() const {
return wuffs_base__token__value_base_detail(this);
}
inline int64_t //
wuffs_base__token::value_base_detail__sign_extended() const {
return wuffs_base__token__value_base_detail__sign_extended(this);
}
inline bool //
wuffs_base__token::continued() const {
return wuffs_base__token__continued(this);
@@ -11160,8 +11176,8 @@ wuffs_base__private_implementation__high_prec_dec__round_just_enough(
//
// On failure, it returns a negative value.
//
// The algorithm is based on an original idea by Michael Eisel and refined by
// Daniel Lemire. See
// The algorithm is based on an original idea by Michael Eisel that was refined
// by Daniel Lemire. See
// https://lemire.me/blog/2020/03/10/fast-float-parsing-in-practice/
//
// Preconditions:
@@ -16521,7 +16537,7 @@ wuffs_cbor__decoder__decode_tokens(
if (v_c_major == 0) {
if (v_c_minor < 24) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)((12582912 | ((uint32_t)(v_c_minor))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)((14680064 | ((uint32_t)(v_c_minor))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__goto_parsed_a_leaf_value__break;
} else if (v_c_minor < 28) {
+17 -18
View File
@@ -225,22 +225,22 @@ parse_flags(int argc, char** argv) {
}
const char* g_vbc_names[16] = {
"0:Filler..........", //
"1:Structure.......", //
"2:String..........", //
"3:UnicodeCodePoint", //
"4:Literal.........", //
"5:Number..........", //
"6:Inline_Integer..", //
"7:Reserved........", //
"8:Reserved........", //
"9:Reserved........", //
"A:Reserved........", //
"B:Reserved........", //
"C:Reserved........", //
"D:Reserved........", //
"E:Reserved........", //
"F:Reserved........", //
"0:Filler...........", //
"1:Structure........", //
"2:String...........", //
"3:UnicodeCodePoint.", //
"4:Literal..........", //
"5:Number...........", //
"6:InlineIntSigned..", //
"7:InlineIntUnsigned", //
"8:Reserved.........", //
"9:Reserved.........", //
"A:Reserved.........", //
"B:Reserved.........", //
"C:Reserved.........", //
"D:Reserved.........", //
"E:Reserved.........", //
"F:Reserved.........", //
};
const int g_base38_decode[38] = {
@@ -357,8 +357,7 @@ main1(int argc, char** argv) {
printf("vmajor=0x%06" PRIX32 ":%s vminor=0x%06" PRIX32 "\n",
vmajor, vmajor_name, vminor);
} else if (vmajor == 0) {
printf("vbc=%s. vbd=0x%06" PRIX32 "\n", g_vbc_names[vbc & 15],
vbd);
printf("vbc=%s vbd=0x%06" PRIX32 "\n", g_vbc_names[vbc & 15], vbd);
} else {
printf("extended... vextension=0x%012" PRIX64 "\n",
wuffs_base__token__value_extension(t));
+2 -2
View File
@@ -161,7 +161,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
if c_minor < 0x18 {
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: (base.TOKEN__VBC__INLINE_INTEGER << 21) |
value_minor: (base.TOKEN__VBC__INLINE_INTEGER_UNSIGNED << 21) |
(c_minor as base.u32),
continued: 0,
length: 1)
@@ -184,7 +184,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
if c_minor < 0x18 {
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: (base.TOKEN__VBC__INLINE_INTEGER << 21) |
value_minor: (base.TOKEN__VBC__INLINE_INTEGER_SIGNED << 21) |
(0x1F_FFFF - (c_minor as base.u32)),
continued: 0,
length: 1)