From c9d4e34dfc682aa14ad6bee907b260a80ea5cf9f Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Tue, 21 Jul 2020 15:20:34 +1000 Subject: [PATCH] Split ETC__VBC__INLINE_INTEGER into {,un}signed --- example/jsonptr/jsonptr.cc | 29 +++++++++++++++------ internal/cgen/base/token-public.h | 18 ++++++++++++- internal/cgen/data/data.go | 25 +++++++++--------- lang/builtin/builtin.go | 3 ++- release/c/wuffs-unsupported-snapshot.c | 24 +++++++++++++++--- script/print-json-token-debug-format.c | 35 +++++++++++++------------- std/cbor/decode_cbor.wuffs | 4 +-- 7 files changed, 93 insertions(+), 45 deletions(-) diff --git a/example/jsonptr/jsonptr.cc b/example/jsonptr/jsonptr.cc index 12aed48a..28dbf9be 100644 --- a/example/jsonptr/jsonptr.cc +++ b/example/jsonptr/jsonptr.cc @@ -1107,15 +1107,23 @@ fail: } const char* // -write_inline_integer(uint64_t vbd, uint8_t* ptr, size_t len) { +write_inline_integer(uint64_t x, bool x_is_signed, uint8_t* ptr, size_t len) { if (g_flags.output_format == file_format::cbor) { return write_dst(ptr, len); } - uint8_t buf[WUFFS_BASE__I64__BYTE_LENGTH__MAX_INCL]; - size_t n = wuffs_base__render_number_i64( - wuffs_base__make_slice_u8(&buf[0], sizeof buf), (int16_t)vbd, - WUFFS_BASE__RENDER_NUMBER_XXX__DEFAULT_OPTIONS); + // Adding the two ETC__BYTE_LENGTH__ETC constants is overkill, but it's + // simpler (for producing a constant-expression array size) than taking the + // maximum of the two. + uint8_t buf[WUFFS_BASE__I64__BYTE_LENGTH__MAX_INCL + + WUFFS_BASE__U64__BYTE_LENGTH__MAX_INCL]; + wuffs_base__slice_u8 dst = wuffs_base__make_slice_u8(&buf[0], sizeof buf); + size_t n = + x_is_signed + ? wuffs_base__render_number_i64( + dst, (int64_t)x, WUFFS_BASE__RENDER_NUMBER_XXX__DEFAULT_OPTIONS) + : wuffs_base__render_number_u64( + dst, x, WUFFS_BASE__RENDER_NUMBER_XXX__DEFAULT_OPTIONS); return write_dst(&buf[0], n); } @@ -1522,10 +1530,17 @@ handle_token(wuffs_base__token t, bool start_of_token_chain) { len)); goto after_value; - case WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER: + case WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED: + case WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED: { + bool x_is_signed = vbc == WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED; + uint64_t x = x_is_signed + ? ((uint64_t)(t.value_base_detail__sign_extended())) + : vbd; TRY(write_inline_integer( - vbd, g_src.data.ptr + g_curr_token_end_src_index - len, len)); + x, x_is_signed, g_src.data.ptr + g_curr_token_end_src_index - len, + len)); goto after_value; + } } if (t.value_major() == WUFFS_CBOR__TOKEN_VALUE_MAJOR) { diff --git a/internal/cgen/base/token-public.h b/internal/cgen/base/token-public.h index 362cc0ef..24a10a7f 100644 --- a/internal/cgen/base/token-public.h +++ b/internal/cgen/base/token-public.h @@ -29,6 +29,7 @@ typedef struct { inline int64_t value_base_category() const; inline uint64_t value_minor() const; inline uint64_t value_base_detail() const; + inline int64_t value_base_detail__sign_extended() const; inline bool continued() const; inline uint64_t length() const; #endif // __cplusplus @@ -63,7 +64,8 @@ wuffs_base__make_token(uint64_t repr) { #define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3 #define WUFFS_BASE__TOKEN__VBC__LITERAL 4 #define WUFFS_BASE__TOKEN__VBC__NUMBER 5 -#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER 6 +#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED 6 +#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED 7 // -------- @@ -205,6 +207,15 @@ wuffs_base__token__value_base_detail(const wuffs_base__token* t) { return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) & 0x1FFFFF; } +static inline int64_t // +wuffs_base__token__value_base_detail__sign_extended( + const wuffs_base__token* t) { + // The VBD is 21 bits in the middle of t->repr. Left shift the high (64 - 21 + // - ETC__SHIFT) bits off, then right shift (sign-extending) back down. + uint64_t u = t->repr << (43 - WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT); + return ((int64_t)u) >> 43; +} + static inline bool // wuffs_base__token__continued(const wuffs_base__token* t) { return t->repr & 0x10000; @@ -247,6 +258,11 @@ wuffs_base__token::value_base_detail() const { return wuffs_base__token__value_base_detail(this); } +inline int64_t // +wuffs_base__token::value_base_detail__sign_extended() const { + return wuffs_base__token__value_base_detail__sign_extended(this); +} + inline bool // wuffs_base__token::continued() const { return wuffs_base__token__continued(this); diff --git a/internal/cgen/data/data.go b/internal/cgen/data/data.go index fd7663aa..457337d0 100644 --- a/internal/cgen/data/data.go +++ b/internal/cgen/data/data.go @@ -68,14 +68,14 @@ const BaseFloatConvSubmoduleCodeC = "" + " upper_delta = +1;\n } else if (hd != ud) {\n // For example:\n // h = 12345???\n // upper = 12346???\n upper_delta = +0;\n }\n } else if (upper_delta == 0) {\n if ((hd != 9) || (ud != 0)) {\n // For example:\n // h = 1234598?\n // upper = 1234600?\n upper_delta = +1;\n }\n }\n\n // We can round up if upper has a different digit than h and either upper\n // is inclusive or upper is bigger than the result of rounding up.\n bool can_round_up =\n (upper_delta > 0) || //\n ((upper_delta == 0) && //\n (inclusive || ((ui + 1) < ((int32_t)(upper.num_digits)))));\n\n // If we can round either way, round to nearest. If we can round only one\n // way, do it. If we can't round, continue the loop.\n if (can_round_down) {\n if (can_round_up) {\n wuffs_base__private_implementation__high_prec_dec__round_nearest(\n h, hi + 1);\n return;\n } else {\n wuffs_base__private_implementat" + "ion__high_prec_dec__round_down(h,\n hi + 1);\n return;\n }\n } else {\n if (can_round_up) {\n wuffs_base__private_implementation__high_prec_dec__round_up(h, hi + 1);\n return;\n }\n }\n }\n}\n\n" + "" + - "// --------\n\n// wuffs_base__private_implementation__parse_number_f64_eisel_lemire produces\n// the IEEE 754 double-precision value for an exact mantissa and base-10\n// exponent. For example:\n// - when parsing \"12345.678e+02\", man is 12345678 and exp10 is -1.\n// - when parsing \"-12\", man is 12 and exp10 is 0. Processing the leading\n// minus sign is the responsibility of the caller, not this function.\n//\n// On success, it returns a non-negative int64_t such that the low 63 bits hold\n// the 11-bit exponent and 52-bit mantissa.\n//\n// On failure, it returns a negative value.\n//\n// The algorithm is based on an original idea by Michael Eisel and refined by\n// Daniel Lemire. See\n// https://lemire.me/blog/2020/03/10/fast-float-parsing-in-practice/\n//\n// Preconditions:\n// - man is non-zero.\n// - exp10 is in the range -326 ..= 310, the same range of the\n// wuffs_base__private_implementation__powers_of_10 array.\nstatic int64_t //\nwuffs_base__private_implementation__parse_number_f64_eisel_lemire(\n uint64_t ma" + - "n,\n int32_t exp10) {\n // Look up the (possibly truncated) base-2 representation of (10 ** exp10).\n // The look-up table was constructed so that it is already normalized: the\n // table entry's mantissa's MSB (most significant bit) is on.\n const uint32_t* po10 =\n &wuffs_base__private_implementation__powers_of_10[5 * (exp10 + 326)];\n\n // Normalize the man argument. The (man != 0) precondition means that a\n // non-zero bit exists.\n uint32_t clz = wuffs_base__count_leading_zeroes_u64(man);\n man <<= clz;\n\n // Calculate the return value's base-2 exponent. We might tweak it by ±1\n // later, but its initial value comes from the look-up table and clz.\n uint64_t ret_exp2 = ((uint64_t)po10[4]) - ((uint64_t)clz);\n\n // Multiply the two mantissas. Normalization means that both mantissas are at\n // least (1<<63), so the 128-bit product must be at least (1<<126). The high\n // 64 bits of the product, x_hi, must therefore be at least (1<<62).\n //\n // As a consequence, x_hi has either 0 or 1 leading zero" + - "es. Shifting x_hi\n // right by either 9 or 10 bits (depending on x_hi's MSB) will therefore\n // leave the top 10 MSBs (bits 54 ..= 63) off and the 11th MSB (bit 53) on.\n#if defined(__SIZEOF_INT128__)\n // See commit 18449ad75d582dd015c236abc85a16f333b796f3 \"Optimize 128-bit muls\n // in parse_number_f64_eisel\" for benchmark numbers.\n __uint128_t x =\n ((__uint128_t)man) * (((uint64_t)po10[2]) | (((uint64_t)po10[3]) << 32));\n uint64_t x_hi = ((uint64_t)(x >> 64));\n uint64_t x_lo = ((uint64_t)(x));\n#else\n wuffs_base__multiply_u64__output x = wuffs_base__multiply_u64(\n man, ((uint64_t)po10[2]) | (((uint64_t)po10[3]) << 32));\n uint64_t x_hi = x.hi;\n uint64_t x_lo = x.lo;\n#endif\n\n // Before we shift right by at least 9 bits, recall that the look-up table\n // entry was possibly truncated. We have so far only calculated a lower bound\n // for the product (man * e), where e is (10 ** exp10). The upper bound would\n // add a further (man * 1) to the 128-bit product, which overflows the lower\n // 64" + - "-bit limb if ((x_lo + man) < man).\n //\n // If overflow occurs, that adds 1 to x_hi. Since we're about to shift right\n // by at least 9 bits, that carried 1 can be ignored unless the higher 64-bit\n // limb's low 9 bits are all on.\n if (((x_hi & 0x1FF) == 0x1FF) && ((x_lo + man) < man)) {\n // Refine our calculation of (man * e). Before, our approximation of e used\n // a \"low resolution\" 64-bit mantissa. Now use a \"high resolution\" 128-bit\n // mantissa. We've already calculated x = (man * bits_0_to_63_incl_of_e).\n // Now calculate y = (man * bits_64_to_127_incl_of_e).\n#if defined(__SIZEOF_INT128__)\n // See commit 18449ad75d582dd015c236abc85a16f333b796f3 \"Optimize 128-bit\n // muls in parse_number_f64_eisel\" for benchmark numbers.\n __uint128_t y = ((__uint128_t)man) *\n (((uint64_t)po10[0]) | (((uint64_t)po10[1]) << 32));\n uint64_t y_hi = ((uint64_t)(y >> 64));\n uint64_t y_lo = ((uint64_t)(y));\n#else\n wuffs_base__multiply_u64__output y = wuffs_base__multiply_u64" + - "(\n man, ((uint64_t)po10[0]) | (((uint64_t)po10[1]) << 32));\n uint64_t y_hi = y.hi;\n uint64_t y_lo = y.lo;\n#endif\n\n // Merge the 128-bit x and 128-bit y, which overlap by 64 bits, to\n // calculate the 192-bit product of the 64-bit man by the 128-bit e.\n // As we exit this if-block, we only care about the high 128 bits\n // (merged_hi and merged_lo) of that 192-bit product.\n uint64_t merged_hi = x_hi;\n uint64_t merged_lo = x_lo + y_hi;\n if (merged_lo < x_lo) {\n merged_hi++; // Carry the overflow bit.\n }\n\n // The \"high resolution\" approximation of e is still a lower bound. Once\n // again, see if the upper bound is large enough to produce a different\n // result. This time, if it does, give up instead of reaching for an even\n // more precise approximation to e.\n //\n // This three-part check is similar to the two-part check that guarded the\n // if block that we're now in, but it has an extra term for the middle 64\n // bits (checking that adding 1 to " + - "merged_lo would overflow).\n if (((merged_hi & 0x1FF) == 0x1FF) && ((merged_lo + 1) == 0) &&\n (y_lo + man < man)) {\n return -1;\n }\n\n // Replace the 128-bit x with merged.\n x_hi = merged_hi;\n x_lo = merged_lo;\n }\n\n // As mentioned above, shifting x_hi right by either 9 or 10 bits will leave\n // the top 10 MSBs (bits 54 ..= 63) off and the 11th MSB (bit 53) on. If the\n // MSB (before shifting) was on, adjust ret_exp2 for the larger shift.\n //\n // Having bit 53 on (and higher bits off) means that ret_mantissa is a 54-bit\n // number.\n uint64_t msb = x_hi >> 63;\n uint64_t ret_mantissa = x_hi >> (msb + 9);\n ret_exp2 -= 1 ^ msb;\n\n // IEEE 754 rounds to-nearest with ties rounded to-even. Rounding to-even can\n // be tricky. If we're half-way between two exactly representable numbers\n // (x's low 73 bits are zero and the next 2 bits that matter are \"01\"), give\n // up instead of trying to pick the winner.\n //\n // Technically, we could tighten the condition by changing \"73\" to \"7" + - "3 or 74,\n // depending on msb\", but a flat \"73\" is simpler.\n if ((x_lo == 0) && ((x_hi & 0x1FF) == 0) && ((ret_mantissa & 3) == 1)) {\n return -1;\n }\n\n // If we're not halfway then it's rounding to-nearest. Starting with a 54-bit\n // number, carry the lowest bit (bit 0) up if it's on. Regardless of whether\n // it was on or off, shifting right by one then produces a 53-bit number. If\n // carrying up overflowed, shift again.\n ret_mantissa += ret_mantissa & 1;\n ret_mantissa >>= 1;\n if ((ret_mantissa >> 53) > 0) {\n ret_mantissa >>= 1;\n ret_exp2++;\n }\n\n // Starting with a 53-bit number, IEEE 754 double-precision normal numbers\n // have an implicit mantissa bit. Mask that away and keep the low 52 bits.\n ret_mantissa &= 0x000FFFFFFFFFFFFF;\n\n // IEEE 754 double-precision floating point has 11 exponent bits. All off (0)\n // means subnormal numbers. All on (2047) means infinity or NaN.\n if ((ret_exp2 <= 0) || (2047 <= ret_exp2)) {\n return -1;\n }\n\n // Pack the bits and return.\n return ((in" + - "t64_t)(ret_mantissa | (ret_exp2 << 52)));\n}\n\n" + + "// --------\n\n// wuffs_base__private_implementation__parse_number_f64_eisel_lemire produces\n// the IEEE 754 double-precision value for an exact mantissa and base-10\n// exponent. For example:\n// - when parsing \"12345.678e+02\", man is 12345678 and exp10 is -1.\n// - when parsing \"-12\", man is 12 and exp10 is 0. Processing the leading\n// minus sign is the responsibility of the caller, not this function.\n//\n// On success, it returns a non-negative int64_t such that the low 63 bits hold\n// the 11-bit exponent and 52-bit mantissa.\n//\n// On failure, it returns a negative value.\n//\n// The algorithm is based on an original idea by Michael Eisel that was refined\n// by Daniel Lemire. See\n// https://lemire.me/blog/2020/03/10/fast-float-parsing-in-practice/\n//\n// Preconditions:\n// - man is non-zero.\n// - exp10 is in the range -326 ..= 310, the same range of the\n// wuffs_base__private_implementation__powers_of_10 array.\nstatic int64_t //\nwuffs_base__private_implementation__parse_number_f64_eisel_lemire(\n uint64" + + "_t man,\n int32_t exp10) {\n // Look up the (possibly truncated) base-2 representation of (10 ** exp10).\n // The look-up table was constructed so that it is already normalized: the\n // table entry's mantissa's MSB (most significant bit) is on.\n const uint32_t* po10 =\n &wuffs_base__private_implementation__powers_of_10[5 * (exp10 + 326)];\n\n // Normalize the man argument. The (man != 0) precondition means that a\n // non-zero bit exists.\n uint32_t clz = wuffs_base__count_leading_zeroes_u64(man);\n man <<= clz;\n\n // Calculate the return value's base-2 exponent. We might tweak it by ±1\n // later, but its initial value comes from the look-up table and clz.\n uint64_t ret_exp2 = ((uint64_t)po10[4]) - ((uint64_t)clz);\n\n // Multiply the two mantissas. Normalization means that both mantissas are at\n // least (1<<63), so the 128-bit product must be at least (1<<126). The high\n // 64 bits of the product, x_hi, must therefore be at least (1<<62).\n //\n // As a consequence, x_hi has either 0 or 1 leading" + + " zeroes. Shifting x_hi\n // right by either 9 or 10 bits (depending on x_hi's MSB) will therefore\n // leave the top 10 MSBs (bits 54 ..= 63) off and the 11th MSB (bit 53) on.\n#if defined(__SIZEOF_INT128__)\n // See commit 18449ad75d582dd015c236abc85a16f333b796f3 \"Optimize 128-bit muls\n // in parse_number_f64_eisel\" for benchmark numbers.\n __uint128_t x =\n ((__uint128_t)man) * (((uint64_t)po10[2]) | (((uint64_t)po10[3]) << 32));\n uint64_t x_hi = ((uint64_t)(x >> 64));\n uint64_t x_lo = ((uint64_t)(x));\n#else\n wuffs_base__multiply_u64__output x = wuffs_base__multiply_u64(\n man, ((uint64_t)po10[2]) | (((uint64_t)po10[3]) << 32));\n uint64_t x_hi = x.hi;\n uint64_t x_lo = x.lo;\n#endif\n\n // Before we shift right by at least 9 bits, recall that the look-up table\n // entry was possibly truncated. We have so far only calculated a lower bound\n // for the product (man * e), where e is (10 ** exp10). The upper bound would\n // add a further (man * 1) to the 128-bit product, which overflows the lower\n " + + "// 64-bit limb if ((x_lo + man) < man).\n //\n // If overflow occurs, that adds 1 to x_hi. Since we're about to shift right\n // by at least 9 bits, that carried 1 can be ignored unless the higher 64-bit\n // limb's low 9 bits are all on.\n if (((x_hi & 0x1FF) == 0x1FF) && ((x_lo + man) < man)) {\n // Refine our calculation of (man * e). Before, our approximation of e used\n // a \"low resolution\" 64-bit mantissa. Now use a \"high resolution\" 128-bit\n // mantissa. We've already calculated x = (man * bits_0_to_63_incl_of_e).\n // Now calculate y = (man * bits_64_to_127_incl_of_e).\n#if defined(__SIZEOF_INT128__)\n // See commit 18449ad75d582dd015c236abc85a16f333b796f3 \"Optimize 128-bit\n // muls in parse_number_f64_eisel\" for benchmark numbers.\n __uint128_t y = ((__uint128_t)man) *\n (((uint64_t)po10[0]) | (((uint64_t)po10[1]) << 32));\n uint64_t y_hi = ((uint64_t)(y >> 64));\n uint64_t y_lo = ((uint64_t)(y));\n#else\n wuffs_base__multiply_u64__output y = wuffs_base__multipl" + + "y_u64(\n man, ((uint64_t)po10[0]) | (((uint64_t)po10[1]) << 32));\n uint64_t y_hi = y.hi;\n uint64_t y_lo = y.lo;\n#endif\n\n // Merge the 128-bit x and 128-bit y, which overlap by 64 bits, to\n // calculate the 192-bit product of the 64-bit man by the 128-bit e.\n // As we exit this if-block, we only care about the high 128 bits\n // (merged_hi and merged_lo) of that 192-bit product.\n uint64_t merged_hi = x_hi;\n uint64_t merged_lo = x_lo + y_hi;\n if (merged_lo < x_lo) {\n merged_hi++; // Carry the overflow bit.\n }\n\n // The \"high resolution\" approximation of e is still a lower bound. Once\n // again, see if the upper bound is large enough to produce a different\n // result. This time, if it does, give up instead of reaching for an even\n // more precise approximation to e.\n //\n // This three-part check is similar to the two-part check that guarded the\n // if block that we're now in, but it has an extra term for the middle 64\n // bits (checking that adding " + + "1 to merged_lo would overflow).\n if (((merged_hi & 0x1FF) == 0x1FF) && ((merged_lo + 1) == 0) &&\n (y_lo + man < man)) {\n return -1;\n }\n\n // Replace the 128-bit x with merged.\n x_hi = merged_hi;\n x_lo = merged_lo;\n }\n\n // As mentioned above, shifting x_hi right by either 9 or 10 bits will leave\n // the top 10 MSBs (bits 54 ..= 63) off and the 11th MSB (bit 53) on. If the\n // MSB (before shifting) was on, adjust ret_exp2 for the larger shift.\n //\n // Having bit 53 on (and higher bits off) means that ret_mantissa is a 54-bit\n // number.\n uint64_t msb = x_hi >> 63;\n uint64_t ret_mantissa = x_hi >> (msb + 9);\n ret_exp2 -= 1 ^ msb;\n\n // IEEE 754 rounds to-nearest with ties rounded to-even. Rounding to-even can\n // be tricky. If we're half-way between two exactly representable numbers\n // (x's low 73 bits are zero and the next 2 bits that matter are \"01\"), give\n // up instead of trying to pick the winner.\n //\n // Technically, we could tighten the condition by changing \"73\" " + + "to \"73 or 74,\n // depending on msb\", but a flat \"73\" is simpler.\n if ((x_lo == 0) && ((x_hi & 0x1FF) == 0) && ((ret_mantissa & 3) == 1)) {\n return -1;\n }\n\n // If we're not halfway then it's rounding to-nearest. Starting with a 54-bit\n // number, carry the lowest bit (bit 0) up if it's on. Regardless of whether\n // it was on or off, shifting right by one then produces a 53-bit number. If\n // carrying up overflowed, shift again.\n ret_mantissa += ret_mantissa & 1;\n ret_mantissa >>= 1;\n if ((ret_mantissa >> 53) > 0) {\n ret_mantissa >>= 1;\n ret_exp2++;\n }\n\n // Starting with a 53-bit number, IEEE 754 double-precision normal numbers\n // have an implicit mantissa bit. Mask that away and keep the low 52 bits.\n ret_mantissa &= 0x000FFFFFFFFFFFFF;\n\n // IEEE 754 double-precision floating point has 11 exponent bits. All off (0)\n // means subnormal numbers. All on (2047) means infinity or NaN.\n if ((ret_exp2 <= 0) || (2047 <= ret_exp2)) {\n return -1;\n }\n\n // Pack the bits and return.\n return" + + " ((int64_t)(ret_mantissa | (ret_exp2 << 52)));\n}\n\n" + "" + "// --------\n\nstatic wuffs_base__result_f64 //\nwuffs_base__private_implementation__parse_number_f64_special(\n wuffs_base__slice_u8 s,\n uint32_t options) {\n do {\n if (options & WUFFS_BASE__PARSE_NUMBER_FXX__REJECT_INF_AND_NAN) {\n goto fail;\n }\n\n uint8_t* p = s.ptr;\n uint8_t* q = s.ptr + s.len;\n\n for (; (p < q) && (*p == '_'); p++) {\n }\n if (p >= q) {\n goto fail;\n }\n\n // Parse sign.\n bool negative = false;\n do {\n if (*p == '+') {\n p++;\n } else if (*p == '-') {\n negative = true;\n p++;\n } else {\n break;\n }\n for (; (p < q) && (*p == '_'); p++) {\n }\n } while (0);\n if (p >= q) {\n goto fail;\n }\n\n bool nan = false;\n switch (p[0]) {\n case 'I':\n case 'i':\n if (((q - p) < 3) || //\n ((p[1] != 'N') && (p[1] != 'n')) || //\n ((p[2] != 'F') && (p[2] != 'f'))) {\n goto fail;\n }\n p += 3;\n\n if ((p >= q) || (*p == '_" + "')) {\n break;\n } else if (((q - p) < 5) || //\n ((p[0] != 'I') && (p[0] != 'i')) || //\n ((p[1] != 'N') && (p[1] != 'n')) || //\n ((p[2] != 'I') && (p[2] != 'i')) || //\n ((p[3] != 'T') && (p[3] != 't')) || //\n ((p[4] != 'Y') && (p[4] != 'y'))) {\n goto fail;\n }\n p += 5;\n\n if ((p >= q) || (*p == '_')) {\n break;\n }\n goto fail;\n\n case 'N':\n case 'n':\n if (((q - p) < 3) || //\n ((p[1] != 'A') && (p[1] != 'a')) || //\n ((p[2] != 'N') && (p[2] != 'n'))) {\n goto fail;\n }\n p += 3;\n\n if ((p >= q) || (*p == '_')) {\n nan = true;\n break;\n }\n goto fail;\n\n default:\n goto fail;\n }\n\n // Finish.\n for (; (p < q) && (*p == '_'); p++) {\n }\n if (p != q) {\n goto fail;\n }\n wuffs_base__result_f64 ret;\n" + @@ -565,11 +565,11 @@ const BaseTokenPrivateH = "" + "" const BaseTokenPublicH = "" + - "// ---------------- Tokens\n\n// wuffs_base__token is an element of a byte stream's tokenization.\n//\n// See https://github.com/google/wuffs/blob/master/doc/note/tokens.md\ntypedef struct {\n uint64_t repr;\n\n#ifdef __cplusplus\n inline int64_t value() const;\n inline int64_t value_extension() const;\n inline int64_t value_major() const;\n inline int64_t value_base_category() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_detail() const;\n inline bool continued() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n" + + "// ---------------- Tokens\n\n// wuffs_base__token is an element of a byte stream's tokenization.\n//\n// See https://github.com/google/wuffs/blob/master/doc/note/tokens.md\ntypedef struct {\n uint64_t repr;\n\n#ifdef __cplusplus\n inline int64_t value() const;\n inline int64_t value_extension() const;\n inline int64_t value_major() const;\n inline int64_t value_base_category() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_detail() const;\n inline int64_t value_base_detail__sign_extended() const;\n inline bool continued() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n" + "" + "// --------\n\n#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 38\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 17\n#define WUFFS_BASE__TOKEN__CONTINUED__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n" + "" + - "// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__STRING 2\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3\n#define WUFFS_BASE__TOKEN__VBC__LITERAL 4\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 5\n#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER 6\n\n" + + "// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__STRING 2\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3\n#define WUFFS_BASE__TOKEN__VBC__LITERAL 4\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 5\n#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED 6\n#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED 7\n\n" + "" + "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002\n\n" + "" + @@ -585,8 +585,9 @@ const BaseTokenPublicH = "" + "dian, little-endian or text. For binary formats, the\n// token length (after adjusting for FORMAT_IGNORE_ETC) discriminates\n// e.g. u16 little-endian vs u32 little-endian.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_BIG_ENDIAN 0x00100\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_LITTLE_ENDIAN 0x00200\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT 0x00400\n\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_IGNORE_FIRST_BYTE 0x01000\n\n" + "" + "// --------\n\n// wuffs_base__token__value returns the token's high 46 bits, sign-extended. A\n// negative value means an extended token, non-negative means a simple token.\nstatic inline int64_t //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE__SHIFT;\n}\n\n// wuffs_base__token__value_extension returns a negative value if the token was\n// not an extended token.\nstatic inline int64_t //\nwuffs_base__token__value_extension(const wuffs_base__token* t) {\n return (~(int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT;\n}\n\n// wuffs_base__token__value_major returns a negative value if the token was not\n// a simple token.\nstatic inline int64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT;\n}\n\n// wuffs_base__token__value_base_category returns a negative value if the token\n// was not a simple token.\nstatic inline int64_t //\nwuffs_base__token__value_base_cat" + - "egory(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0x1FFFFFF;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) & 0x1FFFFF;\n}\n\nstatic inline bool //\nwuffs_base__token__continued(const wuffs_base__token* t) {\n return t->repr & 0x10000;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) & 0xFFFF;\n}\n\n#ifdef __cplusplus\n\ninline int64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_extension() const {\n return wuffs_base__token__value_extension(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_major() const {\n retu" + - "rn wuffs_base__token__value_major(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline bool //\nwuffs_base__token::continued() const {\n return wuffs_base__token__continued(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" + + "egory(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0x1FFFFFF;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) & 0x1FFFFF;\n}\n\nstatic inline int64_t //\nwuffs_base__token__value_base_detail__sign_extended(\n const wuffs_base__token* t) {\n // The VBD is 21 bits in the middle of t->repr. Left shift the high (64 - 21\n // - ETC__SHIFT) bits off, then right shift (sign-extending) back down.\n uint64_t u = t->repr << (43 - WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT);\n return ((int64_t)u) >> 43;\n}\n\nstatic inline bool //\nwuffs_base__token__continued(const wuffs_base__token* t) {\n return t->repr & 0x10000;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token*" + + " t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) & 0xFFFF;\n}\n\n#ifdef __cplusplus\n\ninline int64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_extension() const {\n return wuffs_base__token__value_extension(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_base_detail__sign_extended() const {\n return wuffs_base__token__value_base_detail__sign_extended(this);\n}\n\ninline bool //\nwuffs_base__token::continued() const {\n return wuffs_base__token__continued" + + "(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" + "" + "// --------\n\ntypedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;\n\nstatic inline wuffs_base__slice_token //\nwuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {\n wuffs_base__slice_token ret;\n ret.ptr = ptr;\n ret.len = len;\n return ret;\n}\n\n" + "" + diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index 50434f6a..2ccc0c5e 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -81,7 +81,8 @@ var Consts = [...]struct { {t.IDU32, "3", "TOKEN__VBC__UNICODE_CODE_POINT"}, {t.IDU32, "4", "TOKEN__VBC__LITERAL"}, {t.IDU32, "5", "TOKEN__VBC__NUMBER"}, - {t.IDU32, "6", "TOKEN__VBC__INLINE_INTEGER"}, + {t.IDU32, "6", "TOKEN__VBC__INLINE_INTEGER_SIGNED"}, + {t.IDU32, "7", "TOKEN__VBC__INLINE_INTEGER_UNSIGNED"}, // ---- diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index f19d351e..fcbb5bd6 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -2099,6 +2099,7 @@ typedef struct { inline int64_t value_base_category() const; inline uint64_t value_minor() const; inline uint64_t value_base_detail() const; + inline int64_t value_base_detail__sign_extended() const; inline bool continued() const; inline uint64_t length() const; #endif // __cplusplus @@ -2133,7 +2134,8 @@ wuffs_base__make_token(uint64_t repr) { #define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3 #define WUFFS_BASE__TOKEN__VBC__LITERAL 4 #define WUFFS_BASE__TOKEN__VBC__NUMBER 5 -#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER 6 +#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED 6 +#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED 7 // -------- @@ -2275,6 +2277,15 @@ wuffs_base__token__value_base_detail(const wuffs_base__token* t) { return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) & 0x1FFFFF; } +static inline int64_t // +wuffs_base__token__value_base_detail__sign_extended( + const wuffs_base__token* t) { + // The VBD is 21 bits in the middle of t->repr. Left shift the high (64 - 21 + // - ETC__SHIFT) bits off, then right shift (sign-extending) back down. + uint64_t u = t->repr << (43 - WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT); + return ((int64_t)u) >> 43; +} + static inline bool // wuffs_base__token__continued(const wuffs_base__token* t) { return t->repr & 0x10000; @@ -2317,6 +2328,11 @@ wuffs_base__token::value_base_detail() const { return wuffs_base__token__value_base_detail(this); } +inline int64_t // +wuffs_base__token::value_base_detail__sign_extended() const { + return wuffs_base__token__value_base_detail__sign_extended(this); +} + inline bool // wuffs_base__token::continued() const { return wuffs_base__token__continued(this); @@ -11160,8 +11176,8 @@ wuffs_base__private_implementation__high_prec_dec__round_just_enough( // // On failure, it returns a negative value. // -// The algorithm is based on an original idea by Michael Eisel and refined by -// Daniel Lemire. See +// The algorithm is based on an original idea by Michael Eisel that was refined +// by Daniel Lemire. See // https://lemire.me/blog/2020/03/10/fast-float-parsing-in-practice/ // // Preconditions: @@ -16521,7 +16537,7 @@ wuffs_cbor__decoder__decode_tokens( if (v_c_major == 0) { if (v_c_minor < 24) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)((12582912 | ((uint32_t)(v_c_minor))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((14680064 | ((uint32_t)(v_c_minor))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__goto_parsed_a_leaf_value__break; } else if (v_c_minor < 28) { diff --git a/script/print-json-token-debug-format.c b/script/print-json-token-debug-format.c index 7c306e87..9ae24f23 100644 --- a/script/print-json-token-debug-format.c +++ b/script/print-json-token-debug-format.c @@ -225,22 +225,22 @@ parse_flags(int argc, char** argv) { } const char* g_vbc_names[16] = { - "0:Filler..........", // - "1:Structure.......", // - "2:String..........", // - "3:UnicodeCodePoint", // - "4:Literal.........", // - "5:Number..........", // - "6:Inline_Integer..", // - "7:Reserved........", // - "8:Reserved........", // - "9:Reserved........", // - "A:Reserved........", // - "B:Reserved........", // - "C:Reserved........", // - "D:Reserved........", // - "E:Reserved........", // - "F:Reserved........", // + "0:Filler...........", // + "1:Structure........", // + "2:String...........", // + "3:UnicodeCodePoint.", // + "4:Literal..........", // + "5:Number...........", // + "6:InlineIntSigned..", // + "7:InlineIntUnsigned", // + "8:Reserved.........", // + "9:Reserved.........", // + "A:Reserved.........", // + "B:Reserved.........", // + "C:Reserved.........", // + "D:Reserved.........", // + "E:Reserved.........", // + "F:Reserved.........", // }; const int g_base38_decode[38] = { @@ -357,8 +357,7 @@ main1(int argc, char** argv) { printf("vmajor=0x%06" PRIX32 ":%s vminor=0x%06" PRIX32 "\n", vmajor, vmajor_name, vminor); } else if (vmajor == 0) { - printf("vbc=%s. vbd=0x%06" PRIX32 "\n", g_vbc_names[vbc & 15], - vbd); + printf("vbc=%s vbd=0x%06" PRIX32 "\n", g_vbc_names[vbc & 15], vbd); } else { printf("extended... vextension=0x%012" PRIX64 "\n", wuffs_base__token__value_extension(t)); diff --git a/std/cbor/decode_cbor.wuffs b/std/cbor/decode_cbor.wuffs index 8f8659d0..1766c742 100644 --- a/std/cbor/decode_cbor.wuffs +++ b/std/cbor/decode_cbor.wuffs @@ -161,7 +161,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if c_minor < 0x18 { args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__INLINE_INTEGER << 21) | + value_minor: (base.TOKEN__VBC__INLINE_INTEGER_UNSIGNED << 21) | (c_minor as base.u32), continued: 0, length: 1) @@ -184,7 +184,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if c_minor < 0x18 { args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__INLINE_INTEGER << 21) | + value_minor: (base.TOKEN__VBC__INLINE_INTEGER_SIGNED << 21) | (0x1F_FFFF - (c_minor as base.u32)), continued: 0, length: 1)