From 4284369bb85c63049dc0154154bdf8de3eb2d20e Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Mon, 17 Aug 2020 16:48:30 +1000 Subject: [PATCH] Add TOKEN__VBD__FILLER__PUNCTUATION --- internal/cgen/base/token-public.h | 3 ++- internal/cgen/data/data.go | 2 +- lang/builtin/builtin.go | 3 ++- release/c/wuffs-unsupported-snapshot.c | 13 +++++++------ std/json/decode_json.wuffs | 10 ++++++---- .../data/australian-abc-local-stations.tokens | Bin 67792 -> 89248 bytes test/data/json-quirks.tokens | Bin 976 -> 1152 bytes test/data/json-things.unformatted.tokens | Bin 480 -> 624 bytes 8 files changed, 18 insertions(+), 13 deletions(-) diff --git a/internal/cgen/base/token-public.h b/internal/cgen/base/token-public.h index 8eeb80dc..87c84f5d 100644 --- a/internal/cgen/base/token-public.h +++ b/internal/cgen/base/token-public.h @@ -71,8 +71,9 @@ wuffs_base__make_token(uint64_t repr) { // -------- -#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001 +#define WUFFS_BASE__TOKEN__VBD__FILLER__PUNCTUATION 0x00001 #define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002 +#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00004 // -------- diff --git a/internal/cgen/data/data.go b/internal/cgen/data/data.go index 4bc5dc63..c7eb412c 100644 --- a/internal/cgen/data/data.go +++ b/internal/cgen/data/data.go @@ -315,7 +315,7 @@ const BaseTokenPublicH = "" + "" + "// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__STRING 2\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3\n#define WUFFS_BASE__TOKEN__VBC__LITERAL 4\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 5\n#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED 6\n#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED 7\n\n" + "" + - "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002\n\n" + + "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__FILLER__PUNCTUATION 0x00001\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00004\n\n" + "" + "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000\n\n" + "" + diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index ed29f3ea..a497daf4 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -90,8 +90,9 @@ var Consts = [...]struct { // ---- - {t.IDU32, "0x00001", "TOKEN__VBD__FILLER__COMMENT_LINE"}, + {t.IDU32, "0x00001", "TOKEN__VBD__FILLER__PUNCTUATION"}, {t.IDU32, "0x00002", "TOKEN__VBD__FILLER__COMMENT_BLOCK"}, + {t.IDU32, "0x00004", "TOKEN__VBD__FILLER__COMMENT_LINE"}, // ---- diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 04f69321..908cebd0 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -2243,8 +2243,9 @@ wuffs_base__make_token(uint64_t repr) { // -------- -#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001 +#define WUFFS_BASE__TOKEN__VBD__FILLER__PUNCTUATION 0x00001 #define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002 +#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00004 // -------- @@ -26618,7 +26619,7 @@ wuffs_json__decoder__decode_tokens( } else if (v_class == 2) { (iop_a_src += 1, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); if (0 == (v_expect & (((uint32_t)(1)) << 8))) { if (self->private_impl.f_quirks[13]) { @@ -26637,7 +26638,7 @@ wuffs_json__decoder__decode_tokens( } else if (v_class == 3) { (iop_a_src += 1, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_expect = 7858; goto label__outer__continue; @@ -27372,7 +27373,7 @@ wuffs_json__decoder__decode_comment( if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { if (v_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); } @@ -27389,7 +27390,7 @@ wuffs_json__decoder__decode_comment( if (v_c == 10) { (iop_a_src += 1, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)((v_length + 1))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); status = wuffs_base__make_status(NULL); goto ok; @@ -27397,7 +27398,7 @@ wuffs_json__decoder__decode_comment( (iop_a_src += 1, wuffs_base__make_empty_struct()); if (v_length >= 65533) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)((v_length + 1))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_length = 0; diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index cd601c55..8c3bd7e7 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -899,10 +899,11 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor } else if class == CLASS_COMMA { args.src.skip_u32_fast!(actual: 1, worst_case: 1) - // The ',' is filler. + // The ',' is punctuation (filler). args.dst.write_simple_token_fast!( value_major: 0, - value_minor: 0, + value_minor: (base.TOKEN__VBC__FILLER << 21) | + base.TOKEN__VBD__FILLER__PUNCTUATION, continued: 0, length: 1) // What's valid after a comma depends on whether or not we're in an @@ -924,10 +925,11 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor } else if class == CLASS_COLON { args.src.skip_u32_fast!(actual: 1, worst_case: 1) - // The ':' is filler. + // The ':' is punctuation (filler). args.dst.write_simple_token_fast!( value_major: 0, - value_minor: 0, + value_minor: (base.TOKEN__VBC__FILLER << 21) | + base.TOKEN__VBD__FILLER__PUNCTUATION, continued: 0, length: 1) expect = EXPECT_VALUE diff --git a/test/data/australian-abc-local-stations.tokens b/test/data/australian-abc-local-stations.tokens index 8eba91c0d1042003549f2136021dcc920c0d0ff6..d17a0c3b8103022449f3de7be6feb5253f3b58c4 100644 GIT binary patch delta 24094 zcmcaGk!8V7)(OtM>C?Xpeoq6j~L_RPY^R}aiC@>lGPDK$3U{dB;h$J#OpUH%G9g+a! zC?X4(jrbV;qX=DKHs;kr z7Ma9i%xjG-Fxh~`nAa0UB!R`4HyTA`0*f(k6N<_>H61XuE9k(-YGK!iD+{V1w$Rd+?jCqkGi*d36k1|gyl9I{(JSM!8kOUYfH}Dwq zEvnaY6c#V1QqKItZHRk0+5qZFC%qxW=BEVBg(6bGXUv!F-Rl?ljCp4xtC_@a%)0?uV6p*!Fz*`_kp_Ne-Y+O3 z2l$P7|DlL{;5X)FMiEgEFy=Kx5eX15=Cwf)X%KKm%c2JajCp-f)O-*y=8ZxTQ4mz- zNkI~sY%gen=JNtUW3+;KfgpO~z94AKI~mCs#z{iXsDU%tK?pr`3WSV#42%{zc$qB;7yl%*vCQlGXPyYvmjd_bu)O--esBT1p(F)W8 z5ofdvxXulP(&J}jd>$cL^enp^QNJQJdk!qi%)h&+%n=6#DIA|PwT$A#L+aF8|TRYg%$AZv`;Y@ED6)|fXC zMa>0SG;d6j3r4Fs0_2?0s`&|W#=Kilj5#1@%)1js>%BZDL|nP0&KHHs%YD4@5G3lxlb9gs9JPF|p3%3ti%e2N&!PrOXoc?N z1SMnCj>6;#O2)i8$i_@Qpk&PJh$8YqDVVnbMZ`hb8Lbd$P&P(uFKkdY=ADkB>wz*_ zdwsHiiZSm66g3Vi&b&8JL>g3#c^{yNY)~=geTO3QKt-7cIo2otS202JsDY|6uQ-aP z1XW{RITVozs>Zz9C?W?`oq3IsMJB1CN3nsLF>frglF12b=susIX3V<+Ma=;CAf>#h3z3W8T{+A`3K)d0(Q4T+meJL2gY=ey?eQR#Gcyq378E zE%XR&&@$##K{0rPmNQ!F{-9;d>x`mCLED%&7)2yN+nBcsMWjL7n7126WP>(Zt8wxJ zZDZcGC~5?B&`T!=9b?`B0vcS|nl1oIZ7s8P^$MvKt|U1MJ420Y{B z3A)C-T`0N^=wg(py2iXmP}C^sIisb*1U>X}WrChD?;{jl2lSMAkZY64_w`KBieUkL zV_q&4O%D3#9o7PU^c=ZB-x+n>V)6rh^m16h0KF&fV1PbOQDA`HrdVKL%-e_Jo(l$O zrS2p{W8RY}N(>B*(UNz9p)v0d6g3kJjd^)dL=G4Rqcz|KjGTExP}Bq%8S}=Wh%^`( zqxIu97@_CL2S(@}XaQqq-lZt!1Q?^I%LZd(wC2NV+DUH#R|S?@cm6Z|)hG z81ueH(!)48!2}~LOq|j3@dXpKj`<{0j83kpF|Qx0!KUZ~5ED$5d6Q7oY%n$EEkO}^ zU~0@e2Sr4{%$Ro>iim?5`dC7N8G2J`fmtwWzkKosGiS7Rk%2jSyC}ijnAZfwt_kMo z5qiKJqrGm9(O$PO=B-09C&2=J3~7ReG7oYKe)4(?6SP$Dz`~e!D~d4!md3n$P(&Oo z(bIc@r8DnE6g3+x(bK^LOJlUIuz-~@uOx~t2P?Gkw8;fl#=K@IY8F@-^Ln6&T(Ck5 zo=MimycH-)46M=nE(zA?$$f%#Fj}+mfweQ*7@&fUG4D?llLKtf`|}Mp#=N>HYBtyy z^ID*YJg{*_YZEHiqI)aA)|fX8MOTBZG7oZUo;=?cz4v~>7OnR_$qv2qZeWMr5lFCe zM(e#Vuruaejbir&JG8=Jl0AA18`vB3enrugU~kOJh9WY--k4VcMdW}zdWrVI9=)rl z;DBDQ1~{MGGxf}Xm!UQ*CURM;$54bs_cEBdN8>98V z4BXL6;RJVO9^^s7$@T6gXi>Al9lgkX;EtZz1U%4Fu7ig&TC2ao13mgSc%Vn$0}u56 zs(>f@D3gPyF|Q(u&kH<_Q9CM=7kHwNFI@0M^ZFz&^hTqBmoe`w6oV7If_bl@h-~n3 z=Dmj^^1;iP_XUcGf;al`LV&k1FLD=~adLyVF|Rs`t_|MKXrl`syp4IyQPe2-phtRu zk1`MP2*l)k9}~2;)B+#0nq=|?AG8cP$rpXD#=zGZtpQ!&iyk@)e9`ACF8HE#X(#y^ z^FBoJf`Okg?*|l-1V8jRpWug{9S``SkA{BmGv>`iF-O54y|EDBk3Mv^z#o0+?14Y} z(3wC0+R)i#hX7+gq~e88V_tq__e=^yAEh@4Q|3V)4V~;CW`b6UHiV%k-3?*r!zd5J&|_2} z+!?Kn9}sSg))8(9M;{K|5N^!72HC}vAA}q8?nMz1h%n|ok0Rm_fu2bVBG5~x1rg}O z_7@`1YL`io#=Kc51{*{Mqt^kE&S=w12O=@Xf+CH1_oJAj5QRSG6%d8qWNnCYM$4lI zqKwhz(LO|>kD4e%EAt>vj83+XHsLiuvAiG}W27@0eWvt6v@vRzc2bNpTHcyABPHvRYn^EFNifp8(3Tri$0ikA=a2z6xrvK z;)2oU!UN)*(dNP@#G#kM2jb9cn-6j5{d9$R^bx^;c=U8JAs)T8a3J28_ZYGRCx3`n z=0RTaFquEW1Z}9_ApyN{T#$fXel18qpR2r(;EXmwCy9NV+Ef zPch+rh9tl^*&r1?wh~g&2lgkVqK~c}NOeYC0Wm2JeKN-&4Sl3BAq~CHHX#jt2K_*q zG1}PUhcxuDV}*2Mv_%^M>FC{phIC`zJ18FAkZ#QT5k=%ddNA66yg>%qfc)fw3}as8 zxl_i;3o?w+X5TJkpjE4rGSPcq2AR&hEhv^3WTH3v7i6Nh|1M-I^B|8XO@5zg!n+y8 z7=^g*YFEcEogAMhmZoZ1i^IhHUiVwFlY8 zya_1g2;`v8p*!Rlqjg0Ka?oe&7UZChA6>`^MjIzk$aUuBL$NC%7k%VlLas5|fZKsw z^gj28Tw}D7Ss@R-WKPIKZxT+(L(ka<@|1ZRk=!%+ex3>1xUfLJG4DJiC5)3D^3f}) zf_(G@zaZb47j=T?K|Xr43lyL)F>oj_;?qKHlob@9&wMN>Ko6e_1!#@UNrmVm;0A?g zrS9Z}LSx>UDE^vIh`tu>Kw&W2Ag@4?Gw){(YtrBOo>dUs|+5qjI{K@obw6evbt z`4Ui!9_)|67&iup#;4jFrma5ZEf0x610)U zNu}rwOM_DMxypo6^fGNiDSETxKq=ao_~Z|z=v@VcGW2Xlr#G%F$D0K{t6?#Kt zLzOXFj(AXozGOt88oly$s74e1V24)y5s9tHL2ZRiE{=v{*g^=QrENe$>@ss;_{ z^E(L*=<|>h8iLVAkRCKRqb-wAXhff-3TQOuMP3WdIJuz_ZA@wMhDP*R^#_g4ypK_W zS)mC%jsu!7x{OWAJgi8%CeLp&L2WipzR-j=K{u%xZTZt=gJ$%}oP=g)w6)_4n$c%H zE;OSRV3S(VXRHlc&}Z2bTF?sN$rD=8TiFL%(3c;6XhCm6E3_K($|HH4adJQ_dW1H# z2BR%*ztD=lxLu$PeMsD)4Siy{pbfpASkQ((>U*Kh8EqwuKs$P>bZ9q5>lzfaEAt?) zf12FiZo-SY3iUv{FQDLHf*qm4fnbOoa=PCL-$jJoP> zQa9S5_+*1_^nxa#8)HFFH~M1e1Kns#+9vg&Hwg`T(1z(IC-f-uoIwiI$@M)ZyvW@_ z#>pFc&{xYm=s_Po7U)H9fjRU#^CIVK#>ox6=yANE7rkJ9&}+<#JdwpXS)dPnaKoVw zeF1erANnNTf<9xkPSAxuv@9^GAH5ni=ts|^3H`xnQ|KG|oza#{f9OZAITR)s^LitN zDdXgT2^i~nCZG=lZkT4m2E60S%=tb;- z$;N0y8y_a4FWFa^g1*=(U`jAr#lK*RGui_E2UF130}D(=ceBG(^lH9fD*CAJf~n4E z3$z|g#h9R*hF-5aOjG7TUa~bgewqo|(Bgz?=zZD))6gs957RILV>r#X2lO8!Vb! zu*4Z{mVLt#^w!>kCFnDD0!x*7kT)<))?aFZHnyIy6n)TX!cz2wmD=!=sTRvYv7qeM->YGbr4+OQgZ>}A7h zj9$QMW3)c5z#8=Q?yv^ExGGqKz94?V8ua?-!Wy)J;Yn+Q(e|SRtaV0f+D%xCK9X=? zE&4>{hqcCN3!xR(8KX7L1J*gCtr(uL4!w(ZU>*AEwGZo*d60K-Pv&25g4Q^8SdU%^ z7pzB5$_v(GgvEOFx>#TXdIQ>F1A3+{*nnOEE!cqG-MO#L2qMB*n(cZAJ~Gn>0#1V^bBCI6}^v? zuob=QK4B|**Zsg&^bOn}wxZ8sD{MpGdls+_eH^1<8+xADunldQ$>ayyg3)?$2HTy{ z_GuJs#~3i(Zj82Mhb~vL=?iTDoA5&eh1AU3cg&oQ~$W!r?-|sL% z+Z&*;6TP?!*oj_8HS9$17i`$+#OH=Ou<>Ch`oM<5F7#Lr*oD3#qhS|%Z+^or^a-&C zyU=^{0=v_#71ys$eMZ55Eh9`rH6ggxkO-wAurTNwxTpwHBP z*kg=-p2S{fwE87sFIt;$@`SzUb9M*zD)VeWs+A_+-)n-tP-q|ee7(ay^xZH8`_M-+ z7VL9I+u-zIANowKz<%^7b=Z$yh865bpZZv^AEU6|kG7^}(gC#PGLsDspqIl52hihh z!U6R7J8&QvtxYI!&>3w6FyJ6YQ|Tc3^j717W9Vhbg=1(=o^;$9^>m5J4#&}(gayaZr(hQxSLQ)(5>DQK+yrg6&WGdZ zrLe*Y^l`<26XilNTnhKVyP+z{G1a zkIII#=*w9joJBvbLEs!(9-ZuP4!yHia1MQ5cELIHJv|rBp*1Qdoky=U4bG#_`6Qf2 zUm`N$d@$OPG!M=@^P=vWRJedXPZe+heYHTt1@uLC8!n))@p*6oeNB|YMf4V5z(tIr z`l2!q^49dp^Dmm9H=Zt{S1y;(n+pb)(6=EaTyjQRK(pWydMok5CA0)T=`wnaWN;b1 zXP0mpeZX|WW%Qo#fy?Oi&xgzCNm=0vdX5aZf<9KO86V#cl$p@|(qwNp) za1Cuxb+W>B^y)d_x-EA%F! z!7H?RcmG%D^X?6=(C6JZyh59IpZwqz`oz1yYxIftfY<0bpy4(8faHeP=nLr|yhaP0 z$pUZCS1mfcLC@F)Z_q2+1#i&3e&G$;(xFLjF~(5e2BUAaeCv#MGRlFs=>63XZ_#I; z72ctbrUkr1Uo6n@&KYfE^MQBhEs78C(C0=J-YfGUpA9tG{=EtM7~Fe|<>l|u6ZeJp z=wa~zeTK#11A1yN_<-K9T<`(Ck#^w&+6eQckLWRM@DaUPlJF6&wLE#kNAxo8z(@4> z{O}RITvqrLjCK}5!6)>MT^l~3r@;rG(8r1eKBEsnID9tdMLkun;IlK@v1A)QqfgU5 z_>8_DO5lq!5AsI(fnu^dX%CU(ko-K74UTJ446dEBXmF3187i z;3j;JbTNq>UT7CQy}aYh|ioIK$V`W(=KKj`xwAO4`Ptx))j-hK%9 z>x_01-GsmByI~IeMQ>7n_^Zr=yrOzC|34G7ky(d-=xZ|z{-LkGUGNWWByaMCf9OMb z0{_v+Y90QgXXAqZ=<@>${-gJ-F8oIuB%i{FwrGB;0V7&-d1?Y9+UlpN6ByB=Z|VU? zwCJ1qfiW0$4cBxBMn-4Ub^X&D7#YzTiPJZLgisEnoBn{2(TFbzWzcQ9023qHR=epA zOpIuI?WQ+?RH2+#JADI42<5=q=?|C~m3fd437Y<&iO~f0^u6f@%#3Km#?uo(LMRJ0 zr%wP0p`1A~{QxtgGwLbL(^$|$Yq|kQ2<1fT=?Neql-gXAp=9oLo_WYYwxGA zGolsm(+xmECNz^oFMw2`ELESz!H9Nf<8%WKMrYKuveOGV G7!v>}#|L%* delta 5748 zcmZ3mll8(xmI=-igAFE5Hkf!hV6riz!Q^5_gUOp24JLnPG?;A6v<}{f6nbTmhF_*#QVlIQpo4Fh&GjkhEcIGyi+{|q-`7pP^WM&?N$<90mlbd-A zCLiW8n9R&;Fxi>cU~)5Wz~slg4wH@f3?>)z8BE^HXE6CQpTT5feuv4;{05T`^BYWN z7EqXMEMPFXSioTNW&wlAp9Ksi8w)y2ZWc6{d|1$6GP97uWM?6R$<0CrlMf3SOlB4~ znCvWUFu7USVDe#MgUQSy0h5zO942oTF_`>W#9*?qsKMl7QG>~wMI9zHiy2II7BiUK zET%AdvzWo;&te9Xjl~Tn7mFKA-Yo7gnOVYMva^K2zunA|LB zF!`{g!DMDBgUQZP29ukm0wzC}a+qu^Z7{i5+F$;_$-lbux!CO4}ZOg^k?Fqv7+V6wBC{^TY# z!^wx$3??(H8%%aqH<;Y49x(Z_y2E5+4TH(W8U~X$YZy%atYI+OSkqy0v!=o1!tVDe!jgUQUs29uqQ4JJ1m z8%#cIY%rPG#9*?siNWM%lYq&OO&lg0n;J|mHZ_>M+0h6 zWiZ*<%3yM{mBHl0RtA%qtpg?}TRTkNY;7?4v$erwV;h6X#Wn_$H`_Q&X0|n$>}+c= zx!G1>@@89u$)9ZvCL7xsOfI%Fn7rA}VKTG5!DMH9gUQYI29pol8%$<)FqrJ@U@*Da z!C>-X2ZPDXjs}yR9StToI|fXC?C3Dr*vVjWv6I2%%}xfBKRX#rHg~1jmvwOhgU=N4Mi#-e`KlU(~tn6tpIoZ=-@?uYi$)7z9CL4Ph zOfL3Pn7r7_VDe)xgUQO?29uM$4JI%4c9{Ix+hDS>kHO?(AA`x8eGDdl_A!`j>}xQ& z*wW-!?} z++cEXxWeSc;Rcf*hZ{^*jxd;<9APkdafHL<&k+WbjUx>v7e^XQ-W+K#`E#VfWaB7< z$;D9ylQ%~hO#U2YFxfcTU~+MEz~sx(4wIE*3??VX7))LqV=(z~jKO5(Scl2Qu?CYj z#~Muj9IG%{InH2ma-6~B#c>9cAIBL?R*rX=TpVvOd2_tM_= z8%!=vH<-LR-C*+Pbc4yp83vP!GXf@G&TyEloM|vQIn!YB;!K0dk24J>D`z=OF3vKT zygAEY^5-mt$;#OVlasRzCNIu5nEW`~V6t+K!{p)|gUOq73?_fhF_>(eYcRPu*I@GI zT!YDAe zxWr&`af!j?%_RntKbIIxHZC=oTwEG3`EselWaTo0$;o8~lNXm6OnzKuFj=|WVRCV~ z!Q{>529rORD@<0dFqoWNVK8}dg~8;<6$X=)D;*{mR~k&-Txl@*bEUy#<0^y6#Z?NE z7grffeq5z6S-9F@a&Wc59VTC{ zHJB`1XD~UqPGNF$ox$Y8bq14}>kTG5*BeZ3u6LMxx!z#1aD&0*;0A-qlN$^sUv4m% zEZk@?Ik?ea^5jN?$(I`qCJQ$iOb%`en7p~kVKQ^G!DQ!VgUQX!29pmr8%$iXWiZ*f%V2VIm&4@C zT?UhdyA37>cPmV8?lzcwxZ7YdbC1Df=N^N}%{>m2FZUQs7Vb5e9NcR#d2+A8XCI|N!OrG3lF!^$y!DQingUP}D0h2fPJ4|MtU@+Nvg2CkG2?mo7PcWFwJkeot z@I-^jlP4NXzC2N3GV>&Z$&n`anIK0L!$<8wkCO6M?n0$Gr!DQiC29twl8BCr$ z%V6^5Sq77ZXB$ioo^3FB@@#|2muDMH7M^1;Ie1RMJ!Fqk}fg~8;@D-0$JuQZq(yfR?&=9LbUnO7N1c3x#L zxp|erj&?VDjYE29qzZR+!AZ#$dAZ8iUErYYZkIUSlwsd9B0b;I#&m zC$BY_e0igc#Fa0;4KD|CvP#Be0ht(WZ|s_lY_S!OrE^eVDjay`jdsW8BPw~7BG49HiyZ~ z+YKf=Z#S6Syxm~(;q3;KnRhr$4&GrfdGZc}$(MI1OlIC`Fxh#h!Q|$h29po(G?>i1 z%VBcxE`!OFcNt8+yvty+@NR?2!MhD6Pu^`X`SNaq$-;XKCI{~^m^^up!Q{((0wx>p zb(q||*I@GDy#|w+_Zduf-e)kmd7s1N%lix_3-33W9K2s)a`S$J$%pqFOlCe{FxmNl z!Q|!x4wEk*FqkZS&|q@#L4(PY4;oCqe9&OB@F9cA!G{bcPd;QY`SKxy$-;*XCI=r5 zn7sM0!(`?o29uqS7))+HVlese5rfIhM;#^yA2paf`KZC<%SRO^GaoaU?0n2%a`Q2R z$%l^_OlCgrFgf_R!Q{!u4JKbcZZKK+gu&$C69$tfpD>tw`Gmn_;gbfFgHIYvo_x|^ z^5v5OlZ{U~Om049F!}H)gUQUN4JJFEHkjOe+F|nL(*~1;&lpS&KBF+X`HaEj!)FX8 zGoLk>?0nW>a`Rb-$(PR>Ocp+8Fgf^~!Q{#33?^SbXE0g#yusw)^9GYApEsC%`Mkko z;R^jFB?pD zzHBhL`Le;}!zGX1E`If`v%eM?B3*R=F9DLhg^5okFlP}*km@It9U~=#sgUOTc z7)-u=$6&JXU4zNNcLOGGzUwfV`JTaK=X(Z|o9`J+K77w$GV^_h$-(yxCQrU^F!}O* zg~`kh3?@52Fqqu@z+m#>2L_XwA3979erPax@{H$QcleEF%tWZ`E9 zlY^frOm2Q=F!}H^gUQU#4JJE3H<;Y~++p(N=LVC7Ul>ddeqk_q@(Y8>mtPo67Jg|k zIryc)a_|R($&)`AOuqafV6yQ?hsn(!4JIG{XfT=ilfh)?PX?2lKRHak z{K;Ul@MnX`!Jic-H-9#meE74$Wacjhlbyd9Om6;SKl#!Z!^y&54JHSFHJCj4tHI>U zUkxS;e>0dI{LNtU z;xZef!gOYKkU9|23?dGL2xbnDIu4LJ5OEnq2y=pr2N9D&#AOg6%moqy5tBj0WiG~m z>BiiQ_S2iV84af&<_7tQ2V^{mXyyU=2*eWR1!?1DRG8ijVjTt%%zPkqAflNMqz=Rq z<_D<*5tBj0We_1O01^WclR?B~5FsoG5)%aZToB}QA&^lZq8UUS1`*7{ARh^Xd;}sc z3xj+v0#XMennA>25Wy@8QYQ*h2O=(m2w^dh@gQO{h`08`L4>dzNK6job2*UD{Hk4n#C-fYgCl!kQp;AYw9zxC|nMwLoGZVls%h z3?hWJL1NkOjP05Fu;;G9E-s1`(G*gs>q<3`9%@5tj`?J~smS+z8}z5W#E=G9E-U8-siVVhNjo zw3&c>4q_b!5zM9_bs(bI6r>Ks5;g;=0}+!!#AOg6Yz`6w5tBj0We_230TQzS`P>5J zb4!p>Afg#W90n20Rv;f)fqVoaE?a?oZVgfgBAP+OVGzM=15#%LQU@X~g9u?;kntd5 zGKjbgB82TgVjyBNh`4MA^0__8=k_3mipMzM3K?JiuNF9i1_6Mm0v4jIa z>OjO~5OEnq2nT}1K*VGaaT!Di2Z6+bKt2xw`8*h86o_aJX4Id4D45Z3I&%ofMd=h?oo_E`tc+2#^?vm<%E=M=&N# zcaCH<;N8W*!~h1Y3@psk7eq1|^PWW(nij=q%=-{oWO_mrqw(~`QH%!DAA<H^N&mgXI3}Y++B#)Ve diff --git a/test/data/json-quirks.tokens b/test/data/json-quirks.tokens index 3a36b054b5b3dd196dccc93d33108963143247dd..d7b08ab435bbacda4053291aba14ac4cc3c4c259 100644 GIT binary patch delta 306 zcmcb>-oQCQ*_VrfkpTukEM5j?2$z+Cp@D&cK>$r&n1K^6ufV{-AcHKw(LtG!*9Tc- zasp$rY&^0E0|P@MgBHXL1_lO}$rl)ddFLXln5@85Ec**tgn@zK4}&<8o&`+7wyLNq z7}XgB5h@rM7%gG)j0_4+3=E7`Fg^nV1LI@?=44(^6p;evVA*J75e5dvSQwv?K|l!r DC7%<_ delta 136 zcmZqRyudy|S(cZ9nE?z~85kNE7#M^ZIKf;7Rt5%zjRne#lNU24@Fp^7K|~oC7#Jrj zFa=L;W-8$Q!yt|%D!?2ptIi;Z5Mf|ov}9m}@EI8toF*?|PM$2x62Kda5EW1Y0Be~K AO8@`> diff --git a/test/data/json-things.unformatted.tokens b/test/data/json-things.unformatted.tokens index 1fd7fb8609e788be94a89f8e5f1898de1aa32701..447662de3da0d95e0b28f3fbfd1b06c0f8b11a01 100644 GIT binary patch delta 190 zcmaFB{DEbHGcPLxBLfUDPIOS_x6-N=+V5}p9EWp6PAkV-FF_M*m t1!TG=lKjN?F1i*-e2}~)0}Dhw3j+fa0|SFCvixKRMq^$V6p;eP0s!OH4gCNB delta 57 zcmeys@_>1Q^Tc3