Simplify tokenization of multi-byte squiggles.

This commit is contained in:
Nigel Tao committed 2017-05-21 10:56:21 +10:00
1 parent d5d6447ce5
commit 7df75101cd
2 files changed
+8 -26

No files matched your search

+8 -10
View File
@@ -207,16 +207,14 @@ loop:
if id := squiggles[c]; id != 0 {
i++
if id.Flags() != 0 {
tokens = append(tokens, Token{id, line})
continue
}
for _, x := range lexers[c] {
if hasPrefix(src[i:], x.suffix) {
i += len(x.suffix)
tokens = append(tokens, Token{x.id, line})
continue loop
}
tokens = append(tokens, Token{id, line})
continue
}
for _, x := range lexers[c] {
if hasPrefix(src[i+1:], x.suffix) {
i += len(x.suffix) + 1
tokens = append(tokens, Token{x.id, line})
continue loop
}
}