mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-22 16:47:28 +01:00
Simplify tokenization of multi-byte squiggles.
This commit is contained in:
1 parent
d5d6447ce5
commit
7df75101cd
2 files changed
+8
-26
No files matched your search
@@ -498,22 +498,6 @@ var squiggles = [256]ID{
|
|||||||
'?': IDQuestion,
|
'?': IDQuestion,
|
||||||
':': IDColon,
|
':': IDColon,
|
||||||
';': IDSemicolon,
|
';': IDSemicolon,
|
||||||
|
|
||||||
// 1<<KeyShift is a non-zero ID with zero Flags. It is an invalid ID,
|
|
||||||
// signifying that lookup should continue in the lexers array below.
|
|
||||||
|
|
||||||
'.': 1 << KeyShift,
|
|
||||||
'!': 1 << KeyShift,
|
|
||||||
'&': 1 << KeyShift,
|
|
||||||
'|': 1 << KeyShift,
|
|
||||||
'^': 1 << KeyShift,
|
|
||||||
'+': 1 << KeyShift,
|
|
||||||
'-': 1 << KeyShift,
|
|
||||||
'*': 1 << KeyShift,
|
|
||||||
'/': 1 << KeyShift,
|
|
||||||
'=': 1 << KeyShift,
|
|
||||||
'<': 1 << KeyShift,
|
|
||||||
'>': 1 << KeyShift,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type suffixLexer struct {
|
type suffixLexer struct {
|
||||||
|
|||||||
+8
-10
@@ -207,16 +207,14 @@ loop:
|
|||||||
|
|
||||||
if id := squiggles[c]; id != 0 {
|
if id := squiggles[c]; id != 0 {
|
||||||
i++
|
i++
|
||||||
if id.Flags() != 0 {
|
tokens = append(tokens, Token{id, line})
|
||||||
tokens = append(tokens, Token{id, line})
|
continue
|
||||||
continue
|
}
|
||||||
}
|
for _, x := range lexers[c] {
|
||||||
for _, x := range lexers[c] {
|
if hasPrefix(src[i+1:], x.suffix) {
|
||||||
if hasPrefix(src[i:], x.suffix) {
|
i += len(x.suffix) + 1
|
||||||
i += len(x.suffix)
|
tokens = append(tokens, Token{x.id, line})
|
||||||
tokens = append(tokens, Token{x.id, line})
|
continue loop
|
||||||
continue loop
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user