mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-17 16:32:11 +01:00
Simplify tokenization of multi-byte squiggles.
This commit is contained in:
1 parent
d5d6447ce5
commit
7df75101cd
2 files changed
+8
-26
No files matched your search
@@ -498,22 +498,6 @@ var squiggles = [256]ID{
|
||||
'?': IDQuestion,
|
||||
':': IDColon,
|
||||
';': IDSemicolon,
|
||||
|
||||
// 1<<KeyShift is a non-zero ID with zero Flags. It is an invalid ID,
|
||||
// signifying that lookup should continue in the lexers array below.
|
||||
|
||||
'.': 1 << KeyShift,
|
||||
'!': 1 << KeyShift,
|
||||
'&': 1 << KeyShift,
|
||||
'|': 1 << KeyShift,
|
||||
'^': 1 << KeyShift,
|
||||
'+': 1 << KeyShift,
|
||||
'-': 1 << KeyShift,
|
||||
'*': 1 << KeyShift,
|
||||
'/': 1 << KeyShift,
|
||||
'=': 1 << KeyShift,
|
||||
'<': 1 << KeyShift,
|
||||
'>': 1 << KeyShift,
|
||||
}
|
||||
|
||||
type suffixLexer struct {
|
||||
|
||||
+8
-10
@@ -207,16 +207,14 @@ loop:
|
||||
|
||||
if id := squiggles[c]; id != 0 {
|
||||
i++
|
||||
if id.Flags() != 0 {
|
||||
tokens = append(tokens, Token{id, line})
|
||||
continue
|
||||
}
|
||||
for _, x := range lexers[c] {
|
||||
if hasPrefix(src[i:], x.suffix) {
|
||||
i += len(x.suffix)
|
||||
tokens = append(tokens, Token{x.id, line})
|
||||
continue loop
|
||||
}
|
||||
tokens = append(tokens, Token{id, line})
|
||||
continue
|
||||
}
|
||||
for _, x := range lexers[c] {
|
||||
if hasPrefix(src[i+1:], x.suffix) {
|
||||
i += len(x.suffix) + 1
|
||||
tokens = append(tokens, Token{x.id, line})
|
||||
continue loop
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user