mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-21 00:47:28 +01:00
Add tokens
This commit is contained in:
1 parent
7a142168be
commit
737e31ff3a
17 files changed
+498
-36
No files matched your search
@@ -0,0 +1,19 @@
|
||||
// After editing this file, run "go generate" in the parent directory.
|
||||
|
||||
// Copyright 2020 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// ---------------- Tokens
|
||||
|
||||
// ---------------- Tokens (Utility)
|
||||
@@ -0,0 +1,192 @@
|
||||
// After editing this file, run "go generate" in the parent directory.
|
||||
|
||||
// Copyright 2020 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// ---------------- Tokens
|
||||
|
||||
#define WUFFS_BASE__TOKEN__LENGTH_MASK 0xFFFF
|
||||
#define WUFFS_BASE__TOKEN__MINOR_VALUE_MASK 0xFFFFFF
|
||||
#define WUFFS_BASE__TOKEN__MAJOR_VALUE_MASK 0xFFFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MASK 0xFFFFFFFFFFFF
|
||||
|
||||
#define WUFFS_BASE__TOKEN__LENGTH_SHIFT 0
|
||||
#define WUFFS_BASE__TOKEN__MINOR_VALUE_SHIFT 16
|
||||
#define WUFFS_BASE__TOKEN__MAJOR_VALUE_SHIFT 40
|
||||
#define WUFFS_BASE__TOKEN__VALUE_SHIFT 16
|
||||
|
||||
typedef struct {
|
||||
// Bits 63 .. 40 (24 bits) is the major value.
|
||||
// Bits 39 .. 16 (24 bits) is the minor value.
|
||||
// Bits 15 .. 0 (16 bits) is the length.
|
||||
uint64_t repr;
|
||||
} wuffs_base__token;
|
||||
|
||||
static inline wuffs_base__token //
|
||||
wuffs_base__make_token(uint64_t repr) {
|
||||
wuffs_base__token ret;
|
||||
ret.repr = repr;
|
||||
return ret;
|
||||
}
|
||||
|
||||
typedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;
|
||||
|
||||
static inline wuffs_base__slice_token //
|
||||
wuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {
|
||||
wuffs_base__slice_token ret;
|
||||
ret.ptr = ptr;
|
||||
ret.len = len;
|
||||
return ret;
|
||||
}
|
||||
|
||||
// wuffs_base__token_buffer_meta is the metadata for a
|
||||
// wuffs_base__token_buffer's data.
|
||||
typedef struct {
|
||||
size_t wi; // Write index. Invariant: wi <= len.
|
||||
size_t ri; // Read index. Invariant: ri <= wi.
|
||||
uint64_t pos; // Position of the buffer start relative to the stream start.
|
||||
bool closed; // No further writes are expected.
|
||||
} wuffs_base__token_buffer_meta;
|
||||
|
||||
// wuffs_base__token_buffer is a 1-dimensional buffer (a pointer and length)
|
||||
// plus additional metadata.
|
||||
//
|
||||
// A value with all fields zero is a valid, empty buffer.
|
||||
typedef struct {
|
||||
wuffs_base__slice_token data;
|
||||
wuffs_base__token_buffer_meta meta;
|
||||
|
||||
#ifdef __cplusplus
|
||||
inline void compact();
|
||||
inline uint64_t reader_available() const;
|
||||
inline uint64_t reader_token_position() const;
|
||||
inline uint64_t writer_available() const;
|
||||
inline uint64_t writer_token_position() const;
|
||||
#endif // __cplusplus
|
||||
|
||||
} wuffs_base__token_buffer;
|
||||
|
||||
static inline wuffs_base__token_buffer //
|
||||
wuffs_base__make_token_buffer(wuffs_base__slice_token data,
|
||||
wuffs_base__token_buffer_meta meta) {
|
||||
wuffs_base__token_buffer ret;
|
||||
ret.data = data;
|
||||
ret.meta = meta;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline wuffs_base__token_buffer_meta //
|
||||
wuffs_base__make_token_buffer_meta(size_t wi,
|
||||
size_t ri,
|
||||
uint64_t pos,
|
||||
bool closed) {
|
||||
wuffs_base__token_buffer_meta ret;
|
||||
ret.wi = wi;
|
||||
ret.ri = ri;
|
||||
ret.pos = pos;
|
||||
ret.closed = closed;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline wuffs_base__token_buffer //
|
||||
wuffs_base__empty_token_buffer() {
|
||||
wuffs_base__token_buffer ret;
|
||||
ret.data.ptr = NULL;
|
||||
ret.data.len = 0;
|
||||
ret.meta.wi = 0;
|
||||
ret.meta.ri = 0;
|
||||
ret.meta.pos = 0;
|
||||
ret.meta.closed = false;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline wuffs_base__token_buffer_meta //
|
||||
wuffs_base__empty_token_buffer_meta() {
|
||||
wuffs_base__token_buffer_meta ret;
|
||||
ret.wi = 0;
|
||||
ret.ri = 0;
|
||||
ret.pos = 0;
|
||||
ret.closed = false;
|
||||
return ret;
|
||||
}
|
||||
|
||||
// wuffs_base__token_buffer__compact moves any written but unread tokens to the
|
||||
// start of the buffer.
|
||||
static inline void //
|
||||
wuffs_base__token_buffer__compact(wuffs_base__token_buffer* buf) {
|
||||
if (!buf || (buf->meta.ri == 0)) {
|
||||
return;
|
||||
}
|
||||
buf->meta.pos = wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri);
|
||||
size_t n = buf->meta.wi - buf->meta.ri;
|
||||
if (n != 0) {
|
||||
memmove(buf->data.ptr, buf->data.ptr + buf->meta.ri,
|
||||
n * sizeof(wuffs_base__token));
|
||||
}
|
||||
buf->meta.wi = n;
|
||||
buf->meta.ri = 0;
|
||||
}
|
||||
|
||||
static inline uint64_t //
|
||||
wuffs_base__token_buffer__reader_available(
|
||||
const wuffs_base__token_buffer* buf) {
|
||||
return buf ? buf->meta.wi - buf->meta.ri : 0;
|
||||
}
|
||||
|
||||
static inline uint64_t //
|
||||
wuffs_base__token_buffer__reader_token_position(
|
||||
const wuffs_base__token_buffer* buf) {
|
||||
return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri) : 0;
|
||||
}
|
||||
|
||||
static inline uint64_t //
|
||||
wuffs_base__token_buffer__writer_available(
|
||||
const wuffs_base__token_buffer* buf) {
|
||||
return buf ? buf->data.len - buf->meta.wi : 0;
|
||||
}
|
||||
|
||||
static inline uint64_t //
|
||||
wuffs_base__token_buffer__writer_token_position(
|
||||
const wuffs_base__token_buffer* buf) {
|
||||
return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.wi) : 0;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
inline void //
|
||||
wuffs_base__token_buffer::compact() {
|
||||
wuffs_base__token_buffer__compact(this);
|
||||
}
|
||||
|
||||
inline uint64_t //
|
||||
wuffs_base__token_buffer::reader_available() const {
|
||||
return wuffs_base__token_buffer__reader_available(this);
|
||||
}
|
||||
|
||||
inline uint64_t //
|
||||
wuffs_base__token_buffer::reader_token_position() const {
|
||||
return wuffs_base__token_buffer__reader_token_position(this);
|
||||
}
|
||||
|
||||
inline uint64_t //
|
||||
wuffs_base__token_buffer::writer_available() const {
|
||||
return wuffs_base__token_buffer__writer_available(this);
|
||||
}
|
||||
|
||||
inline uint64_t //
|
||||
wuffs_base__token_buffer::writer_token_position() const {
|
||||
return wuffs_base__token_buffer__writer_token_position(this);
|
||||
}
|
||||
|
||||
#endif // __cplusplus
|
||||
@@ -256,6 +256,8 @@ func insertBaseAllPrivateH(buf *buffer) error {
|
||||
buf.writeb('\n')
|
||||
buf.writes(baseIOPrivateH)
|
||||
buf.writeb('\n')
|
||||
buf.writes(baseTokenPrivateH)
|
||||
buf.writeb('\n')
|
||||
buf.writes(baseMemoryPrivateH)
|
||||
buf.writeb('\n')
|
||||
buf.writes(baseImagePrivateH)
|
||||
@@ -303,6 +305,8 @@ func insertBaseAllPublicH(buf *buffer) error {
|
||||
buf.writeb('\n')
|
||||
buf.writes(baseIOPublicH)
|
||||
buf.writeb('\n')
|
||||
buf.writes(baseTokenPublicH)
|
||||
buf.writeb('\n')
|
||||
buf.writes(baseMemoryPublicH)
|
||||
buf.writeb('\n')
|
||||
buf.writes(baseImagePublicH)
|
||||
|
||||
@@ -251,6 +251,20 @@ const baseRangePublicH = "" +
|
||||
"e__rect_ie_u32 s) const {\n return wuffs_base__rect_ie_u32__equals(this, s);\n}\n\ninline wuffs_base__rect_ie_u32 //\nwuffs_base__rect_ie_u32::intersect(wuffs_base__rect_ie_u32 s) const {\n return wuffs_base__rect_ie_u32__intersect(this, s);\n}\n\ninline wuffs_base__rect_ie_u32 //\nwuffs_base__rect_ie_u32::unite(wuffs_base__rect_ie_u32 s) const {\n return wuffs_base__rect_ie_u32__unite(this, s);\n}\n\ninline bool //\nwuffs_base__rect_ie_u32::contains(uint32_t x, uint32_t y) const {\n return wuffs_base__rect_ie_u32__contains(this, x, y);\n}\n\ninline bool //\nwuffs_base__rect_ie_u32::contains_rect(wuffs_base__rect_ie_u32 s) const {\n return wuffs_base__rect_ie_u32__contains_rect(this, s);\n}\n\ninline uint32_t //\nwuffs_base__rect_ie_u32::width() const {\n return wuffs_base__rect_ie_u32__width(this);\n}\n\ninline uint32_t //\nwuffs_base__rect_ie_u32::height() const {\n return wuffs_base__rect_ie_u32__height(this);\n}\n\n#endif // __cplusplus\n" +
|
||||
""
|
||||
|
||||
const baseTokenPrivateH = "" +
|
||||
"// ---------------- Tokens\n\n" +
|
||||
"" +
|
||||
"// ---------------- Tokens (Utility)\n" +
|
||||
""
|
||||
|
||||
const baseTokenPublicH = "" +
|
||||
"// ---------------- Tokens\n\n#define WUFFS_BASE__TOKEN__LENGTH_MASK 0xFFFF\n#define WUFFS_BASE__TOKEN__MINOR_VALUE_MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__MAJOR_VALUE_MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MASK 0xFFFFFFFFFFFF\n\n#define WUFFS_BASE__TOKEN__LENGTH_SHIFT 0\n#define WUFFS_BASE__TOKEN__MINOR_VALUE_SHIFT 16\n#define WUFFS_BASE__TOKEN__MAJOR_VALUE_SHIFT 40\n#define WUFFS_BASE__TOKEN__VALUE_SHIFT 16\n\ntypedef struct {\n // Bits 63 .. 40 (24 bits) is the major value.\n // Bits 39 .. 16 (24 bits) is the minor value.\n // Bits 15 .. 0 (16 bits) is the length.\n uint64_t repr;\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\ntypedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;\n\nstatic inline wuffs_base__slice_token //\nwuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {\n wuffs_base__slice_token ret;\n ret.ptr = ptr;\n ret.len = len;\n return ret;\n}\n\n// wuffs_bas" +
|
||||
"e__token_buffer_meta is the metadata for a\n// wuffs_base__token_buffer's data.\ntypedef struct {\n size_t wi; // Write index. Invariant: wi <= len.\n size_t ri; // Read index. Invariant: ri <= wi.\n uint64_t pos; // Position of the buffer start relative to the stream start.\n bool closed; // No further writes are expected.\n} wuffs_base__token_buffer_meta;\n\n// wuffs_base__token_buffer is a 1-dimensional buffer (a pointer and length)\n// plus additional metadata.\n//\n// A value with all fields zero is a valid, empty buffer.\ntypedef struct {\n wuffs_base__slice_token data;\n wuffs_base__token_buffer_meta meta;\n\n#ifdef __cplusplus\n inline void compact();\n inline uint64_t reader_available() const;\n inline uint64_t reader_token_position() const;\n inline uint64_t writer_available() const;\n inline uint64_t writer_token_position() const;\n#endif // __cplusplus\n\n} wuffs_base__token_buffer;\n\nstatic inline wuffs_base__token_buffer //\nwuffs_base__make_token_buffer(wuffs_base__slice_token data,\n " +
|
||||
" wuffs_base__token_buffer_meta meta) {\n wuffs_base__token_buffer ret;\n ret.data = data;\n ret.meta = meta;\n return ret;\n}\n\nstatic inline wuffs_base__token_buffer_meta //\nwuffs_base__make_token_buffer_meta(size_t wi,\n size_t ri,\n uint64_t pos,\n bool closed) {\n wuffs_base__token_buffer_meta ret;\n ret.wi = wi;\n ret.ri = ri;\n ret.pos = pos;\n ret.closed = closed;\n return ret;\n}\n\nstatic inline wuffs_base__token_buffer //\nwuffs_base__empty_token_buffer() {\n wuffs_base__token_buffer ret;\n ret.data.ptr = NULL;\n ret.data.len = 0;\n ret.meta.wi = 0;\n ret.meta.ri = 0;\n ret.meta.pos = 0;\n ret.meta.closed = false;\n return ret;\n}\n\nstatic inline wuffs_base__token_buffer_meta //\nwuffs_base__empty_token_buffer_meta() {\n wuffs_base__token_buffer_meta ret;\n ret.wi = 0;\n ret.ri = 0;\n ret.pos = 0;\n ret.closed = false;\n return ret;\n}\n\n// wuffs_base__token_buffer__compact moves any written bu" +
|
||||
"t unread tokens to the\n// start of the buffer.\nstatic inline void //\nwuffs_base__token_buffer__compact(wuffs_base__token_buffer* buf) {\n if (!buf || (buf->meta.ri == 0)) {\n return;\n }\n buf->meta.pos = wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri);\n size_t n = buf->meta.wi - buf->meta.ri;\n if (n != 0) {\n memmove(buf->data.ptr, buf->data.ptr + buf->meta.ri,\n n * sizeof(wuffs_base__token));\n }\n buf->meta.wi = n;\n buf->meta.ri = 0;\n}\n\nstatic inline uint64_t //\nwuffs_base__token_buffer__reader_available(\n const wuffs_base__token_buffer* buf) {\n return buf ? buf->meta.wi - buf->meta.ri : 0;\n}\n\nstatic inline uint64_t //\nwuffs_base__token_buffer__reader_token_position(\n const wuffs_base__token_buffer* buf) {\n return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri) : 0;\n}\n\nstatic inline uint64_t //\nwuffs_base__token_buffer__writer_available(\n const wuffs_base__token_buffer* buf) {\n return buf ? buf->data.len - buf->meta.wi : 0;\n}\n\nstatic inline uint64_t //\nw" +
|
||||
"uffs_base__token_buffer__writer_token_position(\n const wuffs_base__token_buffer* buf) {\n return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.wi) : 0;\n}\n\n#ifdef __cplusplus\n\ninline void //\nwuffs_base__token_buffer::compact() {\n wuffs_base__token_buffer__compact(this);\n}\n\ninline uint64_t //\nwuffs_base__token_buffer::reader_available() const {\n return wuffs_base__token_buffer__reader_available(this);\n}\n\ninline uint64_t //\nwuffs_base__token_buffer::reader_token_position() const {\n return wuffs_base__token_buffer__reader_token_position(this);\n}\n\ninline uint64_t //\nwuffs_base__token_buffer::writer_available() const {\n return wuffs_base__token_buffer__writer_available(this);\n}\n\ninline uint64_t //\nwuffs_base__token_buffer::writer_token_position() const {\n return wuffs_base__token_buffer__writer_token_position(this);\n}\n\n#endif // __cplusplus\n" +
|
||||
""
|
||||
|
||||
const baseCopyright = "" +
|
||||
"// Copyright 2017 The Wuffs Authors.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// https://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n" +
|
||||
""
|
||||
+14
-11
@@ -470,17 +470,20 @@ func isBaseRangeType(qid t.QID) bool {
|
||||
}
|
||||
|
||||
var cTypeNames = [...]string{
|
||||
t.IDI8: "int8_t",
|
||||
t.IDI16: "int16_t",
|
||||
t.IDI32: "int32_t",
|
||||
t.IDI64: "int64_t",
|
||||
t.IDU8: "uint8_t",
|
||||
t.IDU16: "uint16_t",
|
||||
t.IDU32: "uint32_t",
|
||||
t.IDU64: "uint64_t",
|
||||
t.IDBool: "bool",
|
||||
t.IDIOReader: "wuffs_base__io_buffer*",
|
||||
t.IDIOWriter: "wuffs_base__io_buffer*",
|
||||
t.IDI8: "int8_t",
|
||||
t.IDI16: "int16_t",
|
||||
t.IDI32: "int32_t",
|
||||
t.IDI64: "int64_t",
|
||||
t.IDU8: "uint8_t",
|
||||
t.IDU16: "uint16_t",
|
||||
t.IDU32: "uint32_t",
|
||||
t.IDU64: "uint64_t",
|
||||
t.IDBool: "bool",
|
||||
|
||||
t.IDIOReader: "wuffs_base__io_buffer*",
|
||||
t.IDIOWriter: "wuffs_base__io_buffer*",
|
||||
t.IDTokenReader: "wuffs_base__token_buffer*",
|
||||
t.IDTokenWriter: "wuffs_base__token_buffer*",
|
||||
}
|
||||
|
||||
const noSuchCOperator = " no_such_C_operator "
|
||||
|
||||
@@ -462,7 +462,7 @@ func (g *gen) writeFuncImplArgChecks(b *buffer, n *a.Func) error {
|
||||
|
||||
// TODO: Also check elements, for array-typed arguments.
|
||||
switch {
|
||||
case oTyp.IsIOType() || (oTyp.Decorator() == t.IDPtr):
|
||||
case oTyp.IsIOTokenType() || (oTyp.Decorator() == t.IDPtr):
|
||||
checks = append(checks, fmt.Sprintf("!%s%s", aPrefix, o.Name().Str(g.tm)))
|
||||
|
||||
case oTyp.IsRefined():
|
||||
|
||||
@@ -86,6 +86,8 @@ func genBase(out *bytes.Buffer) error {
|
||||
{"base/io-public.h", "baseIOPublicH"},
|
||||
{"base/range-private.h", "baseRangePrivateH"},
|
||||
{"base/range-public.h", "baseRangePublicH"},
|
||||
{"base/token-private.h", "baseTokenPrivateH"},
|
||||
{"base/token-public.h", "baseTokenPublicH"},
|
||||
}
|
||||
|
||||
prefixAfterEditing := []byte("// After editing this file,")
|
||||
|
||||
@@ -59,7 +59,7 @@ func (g *gen) needDerivedVar(name t.ID) bool {
|
||||
func (g *gen) findDerivedVars() {
|
||||
for _, o := range g.currFunk.astFunc.In().Fields() {
|
||||
o := o.AsField()
|
||||
if !o.XType().IsIOType() || !g.needDerivedVar(o.Name()) {
|
||||
if !o.XType().IsIOTokenType() || !g.needDerivedVar(o.Name()) {
|
||||
continue
|
||||
}
|
||||
if g.currFunk.derivedVars == nil {
|
||||
@@ -84,7 +84,7 @@ func (g *gen) writeLoadDerivedVar(b *buffer, hack string, prefix string, name t.
|
||||
return nil
|
||||
}
|
||||
|
||||
if !typ.IsIOType() {
|
||||
if !typ.IsIOTokenType() {
|
||||
return nil
|
||||
}
|
||||
if g.currFunk.derivedVars == nil {
|
||||
@@ -143,7 +143,7 @@ func (g *gen) writeSaveDerivedVar(b *buffer, hack string, prefix string, name t.
|
||||
return nil
|
||||
}
|
||||
|
||||
if !typ.IsIOType() {
|
||||
if !typ.IsIOTokenType() {
|
||||
return nil
|
||||
}
|
||||
if g.currFunk.derivedVars == nil {
|
||||
@@ -275,6 +275,8 @@ func (g *gen) writeVars(b *buffer, f *funk, inStructDecl bool) error {
|
||||
|
||||
if typ.IsIOType() {
|
||||
b.printf("wuffs_base__io_buffer %s%s = wuffs_base__empty_io_buffer();\n", uPrefix, name)
|
||||
} else if typ.IsTokenType() {
|
||||
return fmt.Errorf("TODO: support token_{reader,writer} typed variables")
|
||||
}
|
||||
|
||||
if err := g.writeCTypeName(b, typ, vPrefix, name); err != nil {
|
||||
|
||||
Reference in new issue
Block a user