From 737e31ff3a4f805e5765cc9b5db70b3cd3dfa666 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Tue, 11 Feb 2020 11:23:17 +1100 Subject: [PATCH] Add tokens --- doc/changelog.md | 1 + internal/cgen/base/token-private.h | 19 +++ internal/cgen/base/token-public.h | 192 +++++++++++++++++++++++++ internal/cgen/cgen.go | 4 + internal/cgen/data.go | 14 ++ internal/cgen/expr.go | 25 ++-- internal/cgen/func.go | 2 +- internal/cgen/gen.go | 2 + internal/cgen/var.go | 8 +- lang/ast/ast.go | 14 +- lang/builtin/builtin.go | 20 ++- lang/check/resolve.go | 18 ++- lang/token/list.go | 6 +- release/c/wuffs-unsupported-snapshot.c | 187 +++++++++++++++++++++++- std/json/decode_json.wuffs | 2 +- test/c/std/json.c | 4 +- test/c/testlib/testlib.c | 16 +++ 17 files changed, 498 insertions(+), 36 deletions(-) create mode 100644 internal/cgen/base/token-private.h create mode 100644 internal/cgen/base/token-public.h diff --git a/doc/changelog.md b/doc/changelog.md index a78efd1f..3e7b3abe 100644 --- a/doc/changelog.md +++ b/doc/changelog.md @@ -12,6 +12,7 @@ - Added `WUFFS_BASE__PIXEL_FORMAT__BGR_565`. - Added interfaces. - Added preprocessor. +- Added tokens. - Changed `gif.decoder_workbuf_len_max_incl_worst_case` from 1 to 0. - Made `wuffs_base__pixel_format` a struct. - Made `wuffs_base__pixel_subsampling` a struct. diff --git a/internal/cgen/base/token-private.h b/internal/cgen/base/token-private.h new file mode 100644 index 00000000..df30bcd0 --- /dev/null +++ b/internal/cgen/base/token-private.h @@ -0,0 +1,19 @@ +// After editing this file, run "go generate" in the parent directory. + +// Copyright 2020 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ---------------- Tokens + +// ---------------- Tokens (Utility) diff --git a/internal/cgen/base/token-public.h b/internal/cgen/base/token-public.h new file mode 100644 index 00000000..5028873f --- /dev/null +++ b/internal/cgen/base/token-public.h @@ -0,0 +1,192 @@ +// After editing this file, run "go generate" in the parent directory. + +// Copyright 2020 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ---------------- Tokens + +#define WUFFS_BASE__TOKEN__LENGTH_MASK 0xFFFF +#define WUFFS_BASE__TOKEN__MINOR_VALUE_MASK 0xFFFFFF +#define WUFFS_BASE__TOKEN__MAJOR_VALUE_MASK 0xFFFFFF +#define WUFFS_BASE__TOKEN__VALUE_MASK 0xFFFFFFFFFFFF + +#define WUFFS_BASE__TOKEN__LENGTH_SHIFT 0 +#define WUFFS_BASE__TOKEN__MINOR_VALUE_SHIFT 16 +#define WUFFS_BASE__TOKEN__MAJOR_VALUE_SHIFT 40 +#define WUFFS_BASE__TOKEN__VALUE_SHIFT 16 + +typedef struct { + // Bits 63 .. 40 (24 bits) is the major value. + // Bits 39 .. 16 (24 bits) is the minor value. + // Bits 15 .. 0 (16 bits) is the length. + uint64_t repr; +} wuffs_base__token; + +static inline wuffs_base__token // +wuffs_base__make_token(uint64_t repr) { + wuffs_base__token ret; + ret.repr = repr; + return ret; +} + +typedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token; + +static inline wuffs_base__slice_token // +wuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) { + wuffs_base__slice_token ret; + ret.ptr = ptr; + ret.len = len; + return ret; +} + +// wuffs_base__token_buffer_meta is the metadata for a +// wuffs_base__token_buffer's data. +typedef struct { + size_t wi; // Write index. Invariant: wi <= len. + size_t ri; // Read index. Invariant: ri <= wi. + uint64_t pos; // Position of the buffer start relative to the stream start. + bool closed; // No further writes are expected. +} wuffs_base__token_buffer_meta; + +// wuffs_base__token_buffer is a 1-dimensional buffer (a pointer and length) +// plus additional metadata. +// +// A value with all fields zero is a valid, empty buffer. +typedef struct { + wuffs_base__slice_token data; + wuffs_base__token_buffer_meta meta; + +#ifdef __cplusplus + inline void compact(); + inline uint64_t reader_available() const; + inline uint64_t reader_token_position() const; + inline uint64_t writer_available() const; + inline uint64_t writer_token_position() const; +#endif // __cplusplus + +} wuffs_base__token_buffer; + +static inline wuffs_base__token_buffer // +wuffs_base__make_token_buffer(wuffs_base__slice_token data, + wuffs_base__token_buffer_meta meta) { + wuffs_base__token_buffer ret; + ret.data = data; + ret.meta = meta; + return ret; +} + +static inline wuffs_base__token_buffer_meta // +wuffs_base__make_token_buffer_meta(size_t wi, + size_t ri, + uint64_t pos, + bool closed) { + wuffs_base__token_buffer_meta ret; + ret.wi = wi; + ret.ri = ri; + ret.pos = pos; + ret.closed = closed; + return ret; +} + +static inline wuffs_base__token_buffer // +wuffs_base__empty_token_buffer() { + wuffs_base__token_buffer ret; + ret.data.ptr = NULL; + ret.data.len = 0; + ret.meta.wi = 0; + ret.meta.ri = 0; + ret.meta.pos = 0; + ret.meta.closed = false; + return ret; +} + +static inline wuffs_base__token_buffer_meta // +wuffs_base__empty_token_buffer_meta() { + wuffs_base__token_buffer_meta ret; + ret.wi = 0; + ret.ri = 0; + ret.pos = 0; + ret.closed = false; + return ret; +} + +// wuffs_base__token_buffer__compact moves any written but unread tokens to the +// start of the buffer. +static inline void // +wuffs_base__token_buffer__compact(wuffs_base__token_buffer* buf) { + if (!buf || (buf->meta.ri == 0)) { + return; + } + buf->meta.pos = wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri); + size_t n = buf->meta.wi - buf->meta.ri; + if (n != 0) { + memmove(buf->data.ptr, buf->data.ptr + buf->meta.ri, + n * sizeof(wuffs_base__token)); + } + buf->meta.wi = n; + buf->meta.ri = 0; +} + +static inline uint64_t // +wuffs_base__token_buffer__reader_available( + const wuffs_base__token_buffer* buf) { + return buf ? buf->meta.wi - buf->meta.ri : 0; +} + +static inline uint64_t // +wuffs_base__token_buffer__reader_token_position( + const wuffs_base__token_buffer* buf) { + return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri) : 0; +} + +static inline uint64_t // +wuffs_base__token_buffer__writer_available( + const wuffs_base__token_buffer* buf) { + return buf ? buf->data.len - buf->meta.wi : 0; +} + +static inline uint64_t // +wuffs_base__token_buffer__writer_token_position( + const wuffs_base__token_buffer* buf) { + return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.wi) : 0; +} + +#ifdef __cplusplus + +inline void // +wuffs_base__token_buffer::compact() { + wuffs_base__token_buffer__compact(this); +} + +inline uint64_t // +wuffs_base__token_buffer::reader_available() const { + return wuffs_base__token_buffer__reader_available(this); +} + +inline uint64_t // +wuffs_base__token_buffer::reader_token_position() const { + return wuffs_base__token_buffer__reader_token_position(this); +} + +inline uint64_t // +wuffs_base__token_buffer::writer_available() const { + return wuffs_base__token_buffer__writer_available(this); +} + +inline uint64_t // +wuffs_base__token_buffer::writer_token_position() const { + return wuffs_base__token_buffer__writer_token_position(this); +} + +#endif // __cplusplus diff --git a/internal/cgen/cgen.go b/internal/cgen/cgen.go index 76c38f4e..502f0a3d 100644 --- a/internal/cgen/cgen.go +++ b/internal/cgen/cgen.go @@ -256,6 +256,8 @@ func insertBaseAllPrivateH(buf *buffer) error { buf.writeb('\n') buf.writes(baseIOPrivateH) buf.writeb('\n') + buf.writes(baseTokenPrivateH) + buf.writeb('\n') buf.writes(baseMemoryPrivateH) buf.writeb('\n') buf.writes(baseImagePrivateH) @@ -303,6 +305,8 @@ func insertBaseAllPublicH(buf *buffer) error { buf.writeb('\n') buf.writes(baseIOPublicH) buf.writeb('\n') + buf.writes(baseTokenPublicH) + buf.writeb('\n') buf.writes(baseMemoryPublicH) buf.writeb('\n') buf.writes(baseImagePublicH) diff --git a/internal/cgen/data.go b/internal/cgen/data.go index 5f879232..a9ff5ef3 100644 --- a/internal/cgen/data.go +++ b/internal/cgen/data.go @@ -251,6 +251,20 @@ const baseRangePublicH = "" + "e__rect_ie_u32 s) const {\n return wuffs_base__rect_ie_u32__equals(this, s);\n}\n\ninline wuffs_base__rect_ie_u32 //\nwuffs_base__rect_ie_u32::intersect(wuffs_base__rect_ie_u32 s) const {\n return wuffs_base__rect_ie_u32__intersect(this, s);\n}\n\ninline wuffs_base__rect_ie_u32 //\nwuffs_base__rect_ie_u32::unite(wuffs_base__rect_ie_u32 s) const {\n return wuffs_base__rect_ie_u32__unite(this, s);\n}\n\ninline bool //\nwuffs_base__rect_ie_u32::contains(uint32_t x, uint32_t y) const {\n return wuffs_base__rect_ie_u32__contains(this, x, y);\n}\n\ninline bool //\nwuffs_base__rect_ie_u32::contains_rect(wuffs_base__rect_ie_u32 s) const {\n return wuffs_base__rect_ie_u32__contains_rect(this, s);\n}\n\ninline uint32_t //\nwuffs_base__rect_ie_u32::width() const {\n return wuffs_base__rect_ie_u32__width(this);\n}\n\ninline uint32_t //\nwuffs_base__rect_ie_u32::height() const {\n return wuffs_base__rect_ie_u32__height(this);\n}\n\n#endif // __cplusplus\n" + "" +const baseTokenPrivateH = "" + + "// ---------------- Tokens\n\n" + + "" + + "// ---------------- Tokens (Utility)\n" + + "" + +const baseTokenPublicH = "" + + "// ---------------- Tokens\n\n#define WUFFS_BASE__TOKEN__LENGTH_MASK 0xFFFF\n#define WUFFS_BASE__TOKEN__MINOR_VALUE_MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__MAJOR_VALUE_MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MASK 0xFFFFFFFFFFFF\n\n#define WUFFS_BASE__TOKEN__LENGTH_SHIFT 0\n#define WUFFS_BASE__TOKEN__MINOR_VALUE_SHIFT 16\n#define WUFFS_BASE__TOKEN__MAJOR_VALUE_SHIFT 40\n#define WUFFS_BASE__TOKEN__VALUE_SHIFT 16\n\ntypedef struct {\n // Bits 63 .. 40 (24 bits) is the major value.\n // Bits 39 .. 16 (24 bits) is the minor value.\n // Bits 15 .. 0 (16 bits) is the length.\n uint64_t repr;\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\ntypedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;\n\nstatic inline wuffs_base__slice_token //\nwuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {\n wuffs_base__slice_token ret;\n ret.ptr = ptr;\n ret.len = len;\n return ret;\n}\n\n// wuffs_bas" + + "e__token_buffer_meta is the metadata for a\n// wuffs_base__token_buffer's data.\ntypedef struct {\n size_t wi; // Write index. Invariant: wi <= len.\n size_t ri; // Read index. Invariant: ri <= wi.\n uint64_t pos; // Position of the buffer start relative to the stream start.\n bool closed; // No further writes are expected.\n} wuffs_base__token_buffer_meta;\n\n// wuffs_base__token_buffer is a 1-dimensional buffer (a pointer and length)\n// plus additional metadata.\n//\n// A value with all fields zero is a valid, empty buffer.\ntypedef struct {\n wuffs_base__slice_token data;\n wuffs_base__token_buffer_meta meta;\n\n#ifdef __cplusplus\n inline void compact();\n inline uint64_t reader_available() const;\n inline uint64_t reader_token_position() const;\n inline uint64_t writer_available() const;\n inline uint64_t writer_token_position() const;\n#endif // __cplusplus\n\n} wuffs_base__token_buffer;\n\nstatic inline wuffs_base__token_buffer //\nwuffs_base__make_token_buffer(wuffs_base__slice_token data,\n " + + " wuffs_base__token_buffer_meta meta) {\n wuffs_base__token_buffer ret;\n ret.data = data;\n ret.meta = meta;\n return ret;\n}\n\nstatic inline wuffs_base__token_buffer_meta //\nwuffs_base__make_token_buffer_meta(size_t wi,\n size_t ri,\n uint64_t pos,\n bool closed) {\n wuffs_base__token_buffer_meta ret;\n ret.wi = wi;\n ret.ri = ri;\n ret.pos = pos;\n ret.closed = closed;\n return ret;\n}\n\nstatic inline wuffs_base__token_buffer //\nwuffs_base__empty_token_buffer() {\n wuffs_base__token_buffer ret;\n ret.data.ptr = NULL;\n ret.data.len = 0;\n ret.meta.wi = 0;\n ret.meta.ri = 0;\n ret.meta.pos = 0;\n ret.meta.closed = false;\n return ret;\n}\n\nstatic inline wuffs_base__token_buffer_meta //\nwuffs_base__empty_token_buffer_meta() {\n wuffs_base__token_buffer_meta ret;\n ret.wi = 0;\n ret.ri = 0;\n ret.pos = 0;\n ret.closed = false;\n return ret;\n}\n\n// wuffs_base__token_buffer__compact moves any written bu" + + "t unread tokens to the\n// start of the buffer.\nstatic inline void //\nwuffs_base__token_buffer__compact(wuffs_base__token_buffer* buf) {\n if (!buf || (buf->meta.ri == 0)) {\n return;\n }\n buf->meta.pos = wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri);\n size_t n = buf->meta.wi - buf->meta.ri;\n if (n != 0) {\n memmove(buf->data.ptr, buf->data.ptr + buf->meta.ri,\n n * sizeof(wuffs_base__token));\n }\n buf->meta.wi = n;\n buf->meta.ri = 0;\n}\n\nstatic inline uint64_t //\nwuffs_base__token_buffer__reader_available(\n const wuffs_base__token_buffer* buf) {\n return buf ? buf->meta.wi - buf->meta.ri : 0;\n}\n\nstatic inline uint64_t //\nwuffs_base__token_buffer__reader_token_position(\n const wuffs_base__token_buffer* buf) {\n return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri) : 0;\n}\n\nstatic inline uint64_t //\nwuffs_base__token_buffer__writer_available(\n const wuffs_base__token_buffer* buf) {\n return buf ? buf->data.len - buf->meta.wi : 0;\n}\n\nstatic inline uint64_t //\nw" + + "uffs_base__token_buffer__writer_token_position(\n const wuffs_base__token_buffer* buf) {\n return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.wi) : 0;\n}\n\n#ifdef __cplusplus\n\ninline void //\nwuffs_base__token_buffer::compact() {\n wuffs_base__token_buffer__compact(this);\n}\n\ninline uint64_t //\nwuffs_base__token_buffer::reader_available() const {\n return wuffs_base__token_buffer__reader_available(this);\n}\n\ninline uint64_t //\nwuffs_base__token_buffer::reader_token_position() const {\n return wuffs_base__token_buffer__reader_token_position(this);\n}\n\ninline uint64_t //\nwuffs_base__token_buffer::writer_available() const {\n return wuffs_base__token_buffer__writer_available(this);\n}\n\ninline uint64_t //\nwuffs_base__token_buffer::writer_token_position() const {\n return wuffs_base__token_buffer__writer_token_position(this);\n}\n\n#endif // __cplusplus\n" + + "" + const baseCopyright = "" + "// Copyright 2017 The Wuffs Authors.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// https://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n" + "" diff --git a/internal/cgen/expr.go b/internal/cgen/expr.go index bbb2de3f..7c2874da 100644 --- a/internal/cgen/expr.go +++ b/internal/cgen/expr.go @@ -470,17 +470,20 @@ func isBaseRangeType(qid t.QID) bool { } var cTypeNames = [...]string{ - t.IDI8: "int8_t", - t.IDI16: "int16_t", - t.IDI32: "int32_t", - t.IDI64: "int64_t", - t.IDU8: "uint8_t", - t.IDU16: "uint16_t", - t.IDU32: "uint32_t", - t.IDU64: "uint64_t", - t.IDBool: "bool", - t.IDIOReader: "wuffs_base__io_buffer*", - t.IDIOWriter: "wuffs_base__io_buffer*", + t.IDI8: "int8_t", + t.IDI16: "int16_t", + t.IDI32: "int32_t", + t.IDI64: "int64_t", + t.IDU8: "uint8_t", + t.IDU16: "uint16_t", + t.IDU32: "uint32_t", + t.IDU64: "uint64_t", + t.IDBool: "bool", + + t.IDIOReader: "wuffs_base__io_buffer*", + t.IDIOWriter: "wuffs_base__io_buffer*", + t.IDTokenReader: "wuffs_base__token_buffer*", + t.IDTokenWriter: "wuffs_base__token_buffer*", } const noSuchCOperator = " no_such_C_operator " diff --git a/internal/cgen/func.go b/internal/cgen/func.go index 8380faa9..9a25cf06 100644 --- a/internal/cgen/func.go +++ b/internal/cgen/func.go @@ -462,7 +462,7 @@ func (g *gen) writeFuncImplArgChecks(b *buffer, n *a.Func) error { // TODO: Also check elements, for array-typed arguments. switch { - case oTyp.IsIOType() || (oTyp.Decorator() == t.IDPtr): + case oTyp.IsIOTokenType() || (oTyp.Decorator() == t.IDPtr): checks = append(checks, fmt.Sprintf("!%s%s", aPrefix, o.Name().Str(g.tm))) case oTyp.IsRefined(): diff --git a/internal/cgen/gen.go b/internal/cgen/gen.go index 000805bc..51bca4c5 100644 --- a/internal/cgen/gen.go +++ b/internal/cgen/gen.go @@ -86,6 +86,8 @@ func genBase(out *bytes.Buffer) error { {"base/io-public.h", "baseIOPublicH"}, {"base/range-private.h", "baseRangePrivateH"}, {"base/range-public.h", "baseRangePublicH"}, + {"base/token-private.h", "baseTokenPrivateH"}, + {"base/token-public.h", "baseTokenPublicH"}, } prefixAfterEditing := []byte("// After editing this file,") diff --git a/internal/cgen/var.go b/internal/cgen/var.go index 0ed0b14b..bc6d7fdd 100644 --- a/internal/cgen/var.go +++ b/internal/cgen/var.go @@ -59,7 +59,7 @@ func (g *gen) needDerivedVar(name t.ID) bool { func (g *gen) findDerivedVars() { for _, o := range g.currFunk.astFunc.In().Fields() { o := o.AsField() - if !o.XType().IsIOType() || !g.needDerivedVar(o.Name()) { + if !o.XType().IsIOTokenType() || !g.needDerivedVar(o.Name()) { continue } if g.currFunk.derivedVars == nil { @@ -84,7 +84,7 @@ func (g *gen) writeLoadDerivedVar(b *buffer, hack string, prefix string, name t. return nil } - if !typ.IsIOType() { + if !typ.IsIOTokenType() { return nil } if g.currFunk.derivedVars == nil { @@ -143,7 +143,7 @@ func (g *gen) writeSaveDerivedVar(b *buffer, hack string, prefix string, name t. return nil } - if !typ.IsIOType() { + if !typ.IsIOTokenType() { return nil } if g.currFunk.derivedVars == nil { @@ -275,6 +275,8 @@ func (g *gen) writeVars(b *buffer, f *funk, inStructDecl bool) error { if typ.IsIOType() { b.printf("wuffs_base__io_buffer %s%s = wuffs_base__empty_io_buffer();\n", uPrefix, name) + } else if typ.IsTokenType() { + return fmt.Errorf("TODO: support token_{reader,writer} typed variables") } if err := g.writeCTypeName(b, typ, vPrefix, name); err != nil { diff --git a/lang/ast/ast.go b/lang/ast/ast.go index fdbb5464..2f356d18 100644 --- a/lang/ast/ast.go +++ b/lang/ast/ast.go @@ -700,7 +700,19 @@ func (n *TypeExpr) IsIdeal() bool { } func (n *TypeExpr) IsIOType() bool { - return n.id0 == 0 && n.id1 == t.IDBase && (n.id2 == t.IDIOReader || n.id2 == t.IDIOWriter) + return n.id0 == 0 && n.id1 == t.IDBase && + (n.id2 == t.IDIOReader || n.id2 == t.IDIOWriter) +} + +func (n *TypeExpr) IsTokenType() bool { + return n.id0 == 0 && n.id1 == t.IDBase && + (n.id2 == t.IDTokenReader || n.id2 == t.IDTokenWriter) +} + +func (n *TypeExpr) IsIOTokenType() bool { + return n.id0 == 0 && n.id1 == t.IDBase && + (n.id2 == t.IDIOReader || n.id2 == t.IDIOWriter || + n.id2 == t.IDTokenReader || n.id2 == t.IDTokenWriter) } func (n *TypeExpr) IsNullptr() bool { diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index 98f34da1..b698e6b7 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -83,9 +83,11 @@ var Types = []string{ "rect_ie_u32", "rect_ii_u32", + "status", + "io_reader", "io_writer", - "status", + "token_writer", "frame_config", "image_config", @@ -166,6 +168,14 @@ var Funcs = []string{ "range_ii_u64.intersect(r: range_ii_u64) range_ii_u64", "range_ii_u64.unite(r: range_ii_u64) range_ii_u64", + // ---- status + + // TODO: should we add is_complete? + "status.is_error() bool", + "status.is_note() bool", + "status.is_ok() bool", + "status.is_suspension() bool", + // ---- io_reader "io_reader.can_undo_byte() bool", @@ -310,13 +320,9 @@ var Funcs = []string{ // For now, that's all implicitly checked (i.e. hard coded). "io_writer.copy_n32_from_history_fast!(n: u32, distance: u32) u32", - // ---- status + // ---- token_writer - // TODO: should we add is_complete? - "status.is_error() bool", - "status.is_note() bool", - "status.is_ok() bool", - "status.is_suspension() bool", + "token_writer.write_token?(value: u64[..= 0xFFFFFFFFFFFF], length: u64[..= 0xFFFF])", // ---- frame_config // Duration's upper bound is the maximum possible i64 value. diff --git a/lang/check/resolve.go b/lang/check/resolve.go index c193f992..82c25bc4 100644 --- a/lang/check/resolve.go +++ b/lang/check/resolve.go @@ -54,9 +54,12 @@ var ( typeExprRectIEU32 = a.NewTypeExpr(0, t.IDBase, t.IDRectIEU32, nil, nil, nil) typeExprRectIIU32 = a.NewTypeExpr(0, t.IDBase, t.IDRectIIU32, nil, nil, nil) - typeExprIOReader = a.NewTypeExpr(0, t.IDBase, t.IDIOReader, nil, nil, nil) - typeExprIOWriter = a.NewTypeExpr(0, t.IDBase, t.IDIOWriter, nil, nil, nil) - typeExprStatus = a.NewTypeExpr(0, t.IDBase, t.IDStatus, nil, nil, nil) + typeExprStatus = a.NewTypeExpr(0, t.IDBase, t.IDStatus, nil, nil, nil) + + typeExprIOReader = a.NewTypeExpr(0, t.IDBase, t.IDIOReader, nil, nil, nil) + typeExprIOWriter = a.NewTypeExpr(0, t.IDBase, t.IDIOWriter, nil, nil, nil) + typeExprTokenReader = a.NewTypeExpr(0, t.IDBase, t.IDTokenReader, nil, nil, nil) + typeExprTokenWriter = a.NewTypeExpr(0, t.IDBase, t.IDTokenWriter, nil, nil, nil) typeExprFrameConfig = a.NewTypeExpr(0, t.IDBase, t.IDFrameConfig, nil, nil, nil) typeExprImageConfig = a.NewTypeExpr(0, t.IDBase, t.IDImageConfig, nil, nil, nil) @@ -97,9 +100,12 @@ var builtInTypeMap = typeMap{ t.IDRectIEU32: typeExprRectIEU32, t.IDRectIIU32: typeExprRectIIU32, - t.IDIOReader: typeExprIOReader, - t.IDIOWriter: typeExprIOWriter, - t.IDStatus: typeExprStatus, + t.IDStatus: typeExprStatus, + + t.IDIOReader: typeExprIOReader, + t.IDIOWriter: typeExprIOWriter, + t.IDTokenReader: typeExprTokenReader, + t.IDTokenWriter: typeExprTokenWriter, t.IDFrameConfig: typeExprFrameConfig, t.IDImageConfig: typeExprImageConfig, diff --git a/lang/token/list.go b/lang/token/list.go index fa4d7974..c9797bce 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -467,7 +467,9 @@ const ( IDIOReader = ID(0x125) IDIOWriter = ID(0x126) IDStatus = ID(0x127) - IDUtility = ID(0x128) + IDTokenReader = ID(0x128) + IDTokenWriter = ID(0x129) + IDUtility = ID(0x12A) IDRangeIEU32 = ID(0x130) IDRangeIIU32 = ID(0x131) @@ -810,6 +812,8 @@ var builtInsByID = [nBuiltInIDs]string{ IDIOReader: "io_reader", IDIOWriter: "io_writer", IDStatus: "status", + IDTokenReader: "token_reader", + IDTokenWriter: "token_writer", IDUtility: "utility", IDRangeIEU32: "range_ie_u32", diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 8de7c66a..31853b85 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -1459,6 +1459,183 @@ wuffs_base__io_buffer::writer_io_position() const { return wuffs_base__io_buffer__writer_io_position(this); } +#endif // __cplusplus + + // ---------------- Tokens + +#define WUFFS_BASE__TOKEN__LENGTH_MASK 0xFFFF +#define WUFFS_BASE__TOKEN__MINOR_VALUE_MASK 0xFFFFFF +#define WUFFS_BASE__TOKEN__MAJOR_VALUE_MASK 0xFFFFFF +#define WUFFS_BASE__TOKEN__VALUE_MASK 0xFFFFFFFFFFFF + +#define WUFFS_BASE__TOKEN__LENGTH_SHIFT 0 +#define WUFFS_BASE__TOKEN__MINOR_VALUE_SHIFT 16 +#define WUFFS_BASE__TOKEN__MAJOR_VALUE_SHIFT 40 +#define WUFFS_BASE__TOKEN__VALUE_SHIFT 16 + +typedef struct { + // Bits 63 .. 40 (24 bits) is the major value. + // Bits 39 .. 16 (24 bits) is the minor value. + // Bits 15 .. 0 (16 bits) is the length. + uint64_t repr; +} wuffs_base__token; + +static inline wuffs_base__token // +wuffs_base__make_token(uint64_t repr) { + wuffs_base__token ret; + ret.repr = repr; + return ret; +} + +typedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token; + +static inline wuffs_base__slice_token // +wuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) { + wuffs_base__slice_token ret; + ret.ptr = ptr; + ret.len = len; + return ret; +} + +// wuffs_base__token_buffer_meta is the metadata for a +// wuffs_base__token_buffer's data. +typedef struct { + size_t wi; // Write index. Invariant: wi <= len. + size_t ri; // Read index. Invariant: ri <= wi. + uint64_t pos; // Position of the buffer start relative to the stream start. + bool closed; // No further writes are expected. +} wuffs_base__token_buffer_meta; + +// wuffs_base__token_buffer is a 1-dimensional buffer (a pointer and length) +// plus additional metadata. +// +// A value with all fields zero is a valid, empty buffer. +typedef struct { + wuffs_base__slice_token data; + wuffs_base__token_buffer_meta meta; + +#ifdef __cplusplus + inline void compact(); + inline uint64_t reader_available() const; + inline uint64_t reader_token_position() const; + inline uint64_t writer_available() const; + inline uint64_t writer_token_position() const; +#endif // __cplusplus + +} wuffs_base__token_buffer; + +static inline wuffs_base__token_buffer // +wuffs_base__make_token_buffer(wuffs_base__slice_token data, + wuffs_base__token_buffer_meta meta) { + wuffs_base__token_buffer ret; + ret.data = data; + ret.meta = meta; + return ret; +} + +static inline wuffs_base__token_buffer_meta // +wuffs_base__make_token_buffer_meta(size_t wi, + size_t ri, + uint64_t pos, + bool closed) { + wuffs_base__token_buffer_meta ret; + ret.wi = wi; + ret.ri = ri; + ret.pos = pos; + ret.closed = closed; + return ret; +} + +static inline wuffs_base__token_buffer // +wuffs_base__empty_token_buffer() { + wuffs_base__token_buffer ret; + ret.data.ptr = NULL; + ret.data.len = 0; + ret.meta.wi = 0; + ret.meta.ri = 0; + ret.meta.pos = 0; + ret.meta.closed = false; + return ret; +} + +static inline wuffs_base__token_buffer_meta // +wuffs_base__empty_token_buffer_meta() { + wuffs_base__token_buffer_meta ret; + ret.wi = 0; + ret.ri = 0; + ret.pos = 0; + ret.closed = false; + return ret; +} + +// wuffs_base__token_buffer__compact moves any written but unread tokens to the +// start of the buffer. +static inline void // +wuffs_base__token_buffer__compact(wuffs_base__token_buffer* buf) { + if (!buf || (buf->meta.ri == 0)) { + return; + } + buf->meta.pos = wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri); + size_t n = buf->meta.wi - buf->meta.ri; + if (n != 0) { + memmove(buf->data.ptr, buf->data.ptr + buf->meta.ri, + n * sizeof(wuffs_base__token)); + } + buf->meta.wi = n; + buf->meta.ri = 0; +} + +static inline uint64_t // +wuffs_base__token_buffer__reader_available( + const wuffs_base__token_buffer* buf) { + return buf ? buf->meta.wi - buf->meta.ri : 0; +} + +static inline uint64_t // +wuffs_base__token_buffer__reader_token_position( + const wuffs_base__token_buffer* buf) { + return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri) : 0; +} + +static inline uint64_t // +wuffs_base__token_buffer__writer_available( + const wuffs_base__token_buffer* buf) { + return buf ? buf->data.len - buf->meta.wi : 0; +} + +static inline uint64_t // +wuffs_base__token_buffer__writer_token_position( + const wuffs_base__token_buffer* buf) { + return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.wi) : 0; +} + +#ifdef __cplusplus + +inline void // +wuffs_base__token_buffer::compact() { + wuffs_base__token_buffer__compact(this); +} + +inline uint64_t // +wuffs_base__token_buffer::reader_available() const { + return wuffs_base__token_buffer__reader_available(this); +} + +inline uint64_t // +wuffs_base__token_buffer::reader_token_position() const { + return wuffs_base__token_buffer__reader_token_position(this); +} + +inline uint64_t // +wuffs_base__token_buffer::writer_available() const { + return wuffs_base__token_buffer__writer_available(this); +} + +inline uint64_t // +wuffs_base__token_buffer::writer_token_position() const { + return wuffs_base__token_buffer__writer_token_position(this); +} + #endif // __cplusplus // ---------------- Memory Allocation @@ -4843,7 +5020,7 @@ sizeof__wuffs_json__decoder(); WUFFS_BASE__MAYBE_STATIC wuffs_base__status // wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, - wuffs_base__io_buffer* a_dst, + wuffs_base__token_buffer* a_dst, wuffs_base__io_buffer* a_src); // ---------------- Struct Definitions @@ -4908,7 +5085,7 @@ struct wuffs_json__decoder__struct { } inline wuffs_base__status // - decode_tokens(wuffs_base__io_buffer* a_dst, wuffs_base__io_buffer* a_src) { + decode_tokens(wuffs_base__token_buffer* a_dst, wuffs_base__io_buffer* a_src) { return wuffs_json__decoder__decode_tokens(this, a_dst, a_src); } @@ -6065,6 +6242,10 @@ wuffs_base__io_writer__set(wuffs_base__io_buffer* b, #define wuffs_base__utility__empty_io_reader wuffs_base__empty_io_reader #define wuffs_base__utility__empty_io_writer wuffs_base__empty_io_writer + // ---------------- Tokens + + // ---------------- Tokens (Utility) + // ---------------- Memory Allocation // ---------------- Images @@ -17906,7 +18087,7 @@ sizeof__wuffs_json__decoder() { WUFFS_BASE__MAYBE_STATIC wuffs_base__status // wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, - wuffs_base__io_buffer* a_dst, + wuffs_base__token_buffer* a_dst, wuffs_base__io_buffer* a_src) { if (!self) { return wuffs_base__make_status(wuffs_base__error__bad_receiver); diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index ad88dd7a..db17fec2 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -17,6 +17,6 @@ pub struct decoder?( ) -pub func decoder.decode_tokens?(dst: base.io_writer, src: base.io_reader) { +pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { return ok } diff --git a/test/c/std/json.c b/test/c/std/json.c index f5c80905..2168e6da 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c @@ -77,8 +77,8 @@ const char* test_wuffs_json_decode_tokens() { &dec, sizeof dec, WUFFS_VERSION, WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED)); - wuffs_base__io_buffer got = ((wuffs_base__io_buffer){ - .data = global_got_slice, + wuffs_base__token_buffer got = ((wuffs_base__token_buffer){ + .data = global_got_token_slice, }); wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ .data = global_src_slice, diff --git a/test/c/testlib/testlib.c b/test/c/testlib/testlib.c index 7c27d6a6..235656e9 100644 --- a/test/c/testlib/testlib.c +++ b/test/c/testlib/testlib.c @@ -21,6 +21,7 @@ #include #define BUFFER_SIZE (64 * 1024 * 1024) +#define TOKEN_BUFFER_SIZE (8 * 1024) #define WUFFS_TESTLIB_ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) @@ -30,12 +31,18 @@ uint8_t global_work_array[BUFFER_SIZE]; uint8_t global_src_array[BUFFER_SIZE]; uint8_t global_pixel_array[BUFFER_SIZE]; +wuffs_base__token global_got_token_array[TOKEN_BUFFER_SIZE]; +wuffs_base__token global_want_token_array[TOKEN_BUFFER_SIZE]; + wuffs_base__slice_u8 global_got_slice; wuffs_base__slice_u8 global_want_slice; wuffs_base__slice_u8 global_work_slice; wuffs_base__slice_u8 global_src_slice; wuffs_base__slice_u8 global_pixel_slice; +wuffs_base__slice_token global_got_token_slice; +wuffs_base__slice_token global_want_token_slice; + void wuffs_testlib__initialize_global_xxx_slices() { global_got_slice = ((wuffs_base__slice_u8){ .ptr = global_got_array, @@ -57,6 +64,15 @@ void wuffs_testlib__initialize_global_xxx_slices() { .ptr = global_pixel_array, .len = BUFFER_SIZE, }); + + global_got_token_slice = ((wuffs_base__slice_token){ + .ptr = global_got_token_array, + .len = TOKEN_BUFFER_SIZE, + }); + global_want_token_slice = ((wuffs_base__slice_token){ + .ptr = global_want_token_array, + .len = TOKEN_BUFFER_SIZE, + }); } char fail_msg[65536] = {0};