Add tokens

This commit is contained in:
Nigel Tao committed 2020-02-11 12:02:58 +11:00
1 parent 7a142168be
commit 737e31ff3a
17 files changed
+498 -36

No files matched your search

+1
View File
@@ -12,6 +12,7 @@
- Added `WUFFS_BASE__PIXEL_FORMAT__BGR_565`.
- Added interfaces.
- Added preprocessor.
- Added tokens.
- Changed `gif.decoder_workbuf_len_max_incl_worst_case` from 1 to 0.
- Made `wuffs_base__pixel_format` a struct.
- Made `wuffs_base__pixel_subsampling` a struct.
+19
View File
@@ -0,0 +1,19 @@
// After editing this file, run "go generate" in the parent directory.
// Copyright 2020 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ---------------- Tokens
// ---------------- Tokens (Utility)
+192
View File
@@ -0,0 +1,192 @@
// After editing this file, run "go generate" in the parent directory.
// Copyright 2020 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ---------------- Tokens
#define WUFFS_BASE__TOKEN__LENGTH_MASK 0xFFFF
#define WUFFS_BASE__TOKEN__MINOR_VALUE_MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__MAJOR_VALUE_MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MASK 0xFFFFFFFFFFFF
#define WUFFS_BASE__TOKEN__LENGTH_SHIFT 0
#define WUFFS_BASE__TOKEN__MINOR_VALUE_SHIFT 16
#define WUFFS_BASE__TOKEN__MAJOR_VALUE_SHIFT 40
#define WUFFS_BASE__TOKEN__VALUE_SHIFT 16
typedef struct {
// Bits 63 .. 40 (24 bits) is the major value.
// Bits 39 .. 16 (24 bits) is the minor value.
// Bits 15 .. 0 (16 bits) is the length.
uint64_t repr;
} wuffs_base__token;
static inline wuffs_base__token //
wuffs_base__make_token(uint64_t repr) {
wuffs_base__token ret;
ret.repr = repr;
return ret;
}
typedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;
static inline wuffs_base__slice_token //
wuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {
wuffs_base__slice_token ret;
ret.ptr = ptr;
ret.len = len;
return ret;
}
// wuffs_base__token_buffer_meta is the metadata for a
// wuffs_base__token_buffer's data.
typedef struct {
size_t wi; // Write index. Invariant: wi <= len.
size_t ri; // Read index. Invariant: ri <= wi.
uint64_t pos; // Position of the buffer start relative to the stream start.
bool closed; // No further writes are expected.
} wuffs_base__token_buffer_meta;
// wuffs_base__token_buffer is a 1-dimensional buffer (a pointer and length)
// plus additional metadata.
//
// A value with all fields zero is a valid, empty buffer.
typedef struct {
wuffs_base__slice_token data;
wuffs_base__token_buffer_meta meta;
#ifdef __cplusplus
inline void compact();
inline uint64_t reader_available() const;
inline uint64_t reader_token_position() const;
inline uint64_t writer_available() const;
inline uint64_t writer_token_position() const;
#endif // __cplusplus
} wuffs_base__token_buffer;
static inline wuffs_base__token_buffer //
wuffs_base__make_token_buffer(wuffs_base__slice_token data,
wuffs_base__token_buffer_meta meta) {
wuffs_base__token_buffer ret;
ret.data = data;
ret.meta = meta;
return ret;
}
static inline wuffs_base__token_buffer_meta //
wuffs_base__make_token_buffer_meta(size_t wi,
size_t ri,
uint64_t pos,
bool closed) {
wuffs_base__token_buffer_meta ret;
ret.wi = wi;
ret.ri = ri;
ret.pos = pos;
ret.closed = closed;
return ret;
}
static inline wuffs_base__token_buffer //
wuffs_base__empty_token_buffer() {
wuffs_base__token_buffer ret;
ret.data.ptr = NULL;
ret.data.len = 0;
ret.meta.wi = 0;
ret.meta.ri = 0;
ret.meta.pos = 0;
ret.meta.closed = false;
return ret;
}
static inline wuffs_base__token_buffer_meta //
wuffs_base__empty_token_buffer_meta() {
wuffs_base__token_buffer_meta ret;
ret.wi = 0;
ret.ri = 0;
ret.pos = 0;
ret.closed = false;
return ret;
}
// wuffs_base__token_buffer__compact moves any written but unread tokens to the
// start of the buffer.
static inline void //
wuffs_base__token_buffer__compact(wuffs_base__token_buffer* buf) {
if (!buf || (buf->meta.ri == 0)) {
return;
}
buf->meta.pos = wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri);
size_t n = buf->meta.wi - buf->meta.ri;
if (n != 0) {
memmove(buf->data.ptr, buf->data.ptr + buf->meta.ri,
n * sizeof(wuffs_base__token));
}
buf->meta.wi = n;
buf->meta.ri = 0;
}
static inline uint64_t //
wuffs_base__token_buffer__reader_available(
const wuffs_base__token_buffer* buf) {
return buf ? buf->meta.wi - buf->meta.ri : 0;
}
static inline uint64_t //
wuffs_base__token_buffer__reader_token_position(
const wuffs_base__token_buffer* buf) {
return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri) : 0;
}
static inline uint64_t //
wuffs_base__token_buffer__writer_available(
const wuffs_base__token_buffer* buf) {
return buf ? buf->data.len - buf->meta.wi : 0;
}
static inline uint64_t //
wuffs_base__token_buffer__writer_token_position(
const wuffs_base__token_buffer* buf) {
return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.wi) : 0;
}
#ifdef __cplusplus
inline void //
wuffs_base__token_buffer::compact() {
wuffs_base__token_buffer__compact(this);
}
inline uint64_t //
wuffs_base__token_buffer::reader_available() const {
return wuffs_base__token_buffer__reader_available(this);
}
inline uint64_t //
wuffs_base__token_buffer::reader_token_position() const {
return wuffs_base__token_buffer__reader_token_position(this);
}
inline uint64_t //
wuffs_base__token_buffer::writer_available() const {
return wuffs_base__token_buffer__writer_available(this);
}
inline uint64_t //
wuffs_base__token_buffer::writer_token_position() const {
return wuffs_base__token_buffer__writer_token_position(this);
}
#endif // __cplusplus
+4
View File
@@ -256,6 +256,8 @@ func insertBaseAllPrivateH(buf *buffer) error {
buf.writeb('\n')
buf.writes(baseIOPrivateH)
buf.writeb('\n')
buf.writes(baseTokenPrivateH)
buf.writeb('\n')
buf.writes(baseMemoryPrivateH)
buf.writeb('\n')
buf.writes(baseImagePrivateH)
@@ -303,6 +305,8 @@ func insertBaseAllPublicH(buf *buffer) error {
buf.writeb('\n')
buf.writes(baseIOPublicH)
buf.writeb('\n')
buf.writes(baseTokenPublicH)
buf.writeb('\n')
buf.writes(baseMemoryPublicH)
buf.writeb('\n')
buf.writes(baseImagePublicH)
+14
View File
@@ -251,6 +251,20 @@ const baseRangePublicH = "" +
"e__rect_ie_u32 s) const {\n return wuffs_base__rect_ie_u32__equals(this, s);\n}\n\ninline wuffs_base__rect_ie_u32 //\nwuffs_base__rect_ie_u32::intersect(wuffs_base__rect_ie_u32 s) const {\n return wuffs_base__rect_ie_u32__intersect(this, s);\n}\n\ninline wuffs_base__rect_ie_u32 //\nwuffs_base__rect_ie_u32::unite(wuffs_base__rect_ie_u32 s) const {\n return wuffs_base__rect_ie_u32__unite(this, s);\n}\n\ninline bool //\nwuffs_base__rect_ie_u32::contains(uint32_t x, uint32_t y) const {\n return wuffs_base__rect_ie_u32__contains(this, x, y);\n}\n\ninline bool //\nwuffs_base__rect_ie_u32::contains_rect(wuffs_base__rect_ie_u32 s) const {\n return wuffs_base__rect_ie_u32__contains_rect(this, s);\n}\n\ninline uint32_t //\nwuffs_base__rect_ie_u32::width() const {\n return wuffs_base__rect_ie_u32__width(this);\n}\n\ninline uint32_t //\nwuffs_base__rect_ie_u32::height() const {\n return wuffs_base__rect_ie_u32__height(this);\n}\n\n#endif // __cplusplus\n" +
""
const baseTokenPrivateH = "" +
"// ---------------- Tokens\n\n" +
"" +
"// ---------------- Tokens (Utility)\n" +
""
const baseTokenPublicH = "" +
"// ---------------- Tokens\n\n#define WUFFS_BASE__TOKEN__LENGTH_MASK 0xFFFF\n#define WUFFS_BASE__TOKEN__MINOR_VALUE_MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__MAJOR_VALUE_MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MASK 0xFFFFFFFFFFFF\n\n#define WUFFS_BASE__TOKEN__LENGTH_SHIFT 0\n#define WUFFS_BASE__TOKEN__MINOR_VALUE_SHIFT 16\n#define WUFFS_BASE__TOKEN__MAJOR_VALUE_SHIFT 40\n#define WUFFS_BASE__TOKEN__VALUE_SHIFT 16\n\ntypedef struct {\n // Bits 63 .. 40 (24 bits) is the major value.\n // Bits 39 .. 16 (24 bits) is the minor value.\n // Bits 15 .. 0 (16 bits) is the length.\n uint64_t repr;\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\ntypedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;\n\nstatic inline wuffs_base__slice_token //\nwuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {\n wuffs_base__slice_token ret;\n ret.ptr = ptr;\n ret.len = len;\n return ret;\n}\n\n// wuffs_bas" +
"e__token_buffer_meta is the metadata for a\n// wuffs_base__token_buffer's data.\ntypedef struct {\n size_t wi; // Write index. Invariant: wi <= len.\n size_t ri; // Read index. Invariant: ri <= wi.\n uint64_t pos; // Position of the buffer start relative to the stream start.\n bool closed; // No further writes are expected.\n} wuffs_base__token_buffer_meta;\n\n// wuffs_base__token_buffer is a 1-dimensional buffer (a pointer and length)\n// plus additional metadata.\n//\n// A value with all fields zero is a valid, empty buffer.\ntypedef struct {\n wuffs_base__slice_token data;\n wuffs_base__token_buffer_meta meta;\n\n#ifdef __cplusplus\n inline void compact();\n inline uint64_t reader_available() const;\n inline uint64_t reader_token_position() const;\n inline uint64_t writer_available() const;\n inline uint64_t writer_token_position() const;\n#endif // __cplusplus\n\n} wuffs_base__token_buffer;\n\nstatic inline wuffs_base__token_buffer //\nwuffs_base__make_token_buffer(wuffs_base__slice_token data,\n " +
" wuffs_base__token_buffer_meta meta) {\n wuffs_base__token_buffer ret;\n ret.data = data;\n ret.meta = meta;\n return ret;\n}\n\nstatic inline wuffs_base__token_buffer_meta //\nwuffs_base__make_token_buffer_meta(size_t wi,\n size_t ri,\n uint64_t pos,\n bool closed) {\n wuffs_base__token_buffer_meta ret;\n ret.wi = wi;\n ret.ri = ri;\n ret.pos = pos;\n ret.closed = closed;\n return ret;\n}\n\nstatic inline wuffs_base__token_buffer //\nwuffs_base__empty_token_buffer() {\n wuffs_base__token_buffer ret;\n ret.data.ptr = NULL;\n ret.data.len = 0;\n ret.meta.wi = 0;\n ret.meta.ri = 0;\n ret.meta.pos = 0;\n ret.meta.closed = false;\n return ret;\n}\n\nstatic inline wuffs_base__token_buffer_meta //\nwuffs_base__empty_token_buffer_meta() {\n wuffs_base__token_buffer_meta ret;\n ret.wi = 0;\n ret.ri = 0;\n ret.pos = 0;\n ret.closed = false;\n return ret;\n}\n\n// wuffs_base__token_buffer__compact moves any written bu" +
"t unread tokens to the\n// start of the buffer.\nstatic inline void //\nwuffs_base__token_buffer__compact(wuffs_base__token_buffer* buf) {\n if (!buf || (buf->meta.ri == 0)) {\n return;\n }\n buf->meta.pos = wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri);\n size_t n = buf->meta.wi - buf->meta.ri;\n if (n != 0) {\n memmove(buf->data.ptr, buf->data.ptr + buf->meta.ri,\n n * sizeof(wuffs_base__token));\n }\n buf->meta.wi = n;\n buf->meta.ri = 0;\n}\n\nstatic inline uint64_t //\nwuffs_base__token_buffer__reader_available(\n const wuffs_base__token_buffer* buf) {\n return buf ? buf->meta.wi - buf->meta.ri : 0;\n}\n\nstatic inline uint64_t //\nwuffs_base__token_buffer__reader_token_position(\n const wuffs_base__token_buffer* buf) {\n return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri) : 0;\n}\n\nstatic inline uint64_t //\nwuffs_base__token_buffer__writer_available(\n const wuffs_base__token_buffer* buf) {\n return buf ? buf->data.len - buf->meta.wi : 0;\n}\n\nstatic inline uint64_t //\nw" +
"uffs_base__token_buffer__writer_token_position(\n const wuffs_base__token_buffer* buf) {\n return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.wi) : 0;\n}\n\n#ifdef __cplusplus\n\ninline void //\nwuffs_base__token_buffer::compact() {\n wuffs_base__token_buffer__compact(this);\n}\n\ninline uint64_t //\nwuffs_base__token_buffer::reader_available() const {\n return wuffs_base__token_buffer__reader_available(this);\n}\n\ninline uint64_t //\nwuffs_base__token_buffer::reader_token_position() const {\n return wuffs_base__token_buffer__reader_token_position(this);\n}\n\ninline uint64_t //\nwuffs_base__token_buffer::writer_available() const {\n return wuffs_base__token_buffer__writer_available(this);\n}\n\ninline uint64_t //\nwuffs_base__token_buffer::writer_token_position() const {\n return wuffs_base__token_buffer__writer_token_position(this);\n}\n\n#endif // __cplusplus\n" +
""
const baseCopyright = "" +
"// Copyright 2017 The Wuffs Authors.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// https://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n" +
""
+14 -11
View File
@@ -470,17 +470,20 @@ func isBaseRangeType(qid t.QID) bool {
}
var cTypeNames = [...]string{
t.IDI8: "int8_t",
t.IDI16: "int16_t",
t.IDI32: "int32_t",
t.IDI64: "int64_t",
t.IDU8: "uint8_t",
t.IDU16: "uint16_t",
t.IDU32: "uint32_t",
t.IDU64: "uint64_t",
t.IDBool: "bool",
t.IDIOReader: "wuffs_base__io_buffer*",
t.IDIOWriter: "wuffs_base__io_buffer*",
t.IDI8: "int8_t",
t.IDI16: "int16_t",
t.IDI32: "int32_t",
t.IDI64: "int64_t",
t.IDU8: "uint8_t",
t.IDU16: "uint16_t",
t.IDU32: "uint32_t",
t.IDU64: "uint64_t",
t.IDBool: "bool",
t.IDIOReader: "wuffs_base__io_buffer*",
t.IDIOWriter: "wuffs_base__io_buffer*",
t.IDTokenReader: "wuffs_base__token_buffer*",
t.IDTokenWriter: "wuffs_base__token_buffer*",
}
const noSuchCOperator = " no_such_C_operator "
+1 -1
View File
@@ -462,7 +462,7 @@ func (g *gen) writeFuncImplArgChecks(b *buffer, n *a.Func) error {
// TODO: Also check elements, for array-typed arguments.
switch {
case oTyp.IsIOType() || (oTyp.Decorator() == t.IDPtr):
case oTyp.IsIOTokenType() || (oTyp.Decorator() == t.IDPtr):
checks = append(checks, fmt.Sprintf("!%s%s", aPrefix, o.Name().Str(g.tm)))
case oTyp.IsRefined():
+2
View File
@@ -86,6 +86,8 @@ func genBase(out *bytes.Buffer) error {
{"base/io-public.h", "baseIOPublicH"},
{"base/range-private.h", "baseRangePrivateH"},
{"base/range-public.h", "baseRangePublicH"},
{"base/token-private.h", "baseTokenPrivateH"},
{"base/token-public.h", "baseTokenPublicH"},
}
prefixAfterEditing := []byte("// After editing this file,")
+5 -3
View File
@@ -59,7 +59,7 @@ func (g *gen) needDerivedVar(name t.ID) bool {
func (g *gen) findDerivedVars() {
for _, o := range g.currFunk.astFunc.In().Fields() {
o := o.AsField()
if !o.XType().IsIOType() || !g.needDerivedVar(o.Name()) {
if !o.XType().IsIOTokenType() || !g.needDerivedVar(o.Name()) {
continue
}
if g.currFunk.derivedVars == nil {
@@ -84,7 +84,7 @@ func (g *gen) writeLoadDerivedVar(b *buffer, hack string, prefix string, name t.
return nil
}
if !typ.IsIOType() {
if !typ.IsIOTokenType() {
return nil
}
if g.currFunk.derivedVars == nil {
@@ -143,7 +143,7 @@ func (g *gen) writeSaveDerivedVar(b *buffer, hack string, prefix string, name t.
return nil
}
if !typ.IsIOType() {
if !typ.IsIOTokenType() {
return nil
}
if g.currFunk.derivedVars == nil {
@@ -275,6 +275,8 @@ func (g *gen) writeVars(b *buffer, f *funk, inStructDecl bool) error {
if typ.IsIOType() {
b.printf("wuffs_base__io_buffer %s%s = wuffs_base__empty_io_buffer();\n", uPrefix, name)
} else if typ.IsTokenType() {
return fmt.Errorf("TODO: support token_{reader,writer} typed variables")
}
if err := g.writeCTypeName(b, typ, vPrefix, name); err != nil {
+13 -1
View File
@@ -700,7 +700,19 @@ func (n *TypeExpr) IsIdeal() bool {
}
func (n *TypeExpr) IsIOType() bool {
return n.id0 == 0 && n.id1 == t.IDBase && (n.id2 == t.IDIOReader || n.id2 == t.IDIOWriter)
return n.id0 == 0 && n.id1 == t.IDBase &&
(n.id2 == t.IDIOReader || n.id2 == t.IDIOWriter)
}
func (n *TypeExpr) IsTokenType() bool {
return n.id0 == 0 && n.id1 == t.IDBase &&
(n.id2 == t.IDTokenReader || n.id2 == t.IDTokenWriter)
}
func (n *TypeExpr) IsIOTokenType() bool {
return n.id0 == 0 && n.id1 == t.IDBase &&
(n.id2 == t.IDIOReader || n.id2 == t.IDIOWriter ||
n.id2 == t.IDTokenReader || n.id2 == t.IDTokenWriter)
}
func (n *TypeExpr) IsNullptr() bool {
+13 -7
View File
@@ -83,9 +83,11 @@ var Types = []string{
"rect_ie_u32",
"rect_ii_u32",
"status",
"io_reader",
"io_writer",
"status",
"token_writer",
"frame_config",
"image_config",
@@ -166,6 +168,14 @@ var Funcs = []string{
"range_ii_u64.intersect(r: range_ii_u64) range_ii_u64",
"range_ii_u64.unite(r: range_ii_u64) range_ii_u64",
// ---- status
// TODO: should we add is_complete?
"status.is_error() bool",
"status.is_note() bool",
"status.is_ok() bool",
"status.is_suspension() bool",
// ---- io_reader
"io_reader.can_undo_byte() bool",
@@ -310,13 +320,9 @@ var Funcs = []string{
// For now, that's all implicitly checked (i.e. hard coded).
"io_writer.copy_n32_from_history_fast!(n: u32, distance: u32) u32",
// ---- status
// ---- token_writer
// TODO: should we add is_complete?
"status.is_error() bool",
"status.is_note() bool",
"status.is_ok() bool",
"status.is_suspension() bool",
"token_writer.write_token?(value: u64[..= 0xFFFFFFFFFFFF], length: u64[..= 0xFFFF])",
// ---- frame_config
// Duration's upper bound is the maximum possible i64 value.
+12 -6
View File
@@ -54,9 +54,12 @@ var (
typeExprRectIEU32 = a.NewTypeExpr(0, t.IDBase, t.IDRectIEU32, nil, nil, nil)
typeExprRectIIU32 = a.NewTypeExpr(0, t.IDBase, t.IDRectIIU32, nil, nil, nil)
typeExprIOReader = a.NewTypeExpr(0, t.IDBase, t.IDIOReader, nil, nil, nil)
typeExprIOWriter = a.NewTypeExpr(0, t.IDBase, t.IDIOWriter, nil, nil, nil)
typeExprStatus = a.NewTypeExpr(0, t.IDBase, t.IDStatus, nil, nil, nil)
typeExprStatus = a.NewTypeExpr(0, t.IDBase, t.IDStatus, nil, nil, nil)
typeExprIOReader = a.NewTypeExpr(0, t.IDBase, t.IDIOReader, nil, nil, nil)
typeExprIOWriter = a.NewTypeExpr(0, t.IDBase, t.IDIOWriter, nil, nil, nil)
typeExprTokenReader = a.NewTypeExpr(0, t.IDBase, t.IDTokenReader, nil, nil, nil)
typeExprTokenWriter = a.NewTypeExpr(0, t.IDBase, t.IDTokenWriter, nil, nil, nil)
typeExprFrameConfig = a.NewTypeExpr(0, t.IDBase, t.IDFrameConfig, nil, nil, nil)
typeExprImageConfig = a.NewTypeExpr(0, t.IDBase, t.IDImageConfig, nil, nil, nil)
@@ -97,9 +100,12 @@ var builtInTypeMap = typeMap{
t.IDRectIEU32: typeExprRectIEU32,
t.IDRectIIU32: typeExprRectIIU32,
t.IDIOReader: typeExprIOReader,
t.IDIOWriter: typeExprIOWriter,
t.IDStatus: typeExprStatus,
t.IDStatus: typeExprStatus,
t.IDIOReader: typeExprIOReader,
t.IDIOWriter: typeExprIOWriter,
t.IDTokenReader: typeExprTokenReader,
t.IDTokenWriter: typeExprTokenWriter,
t.IDFrameConfig: typeExprFrameConfig,
t.IDImageConfig: typeExprImageConfig,
+5 -1
View File
@@ -467,7 +467,9 @@ const (
IDIOReader = ID(0x125)
IDIOWriter = ID(0x126)
IDStatus = ID(0x127)
IDUtility = ID(0x128)
IDTokenReader = ID(0x128)
IDTokenWriter = ID(0x129)
IDUtility = ID(0x12A)
IDRangeIEU32 = ID(0x130)
IDRangeIIU32 = ID(0x131)
@@ -810,6 +812,8 @@ var builtInsByID = [nBuiltInIDs]string{
IDIOReader: "io_reader",
IDIOWriter: "io_writer",
IDStatus: "status",
IDTokenReader: "token_reader",
IDTokenWriter: "token_writer",
IDUtility: "utility",
IDRangeIEU32: "range_ie_u32",
+184 -3
View File
@@ -1459,6 +1459,183 @@ wuffs_base__io_buffer::writer_io_position() const {
return wuffs_base__io_buffer__writer_io_position(this);
}
#endif // __cplusplus
// ---------------- Tokens
#define WUFFS_BASE__TOKEN__LENGTH_MASK 0xFFFF
#define WUFFS_BASE__TOKEN__MINOR_VALUE_MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__MAJOR_VALUE_MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MASK 0xFFFFFFFFFFFF
#define WUFFS_BASE__TOKEN__LENGTH_SHIFT 0
#define WUFFS_BASE__TOKEN__MINOR_VALUE_SHIFT 16
#define WUFFS_BASE__TOKEN__MAJOR_VALUE_SHIFT 40
#define WUFFS_BASE__TOKEN__VALUE_SHIFT 16
typedef struct {
// Bits 63 .. 40 (24 bits) is the major value.
// Bits 39 .. 16 (24 bits) is the minor value.
// Bits 15 .. 0 (16 bits) is the length.
uint64_t repr;
} wuffs_base__token;
static inline wuffs_base__token //
wuffs_base__make_token(uint64_t repr) {
wuffs_base__token ret;
ret.repr = repr;
return ret;
}
typedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;
static inline wuffs_base__slice_token //
wuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {
wuffs_base__slice_token ret;
ret.ptr = ptr;
ret.len = len;
return ret;
}
// wuffs_base__token_buffer_meta is the metadata for a
// wuffs_base__token_buffer's data.
typedef struct {
size_t wi; // Write index. Invariant: wi <= len.
size_t ri; // Read index. Invariant: ri <= wi.
uint64_t pos; // Position of the buffer start relative to the stream start.
bool closed; // No further writes are expected.
} wuffs_base__token_buffer_meta;
// wuffs_base__token_buffer is a 1-dimensional buffer (a pointer and length)
// plus additional metadata.
//
// A value with all fields zero is a valid, empty buffer.
typedef struct {
wuffs_base__slice_token data;
wuffs_base__token_buffer_meta meta;
#ifdef __cplusplus
inline void compact();
inline uint64_t reader_available() const;
inline uint64_t reader_token_position() const;
inline uint64_t writer_available() const;
inline uint64_t writer_token_position() const;
#endif // __cplusplus
} wuffs_base__token_buffer;
static inline wuffs_base__token_buffer //
wuffs_base__make_token_buffer(wuffs_base__slice_token data,
wuffs_base__token_buffer_meta meta) {
wuffs_base__token_buffer ret;
ret.data = data;
ret.meta = meta;
return ret;
}
static inline wuffs_base__token_buffer_meta //
wuffs_base__make_token_buffer_meta(size_t wi,
size_t ri,
uint64_t pos,
bool closed) {
wuffs_base__token_buffer_meta ret;
ret.wi = wi;
ret.ri = ri;
ret.pos = pos;
ret.closed = closed;
return ret;
}
static inline wuffs_base__token_buffer //
wuffs_base__empty_token_buffer() {
wuffs_base__token_buffer ret;
ret.data.ptr = NULL;
ret.data.len = 0;
ret.meta.wi = 0;
ret.meta.ri = 0;
ret.meta.pos = 0;
ret.meta.closed = false;
return ret;
}
static inline wuffs_base__token_buffer_meta //
wuffs_base__empty_token_buffer_meta() {
wuffs_base__token_buffer_meta ret;
ret.wi = 0;
ret.ri = 0;
ret.pos = 0;
ret.closed = false;
return ret;
}
// wuffs_base__token_buffer__compact moves any written but unread tokens to the
// start of the buffer.
static inline void //
wuffs_base__token_buffer__compact(wuffs_base__token_buffer* buf) {
if (!buf || (buf->meta.ri == 0)) {
return;
}
buf->meta.pos = wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri);
size_t n = buf->meta.wi - buf->meta.ri;
if (n != 0) {
memmove(buf->data.ptr, buf->data.ptr + buf->meta.ri,
n * sizeof(wuffs_base__token));
}
buf->meta.wi = n;
buf->meta.ri = 0;
}
static inline uint64_t //
wuffs_base__token_buffer__reader_available(
const wuffs_base__token_buffer* buf) {
return buf ? buf->meta.wi - buf->meta.ri : 0;
}
static inline uint64_t //
wuffs_base__token_buffer__reader_token_position(
const wuffs_base__token_buffer* buf) {
return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri) : 0;
}
static inline uint64_t //
wuffs_base__token_buffer__writer_available(
const wuffs_base__token_buffer* buf) {
return buf ? buf->data.len - buf->meta.wi : 0;
}
static inline uint64_t //
wuffs_base__token_buffer__writer_token_position(
const wuffs_base__token_buffer* buf) {
return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.wi) : 0;
}
#ifdef __cplusplus
inline void //
wuffs_base__token_buffer::compact() {
wuffs_base__token_buffer__compact(this);
}
inline uint64_t //
wuffs_base__token_buffer::reader_available() const {
return wuffs_base__token_buffer__reader_available(this);
}
inline uint64_t //
wuffs_base__token_buffer::reader_token_position() const {
return wuffs_base__token_buffer__reader_token_position(this);
}
inline uint64_t //
wuffs_base__token_buffer::writer_available() const {
return wuffs_base__token_buffer__writer_available(this);
}
inline uint64_t //
wuffs_base__token_buffer::writer_token_position() const {
return wuffs_base__token_buffer__writer_token_position(this);
}
#endif // __cplusplus
// ---------------- Memory Allocation
@@ -4843,7 +5020,7 @@ sizeof__wuffs_json__decoder();
WUFFS_BASE__MAYBE_STATIC wuffs_base__status //
wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
wuffs_base__io_buffer* a_dst,
wuffs_base__token_buffer* a_dst,
wuffs_base__io_buffer* a_src);
// ---------------- Struct Definitions
@@ -4908,7 +5085,7 @@ struct wuffs_json__decoder__struct {
}
inline wuffs_base__status //
decode_tokens(wuffs_base__io_buffer* a_dst, wuffs_base__io_buffer* a_src) {
decode_tokens(wuffs_base__token_buffer* a_dst, wuffs_base__io_buffer* a_src) {
return wuffs_json__decoder__decode_tokens(this, a_dst, a_src);
}
@@ -6065,6 +6242,10 @@ wuffs_base__io_writer__set(wuffs_base__io_buffer* b,
#define wuffs_base__utility__empty_io_reader wuffs_base__empty_io_reader
#define wuffs_base__utility__empty_io_writer wuffs_base__empty_io_writer
// ---------------- Tokens
// ---------------- Tokens (Utility)
// ---------------- Memory Allocation
// ---------------- Images
@@ -17906,7 +18087,7 @@ sizeof__wuffs_json__decoder() {
WUFFS_BASE__MAYBE_STATIC wuffs_base__status //
wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
wuffs_base__io_buffer* a_dst,
wuffs_base__token_buffer* a_dst,
wuffs_base__io_buffer* a_src) {
if (!self) {
return wuffs_base__make_status(wuffs_base__error__bad_receiver);
+1 -1
View File
@@ -17,6 +17,6 @@
pub struct decoder?(
)
pub func decoder.decode_tokens?(dst: base.io_writer, src: base.io_reader) {
pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
return ok
}
+2 -2
View File
@@ -77,8 +77,8 @@ const char* test_wuffs_json_decode_tokens() {
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
wuffs_base__io_buffer got = ((wuffs_base__io_buffer){
.data = global_got_slice,
wuffs_base__token_buffer got = ((wuffs_base__token_buffer){
.data = global_got_token_slice,
});
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = global_src_slice,
+16
View File
@@ -21,6 +21,7 @@
#include <unistd.h>
#define BUFFER_SIZE (64 * 1024 * 1024)
#define TOKEN_BUFFER_SIZE (8 * 1024)
#define WUFFS_TESTLIB_ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))
@@ -30,12 +31,18 @@ uint8_t global_work_array[BUFFER_SIZE];
uint8_t global_src_array[BUFFER_SIZE];
uint8_t global_pixel_array[BUFFER_SIZE];
wuffs_base__token global_got_token_array[TOKEN_BUFFER_SIZE];
wuffs_base__token global_want_token_array[TOKEN_BUFFER_SIZE];
wuffs_base__slice_u8 global_got_slice;
wuffs_base__slice_u8 global_want_slice;
wuffs_base__slice_u8 global_work_slice;
wuffs_base__slice_u8 global_src_slice;
wuffs_base__slice_u8 global_pixel_slice;
wuffs_base__slice_token global_got_token_slice;
wuffs_base__slice_token global_want_token_slice;
void wuffs_testlib__initialize_global_xxx_slices() {
global_got_slice = ((wuffs_base__slice_u8){
.ptr = global_got_array,
@@ -57,6 +64,15 @@ void wuffs_testlib__initialize_global_xxx_slices() {
.ptr = global_pixel_array,
.len = BUFFER_SIZE,
});
global_got_token_slice = ((wuffs_base__slice_token){
.ptr = global_got_token_array,
.len = TOKEN_BUFFER_SIZE,
});
global_want_token_slice = ((wuffs_base__slice_token){
.ptr = global_want_token_array,
.len = TOKEN_BUFFER_SIZE,
});
}
char fail_msg[65536] = {0};