Add tokens

This commit is contained in:
Nigel Tao committed 2020-02-11 12:02:58 +11:00
1 parent 7a142168be
commit 737e31ff3a
17 files changed
+498 -36

No files matched your search

+184 -3
View File
@@ -1459,6 +1459,183 @@ wuffs_base__io_buffer::writer_io_position() const {
return wuffs_base__io_buffer__writer_io_position(this);
}
#endif // __cplusplus
// ---------------- Tokens
#define WUFFS_BASE__TOKEN__LENGTH_MASK 0xFFFF
#define WUFFS_BASE__TOKEN__MINOR_VALUE_MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__MAJOR_VALUE_MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MASK 0xFFFFFFFFFFFF
#define WUFFS_BASE__TOKEN__LENGTH_SHIFT 0
#define WUFFS_BASE__TOKEN__MINOR_VALUE_SHIFT 16
#define WUFFS_BASE__TOKEN__MAJOR_VALUE_SHIFT 40
#define WUFFS_BASE__TOKEN__VALUE_SHIFT 16
typedef struct {
// Bits 63 .. 40 (24 bits) is the major value.
// Bits 39 .. 16 (24 bits) is the minor value.
// Bits 15 .. 0 (16 bits) is the length.
uint64_t repr;
} wuffs_base__token;
static inline wuffs_base__token //
wuffs_base__make_token(uint64_t repr) {
wuffs_base__token ret;
ret.repr = repr;
return ret;
}
typedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;
static inline wuffs_base__slice_token //
wuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {
wuffs_base__slice_token ret;
ret.ptr = ptr;
ret.len = len;
return ret;
}
// wuffs_base__token_buffer_meta is the metadata for a
// wuffs_base__token_buffer's data.
typedef struct {
size_t wi; // Write index. Invariant: wi <= len.
size_t ri; // Read index. Invariant: ri <= wi.
uint64_t pos; // Position of the buffer start relative to the stream start.
bool closed; // No further writes are expected.
} wuffs_base__token_buffer_meta;
// wuffs_base__token_buffer is a 1-dimensional buffer (a pointer and length)
// plus additional metadata.
//
// A value with all fields zero is a valid, empty buffer.
typedef struct {
wuffs_base__slice_token data;
wuffs_base__token_buffer_meta meta;
#ifdef __cplusplus
inline void compact();
inline uint64_t reader_available() const;
inline uint64_t reader_token_position() const;
inline uint64_t writer_available() const;
inline uint64_t writer_token_position() const;
#endif // __cplusplus
} wuffs_base__token_buffer;
static inline wuffs_base__token_buffer //
wuffs_base__make_token_buffer(wuffs_base__slice_token data,
wuffs_base__token_buffer_meta meta) {
wuffs_base__token_buffer ret;
ret.data = data;
ret.meta = meta;
return ret;
}
static inline wuffs_base__token_buffer_meta //
wuffs_base__make_token_buffer_meta(size_t wi,
size_t ri,
uint64_t pos,
bool closed) {
wuffs_base__token_buffer_meta ret;
ret.wi = wi;
ret.ri = ri;
ret.pos = pos;
ret.closed = closed;
return ret;
}
static inline wuffs_base__token_buffer //
wuffs_base__empty_token_buffer() {
wuffs_base__token_buffer ret;
ret.data.ptr = NULL;
ret.data.len = 0;
ret.meta.wi = 0;
ret.meta.ri = 0;
ret.meta.pos = 0;
ret.meta.closed = false;
return ret;
}
static inline wuffs_base__token_buffer_meta //
wuffs_base__empty_token_buffer_meta() {
wuffs_base__token_buffer_meta ret;
ret.wi = 0;
ret.ri = 0;
ret.pos = 0;
ret.closed = false;
return ret;
}
// wuffs_base__token_buffer__compact moves any written but unread tokens to the
// start of the buffer.
static inline void //
wuffs_base__token_buffer__compact(wuffs_base__token_buffer* buf) {
if (!buf || (buf->meta.ri == 0)) {
return;
}
buf->meta.pos = wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri);
size_t n = buf->meta.wi - buf->meta.ri;
if (n != 0) {
memmove(buf->data.ptr, buf->data.ptr + buf->meta.ri,
n * sizeof(wuffs_base__token));
}
buf->meta.wi = n;
buf->meta.ri = 0;
}
static inline uint64_t //
wuffs_base__token_buffer__reader_available(
const wuffs_base__token_buffer* buf) {
return buf ? buf->meta.wi - buf->meta.ri : 0;
}
static inline uint64_t //
wuffs_base__token_buffer__reader_token_position(
const wuffs_base__token_buffer* buf) {
return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri) : 0;
}
static inline uint64_t //
wuffs_base__token_buffer__writer_available(
const wuffs_base__token_buffer* buf) {
return buf ? buf->data.len - buf->meta.wi : 0;
}
static inline uint64_t //
wuffs_base__token_buffer__writer_token_position(
const wuffs_base__token_buffer* buf) {
return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.wi) : 0;
}
#ifdef __cplusplus
inline void //
wuffs_base__token_buffer::compact() {
wuffs_base__token_buffer__compact(this);
}
inline uint64_t //
wuffs_base__token_buffer::reader_available() const {
return wuffs_base__token_buffer__reader_available(this);
}
inline uint64_t //
wuffs_base__token_buffer::reader_token_position() const {
return wuffs_base__token_buffer__reader_token_position(this);
}
inline uint64_t //
wuffs_base__token_buffer::writer_available() const {
return wuffs_base__token_buffer__writer_available(this);
}
inline uint64_t //
wuffs_base__token_buffer::writer_token_position() const {
return wuffs_base__token_buffer__writer_token_position(this);
}
#endif // __cplusplus
// ---------------- Memory Allocation
@@ -4843,7 +5020,7 @@ sizeof__wuffs_json__decoder();
WUFFS_BASE__MAYBE_STATIC wuffs_base__status //
wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
wuffs_base__io_buffer* a_dst,
wuffs_base__token_buffer* a_dst,
wuffs_base__io_buffer* a_src);
// ---------------- Struct Definitions
@@ -4908,7 +5085,7 @@ struct wuffs_json__decoder__struct {
}
inline wuffs_base__status //
decode_tokens(wuffs_base__io_buffer* a_dst, wuffs_base__io_buffer* a_src) {
decode_tokens(wuffs_base__token_buffer* a_dst, wuffs_base__io_buffer* a_src) {
return wuffs_json__decoder__decode_tokens(this, a_dst, a_src);
}
@@ -6065,6 +6242,10 @@ wuffs_base__io_writer__set(wuffs_base__io_buffer* b,
#define wuffs_base__utility__empty_io_reader wuffs_base__empty_io_reader
#define wuffs_base__utility__empty_io_writer wuffs_base__empty_io_writer
// ---------------- Tokens
// ---------------- Tokens (Utility)
// ---------------- Memory Allocation
// ---------------- Images
@@ -17906,7 +18087,7 @@ sizeof__wuffs_json__decoder() {
WUFFS_BASE__MAYBE_STATIC wuffs_base__status //
wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
wuffs_base__io_buffer* a_dst,
wuffs_base__token_buffer* a_dst,
wuffs_base__io_buffer* a_src) {
if (!self) {
return wuffs_base__make_status(wuffs_base__error__bad_receiver);