Add aux library

This commit is contained in:
Nigel Tao committed 2020-08-04 23:43:38 +10:00
1 parent a4f2bbb249
commit cf6c578973
14 files changed
+1278 -449

No files matched your search

+139 -443
View File
@@ -25,9 +25,9 @@ See the "const char* g_usage" string below for details.
This program uses Wuffs' JSON decoder at a relatively high level, building
in-memory representations of JSON 'things' (e.g. numbers, strings, objects).
After the entire input has been converted, walking the tree prints the output
(in sorted order). The core conversion mechanism is to call JsonThing::parse,
which consumes a variable number of tokens (the output of Wuffs' JSON decoder).
JsonThing::parse can call itself recursively, as JSON values can nest.
(in sorted order). The wuffs_aux::DecodeJson library function converts the
lower level token stream to higher level callbacks. This .cc file deals only
with those callbacks, not with tokens per se.
This approach is centered around JSON things. Each JSON thing comprises one or
more JSON tokens.
@@ -61,9 +61,7 @@ for a C++ compiler $CXX, such as clang++ or g++.
#error "This C++ program requires -std=c++11 or later"
#endif
#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdio.h>
#include <iostream>
#include <map>
@@ -86,6 +84,8 @@ for a C++ compiler $CXX, such as clang++ or g++.
// modules we use makes that process explicit. Preprocessing means that such
// code simply isn't compiled.
#define WUFFS_CONFIG__MODULES
#define WUFFS_CONFIG__MODULE__AUX__BASE
#define WUFFS_CONFIG__MODULE__AUX__JSON
#define WUFFS_CONFIG__MODULE__BASE
#define WUFFS_CONFIG__MODULE__JSON
@@ -179,6 +179,8 @@ static const char* g_usage =
// ----
std::vector<uint32_t> g_quirks;
struct {
int remaining_argc;
char** remaining_argv;
@@ -229,7 +231,7 @@ parse_flags(int argc, char** argv) {
return g_usage;
}
if (!strcmp(arg, "input-json-extra-comma")) {
g_flags.input_json_extra_comma = true;
g_quirks.push_back(WUFFS_JSON__QUIRK_ALLOW_EXTRA_COMMA);
continue;
}
if (!strcmp(arg, "strict-json-pointer-syntax")) {
@@ -247,152 +249,8 @@ parse_flags(int argc, char** argv) {
// ----
#define WORK_BUFFER_ARRAY_SIZE \
WUFFS_JSON__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE
#ifndef SRC_BUFFER_ARRAY_SIZE
#define SRC_BUFFER_ARRAY_SIZE (4 * 1024)
#endif
#ifndef TOKEN_BUFFER_ARRAY_SIZE
#define TOKEN_BUFFER_ARRAY_SIZE (1 * 1024)
#endif
class TokenStream {
public:
struct Result {
std::string status_msg;
wuffs_base__token token;
// src_data is a sub-slice of m_src (a slice is a pointer-length pair).
// Calling TokenStream::peek or TokenStream::next may change the backing
// array's contents, so handling a TokenStream::Result may require copying
// this src_data slice's contents.
wuffs_base__slice_u8 src_data;
Result(std::string s)
: status_msg(s),
token(wuffs_base__make_token(0)),
src_data(wuffs_base__empty_slice_u8()) {}
Result(std::string s, wuffs_base__token t, wuffs_base__slice_u8 d)
: status_msg(s), token(t), src_data(d) {}
};
TokenStream(int input_file_descriptor)
: m_status(wuffs_base__make_status(nullptr)),
m_src(wuffs_base__make_io_buffer(
wuffs_base__make_slice_u8(m_src_array, SRC_BUFFER_ARRAY_SIZE),
wuffs_base__empty_io_buffer_meta())),
m_tok(wuffs_base__make_token_buffer(
wuffs_base__make_slice_token(m_tok_array, TOKEN_BUFFER_ARRAY_SIZE),
wuffs_base__empty_token_buffer_meta())),
m_input_file_descriptor(input_file_descriptor),
m_curr_token_end_src_index(0) {
m_status =
m_dec.initialize(sizeof__wuffs_json__decoder(), WUFFS_VERSION, 0);
if (m_status.is_ok()) {
// Uncomment this line to enable the WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X
// option, discussed in a separate comment.
// m_dec.set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X, true);
if (g_flags.input_json_extra_comma) {
m_dec.set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_EXTRA_COMMA, true);
}
}
}
Result peek() { return peek_or_next(false); }
Result next() { return peek_or_next(true); }
private:
Result peek_or_next(bool next) {
while (m_tok.meta.ri >= m_tok.meta.wi) {
if (m_status.repr == nullptr) {
// No-op.
} else if (m_status.repr == wuffs_base__suspension__short_read) {
if (m_curr_token_end_src_index != m_src.meta.ri) {
return Result(
"TokenStream: internal error: inconsistent src indexes");
}
const char* z = read_src();
m_curr_token_end_src_index = m_src.meta.ri;
if (z) {
return Result(z);
}
} else if (m_status.repr == wuffs_base__suspension__short_write) {
m_tok.compact();
} else {
return Result(m_status.message());
}
m_status =
m_dec.decode_tokens(&m_tok, &m_src,
wuffs_base__make_slice_u8(
m_work_buffer_array, WORK_BUFFER_ARRAY_SIZE));
}
wuffs_base__token t = m_tok.data.ptr[m_tok.meta.ri];
size_t i = m_curr_token_end_src_index;
uint64_t n = t.length();
if ((m_src.meta.ri < i) || ((m_src.meta.ri - i) < n)) {
return Result("TokenStream: internal error: inconsistent src indexes");
}
if (next) {
m_tok.meta.ri++;
m_curr_token_end_src_index += n;
}
return Result("", t, wuffs_base__make_slice_u8(m_src.data.ptr + i, n));
}
const char* //
read_src() {
if (m_src.meta.closed) {
return "main: internal error: read requested on a closed source";
}
m_src.compact();
if (m_src.meta.wi >= m_src.data.len) {
return "main: src buffer is full";
}
while (true) {
ssize_t n = read(m_input_file_descriptor, m_src.writer_pointer(),
m_src.writer_length());
if (n >= 0) {
m_src.meta.wi += n;
m_src.meta.closed = n == 0;
break;
} else if (errno != EINTR) {
return strerror(errno);
}
}
return nullptr;
}
wuffs_base__status m_status;
wuffs_base__io_buffer m_src;
wuffs_base__token_buffer m_tok;
int m_input_file_descriptor;
// m_curr_token_end_src_index is the m_src.data.ptr index of the end of the
// current token. An invariant is that (m_curr_token_end_src_index <=
// m_src.meta.ri).
size_t m_curr_token_end_src_index;
wuffs_base__token m_tok_array[TOKEN_BUFFER_ARRAY_SIZE];
uint8_t m_src_array[SRC_BUFFER_ARRAY_SIZE];
#if WORK_BUFFER_ARRAY_SIZE > 0
uint8_t m_work_buffer_array[WORK_BUFFER_ARRAY_SIZE];
#else
// Not all C/C++ compilers support 0-length arrays.
uint8_t m_work_buffer_array[1];
#endif
wuffs_json__decoder m_dec;
};
// ----
class JsonThing {
public:
struct Result;
using Vector = std::vector<JsonThing>;
// We use a std::map in this example program to avoid dependencies outside of
@@ -421,289 +279,8 @@ class JsonThing {
Vector a;
Map o;
} value;
static JsonThing::Result parse(TokenStream& ts);
private:
static JsonThing::Result parse_array(TokenStream& ts);
static JsonThing::Result parse_literal(TokenStream::Result tsr);
static JsonThing::Result parse_number(TokenStream::Result tsr);
static JsonThing::Result parse_object(TokenStream& ts);
static JsonThing::Result parse_string(TokenStream& ts,
TokenStream::Result tsr);
};
struct JsonThing::Result {
std::string status_msg;
JsonThing thing;
Result(std::string s) : status_msg(s), thing(JsonThing()) {}
Result(std::string s, JsonThing t) : status_msg(s), thing(t) {}
};
JsonThing::Result //
JsonThing::parse(TokenStream& ts) {
while (true) {
TokenStream::Result tsr = ts.next();
if (!tsr.status_msg.empty()) {
return Result(std::move(tsr.status_msg));
}
int64_t vbc = tsr.token.value_base_category();
uint64_t vbd = tsr.token.value_base_detail();
switch (vbc) {
case WUFFS_BASE__TOKEN__VBC__FILLER:
continue;
case WUFFS_BASE__TOKEN__VBC__STRUCTURE:
if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH) {
if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) {
return parse_array(ts);
} else if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT) {
return parse_object(ts);
}
}
break;
case WUFFS_BASE__TOKEN__VBC__STRING:
return parse_string(ts, tsr);
case WUFFS_BASE__TOKEN__VBC__LITERAL:
return parse_literal(tsr);
case WUFFS_BASE__TOKEN__VBC__NUMBER:
return parse_number(tsr);
}
return Result("main: internal error: unexpected token");
}
}
JsonThing::Result //
JsonThing::parse_array(TokenStream& ts) {
JsonThing jt;
jt.kind = Kind::Array;
while (true) {
TokenStream::Result tsr = ts.peek();
if (!tsr.status_msg.empty()) {
return Result(std::move(tsr.status_msg));
}
int64_t vbc = tsr.token.value_base_category();
uint64_t vbd = tsr.token.value_base_detail();
if (vbc == WUFFS_BASE__TOKEN__VBC__FILLER) {
ts.next();
continue;
} else if ((vbc == WUFFS_BASE__TOKEN__VBC__STRUCTURE) &&
(vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP)) {
ts.next();
break;
}
JsonThing::Result jtr = JsonThing::parse(ts);
if (!jtr.status_msg.empty()) {
return Result(std::move(jtr.status_msg));
}
jt.value.a.push_back(std::move(jtr.thing));
}
return Result("", jt);
}
JsonThing::Result //
JsonThing::parse_literal(TokenStream::Result tsr) {
uint64_t vbd = tsr.token.value_base_detail();
if (vbd & WUFFS_BASE__TOKEN__VBD__LITERAL__NULL) {
JsonThing jt;
jt.kind = Kind::Null;
return Result("", jt);
} else if (vbd & WUFFS_BASE__TOKEN__VBD__LITERAL__FALSE) {
JsonThing jt;
jt.kind = Kind::Bool;
jt.value.b = false;
return Result("", jt);
} else if (vbd & WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE) {
JsonThing jt;
jt.kind = Kind::Bool;
jt.value.b = true;
return Result("", jt);
}
return Result("main: internal error: unexpected token");
}
JsonThing::Result //
JsonThing::parse_number(TokenStream::Result tsr) {
// Parsing the number from its string representation (converting from "123"
// to 123) isn't necessary for the jsonfindptrs program, but if you're
// copy/pasting this JsonThing code, here's how to do it.
uint64_t vbd = tsr.token.value_base_detail();
if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT) {
if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED) {
static constexpr int64_t m = 0x001FFFFFFFFFFFFF; // ((1<<53) - 1).
wuffs_base__result_i64 r = wuffs_base__parse_number_i64(
tsr.src_data, WUFFS_BASE__PARSE_NUMBER_XXX__DEFAULT_OPTIONS);
if (!r.status.is_ok()) {
return Result(r.status.message());
} else if ((r.value < -m) || (+m < r.value)) {
return Result(wuffs_base__error__out_of_bounds);
}
JsonThing jt;
jt.kind = Kind::Int64;
jt.value.i = r.value;
return Result("", jt);
} else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT) {
wuffs_base__result_f64 r = wuffs_base__parse_number_f64(
tsr.src_data, WUFFS_BASE__PARSE_NUMBER_XXX__DEFAULT_OPTIONS);
if (!r.status.is_ok()) {
return Result(r.status.message());
}
JsonThing jt;
jt.kind = Kind::Float64;
jt.value.f = r.value;
return Result("", jt);
}
}
return Result("main: internal error: unexpected number");
}
JsonThing::Result //
JsonThing::parse_object(TokenStream& ts) {
JsonThing jt;
jt.kind = Kind::Object;
std::string key;
bool have_key = false;
while (true) {
TokenStream::Result tsr = ts.peek();
if (!tsr.status_msg.empty()) {
return Result(std::move(tsr.status_msg));
}
int64_t vbc = tsr.token.value_base_category();
uint64_t vbd = tsr.token.value_base_detail();
if (vbc == WUFFS_BASE__TOKEN__VBC__FILLER) {
ts.next();
continue;
} else if ((vbc == WUFFS_BASE__TOKEN__VBC__STRUCTURE) &&
(vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP)) {
ts.next();
break;
}
JsonThing::Result jtr = JsonThing::parse(ts);
if (!jtr.status_msg.empty()) {
return Result(std::move(jtr.status_msg));
}
if (have_key) {
have_key = false;
auto iter = jt.value.o.find(key);
if (iter == jt.value.o.end()) {
jt.value.o.insert(
iter, Map::value_type(std::move(key), std::move(jtr.thing)));
} else {
return Result("main: duplicate key: " + key);
}
} else if (jtr.thing.kind == Kind::String) {
have_key = true;
key = std::move(jtr.thing.value.s);
} else {
return Result("main: internal error: unexpected non-string key");
}
}
if (have_key) {
return Result("main: internal error: unpaired key");
}
return Result("", jt);
}
JsonThing::Result //
JsonThing::parse_string(TokenStream& ts, TokenStream::Result tsr) {
JsonThing jt;
jt.kind = Kind::String;
while (true) {
int64_t vbc = tsr.token.value_base_category();
uint64_t vbd = tsr.token.value_base_detail();
switch (vbc) {
case WUFFS_BASE__TOKEN__VBC__STRING: {
if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP) {
// No-op.
} else if (vbd &
WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY) {
const char* ptr = // Convert from (uint8_t*).
static_cast<const char*>(static_cast<void*>(tsr.src_data.ptr));
jt.value.s.append(ptr, tsr.src_data.len);
} else if (
vbd &
WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X) {
// We shouldn't get here unless we enable the
// WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X option. The jsonfindptrs
// program doesn't enable that by default, but if you're copy/pasting
// this JsonThing code and your program does enable that option,
// here's how to handle it.
//
// As per the quirk documentation, there are two options for how to
// interpret a backslash-x: as a byte or as a Unicode code point.
// This implementation chooses as a byte.
wuffs_base__slice_u8 encoded = tsr.src_data;
if (encoded.len & 3) {
return Result(
"main: internal error: \\x token length not a multiple of 4",
JsonThing());
}
while (encoded.len) {
uint8_t decoded[64];
const bool src_closed = true;
wuffs_base__transform__output o = wuffs_base__base_16__decode4(
wuffs_base__make_slice_u8(&decoded[0], 64), encoded, src_closed,
WUFFS_BASE__BASE_16__DEFAULT_OPTIONS);
if (o.status.is_error()) {
return Result(o.status.message(), JsonThing());
} else if ((o.num_dst > 64) || (o.num_src > encoded.len)) {
return Result(
"main: internal error: inconsistent hexadecimal decoding",
JsonThing());
}
const char* ptr = // Convert from (uint8_t*).
static_cast<const char*>(static_cast<void*>(&decoded[0]));
jt.value.s.append(ptr, o.num_dst);
encoded.ptr += o.num_src;
encoded.len -= o.num_src;
}
} else {
return Result(
"main: internal error: unexpected string-token conversion",
JsonThing());
}
break;
}
case WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT: {
uint8_t u[WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL];
size_t n = wuffs_base__utf_8__encode(
wuffs_base__make_slice_u8(&u[0],
WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL),
vbd);
const char* ptr = // Convert from (uint8_t*).
static_cast<const char*>(static_cast<void*>(&u[0]));
jt.value.s.append(ptr, n);
break;
}
default:
return Result("main: internal error: unexpected token");
}
if (!tsr.token.continued()) {
break;
}
tsr = ts.next();
if (!tsr.status_msg.empty()) {
return Result(std::move(tsr.status_msg));
}
}
return Result("", jt);
}
// ----
std::string //
@@ -776,27 +353,146 @@ print_json_pointers(JsonThing& jt, std::string s, uint32_t depth) {
return "";
}
// ----
class Callbacks : public wuffs_aux::DecodeJsonCallbacks {
public:
struct Entry {
Entry(JsonThing&& jt)
: thing(std::move(jt)), has_map_key(false), map_key() {}
JsonThing thing;
bool has_map_key;
std::string map_key;
};
Callbacks() = default;
std::string Append(JsonThing&& jt) {
if (m_stack.empty()) {
m_stack.push_back(Entry(std::move(jt)));
return "";
}
Entry& top = m_stack.back();
switch (top.thing.kind) {
case JsonThing::Kind::Array:
top.thing.value.a.push_back(std::move(jt));
return "";
case JsonThing::Kind::Object:
if (top.has_map_key) {
top.has_map_key = false;
auto iter = top.thing.value.o.find(top.map_key);
if (iter != top.thing.value.o.end()) {
return "main: duplicate key: " + top.map_key;
}
top.thing.value.o.insert(
iter, JsonThing::Map::value_type(std::move(top.map_key),
std::move(jt)));
return "";
} else if (jt.kind == JsonThing::Kind::String) {
top.has_map_key = true;
top.map_key = std::move(jt.value.s);
return "";
}
return "main: internal error: non-string map key";
}
return "main: internal error: non-container stack entry";
}
virtual std::string AppendNull() {
JsonThing jt;
jt.kind = JsonThing::Kind::Null;
return Append(std::move(jt));
}
virtual std::string AppendBool(bool val) {
JsonThing jt;
jt.kind = JsonThing::Kind::Bool;
jt.value.b = val;
return Append(std::move(jt));
}
virtual std::string AppendI64(int64_t val) {
JsonThing jt;
jt.kind = JsonThing::Kind::Int64;
jt.value.i = val;
return Append(std::move(jt));
}
virtual std::string AppendF64(double val) {
JsonThing jt;
jt.kind = JsonThing::Kind::Float64;
jt.value.f = val;
return Append(std::move(jt));
}
virtual std::string AppendString(std::string&& val) {
JsonThing jt;
jt.kind = JsonThing::Kind::String;
jt.value.s = std::move(val);
return Append(std::move(jt));
}
virtual std::string Push(uint32_t flags) {
if (flags & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) {
JsonThing jt;
jt.kind = JsonThing::Kind::Array;
m_stack.push_back(std::move(jt));
return "";
} else if (flags & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT) {
JsonThing jt;
jt.kind = JsonThing::Kind::Object;
m_stack.push_back(std::move(jt));
return "";
}
return "main: internal error: bad push";
}
virtual std::string Pop(uint32_t flags) {
if (m_stack.empty()) {
return "main: internal error: bad pop";
}
JsonThing jt = std::move(m_stack.back().thing);
m_stack.pop_back();
return Append(std::move(jt));
}
virtual void Done(wuffs_aux::DecodeJsonResult& result,
wuffs_aux::sync_io::Input& input,
wuffs_aux::IOBuffer& buffer) {
if (!result.error_message.empty()) {
return;
} else if (m_stack.size() != 1) {
result.error_message = "main: internal error: bad depth";
return;
}
result.error_message = print_json_pointers(m_stack.back().thing, "", 0);
}
private:
std::vector<Entry> m_stack;
};
// ----
std::string //
main1(int argc, char** argv) {
TRY(parse_flags(argc, argv));
int input_file_descriptor = 0; // A 0 default means stdin.
FILE* in = stdin;
if (g_flags.remaining_argc > 1) {
return g_usage;
} else if (g_flags.remaining_argc == 1) {
const char* arg = g_flags.remaining_argv[0];
input_file_descriptor = open(arg, O_RDONLY);
if (input_file_descriptor < 0) {
return std::string("main: cannot read ") + arg + ": " + strerror(errno);
in = fopen(g_flags.remaining_argv[0], "r");
if (!in) {
return std::string("main: cannot read input file");
}
}
TokenStream ts(input_file_descriptor);
JsonThing::Result jtr = JsonThing::parse(ts);
if (!jtr.status_msg.empty()) {
return jtr.status_msg;
}
return print_json_pointers(jtr.thing, "", 0);
return wuffs_aux::DecodeJson(
Callbacks(), wuffs_aux::sync_io::FileInput(in),
wuffs_base__make_slice_u32(g_quirks.data(), g_quirks.size()))
.error_message;
}
// ----