diff --git a/cmd/wuffs-c/release.go b/cmd/wuffs-c/release.go index d1e93548..89cb6719 100644 --- a/cmd/wuffs-c/release.go +++ b/cmd/wuffs-c/release.go @@ -24,6 +24,8 @@ import ( "sort" "strings" + "github.com/google/wuffs/internal/cgen/data" + cf "github.com/google/wuffs/cmd/commonflags" ) @@ -105,7 +107,16 @@ func doGenrelease(args []string) error { } } - out.Write(grImplStartsHere[1:]) // [1:] skips the initial '\n'. + out.WriteString("#if defined(__cplusplus) && (__cplusplus >= 201103L)\n\n") + out.WriteString(data.AuxBaseHh) + out.WriteString("\n") + for _, f := range data.AuxNonBaseHhFiles { + out.WriteString(f) + out.WriteString("\n") + } + out.WriteString("#endif // defined(__cplusplus) && (__cplusplus >= 201103L)\n") + + out.Write(grImplStartsHere) out.WriteString("\n") h.seen = map[string]bool{} @@ -115,6 +126,15 @@ func doGenrelease(args []string) error { } } + out.WriteString("#if defined(__cplusplus) && (__cplusplus >= 201103L)\n\n") + out.WriteString(data.AuxBaseCc) + out.WriteString("\n") + for _, f := range data.AuxNonBaseCcFiles { + out.WriteString(f) + out.WriteString("\n") + } + out.WriteString("#endif // defined(__cplusplus) && (__cplusplus >= 201103L)\n\n") + out.Write(grImplEndsHere) out.WriteString(grPragmaPop) out.WriteString("#endif // WUFFS_INCLUDE_GUARD\n") diff --git a/doc/changelog.md b/doc/changelog.md index 8f045550..25547784 100644 --- a/doc/changelog.md +++ b/doc/changelog.md @@ -6,6 +6,7 @@ - Added `WUFFS_BASE__PIXEL_BLEND__SRC_OVER`. - Added `WUFFS_BASE__PIXEL_FORMAT__BGR_565`. - Added `WUFFS_CONFIG__MODULE__BASE__ETC` sub-modules. +- Added `aux` library. - Added `base` library support for UTF-8. - Added `base` library support for `atoi`-like string conversion. - Added `endwhile` syntax. diff --git a/doc/glossary.md b/doc/glossary.md index 33b9f3b7..af399e12 100644 --- a/doc/glossary.md +++ b/doc/glossary.md @@ -1,5 +1,10 @@ # Glossary +#### Auxiliary Code + +Additional library code that is not written in the Wuffs programming language. +See the [auxiliary code](/doc/note/auxiliary-code.md) note for more details. + #### Axiom A named rule for asserting new facts. See the diff --git a/doc/note/auxiliary-code.md b/doc/note/auxiliary-code.md new file mode 100644 index 00000000..136465c2 --- /dev/null +++ b/doc/note/auxiliary-code.md @@ -0,0 +1,47 @@ +# Auxiliary Code + +Wuffs is a [memory-safe](/doc/note/memory-safety.md) programming language, +achieving that safety in part because Wuffs code doesn't even have the +*capability* to dynamically allocate and free memory. Wuffs code is also +transpiled to C (which is very portable and easy to bind to other, higher-level +languages), but C lacks modern conveniences like a built-in string type that's +safe and easy to use. + +Wuffs' C/C++ form (a "single file library") also contains auxiliary C++ code +(in the `wuffs_aux` namespace) that compensates for that. For example, the JSON +decoder that is written in the memory-safe Wuffs language works with low-level +[tokens](/doc/note/tokens.md). The high-level `wuffs_aux::DecodeJson` auxiliary +function instead uses e.g. a `std::string` and a `double` for the JSON inputs +`"foo\tbar"` and `0.3`. This is certainly a more *convenient* API for the +programmer who uses Wuffs-the-library, but there are several trade-offs: + +- Technically, `0.99999999999999999` and `1.0` are different JSON values, and + the low-level Wuffs JSON decoder can distinguish them, but the high-level + `wuffs_aux` JSON decoder will treat them identically (as a `double`). +- The low-level API is more modular and can work with a variety of + StringToDouble implementations (which convert the string `0.3` to the number + `0.3`). The high-level API always uses Wuffs' own StringToDouble + implementation, which makes one particular choice on the [binary size versus + runtime performance](https://github.com/google/double-conversion/issues/137) + frontier. Other choices are viable too, especially when integrating with an + existing C/C++ project that already uses another StringToDouble library. +- In the worst case (when the JSON input is just one giant string), `wuffs_aux` + requires `O(N)` memory, where `N` is the input length. In comparison, the + [example/jsonptr](/example/jsonptr/jsonptr.cc) program, which works with the + low-level token API, can process arbitrarily long input in `O(1)` memory. +- The auxiliary code is hand-written C++. It's carefully written and there's + [not a lot of it](/internal/cgen/aux), but unlike code written in the Wuffs + language, its memory-safety is not enforced by the Wuffs toolchain. +- For simplicity, it assumes that all I/O can be performed synchronously. This + is trivially true if the input is already entirely in memory (e.g. as a + `std::vector` or a `std::string`). However, the auxiliary code should not be + used on e.g. the GUI main thread if it could wait for network I/O (and make + the GUI unresponsive). In such cases, use the low-level Wuffs API instead. + +Similarly, decoding an image using the written-in-Wuffs low-level API involves +[multiple steps](/doc/note/memory-safety.md#allocation-free-apis) and the +`wuffs_aux::DecodeImage` high-level API provides something more convenient, +albeit with similar trade-offs. + +Grepping the [examples directory](/example) for `wuffs_aux` should reveal code +examples with and without using the auxiliary code library. diff --git a/doc/note/memory-safety.md b/doc/note/memory-safety.md index 2072a74e..0a217580 100644 --- a/doc/note/memory-safety.md +++ b/doc/note/memory-safety.md @@ -63,6 +63,11 @@ kernel level. The [example/jsonptr](/example/jsonptr/jsonptr.cc) program, which parses and filters [JSON](https://www.json.org/), is another example that does useful work within a strict sandbox. +Alternatively, Wuffs-the-library's [auxiliary +code](/doc/note/auxiliary-code.md), augmenting the Wuffs-the-language code with +hand-written C++ helpers, provides higher level (allocating) APIs, albeit with +several trade-offs. + ## Thread-Safety diff --git a/example/jsonfindptrs/jsonfindptrs.cc b/example/jsonfindptrs/jsonfindptrs.cc index de601157..b21ea0a7 100644 --- a/example/jsonfindptrs/jsonfindptrs.cc +++ b/example/jsonfindptrs/jsonfindptrs.cc @@ -25,9 +25,9 @@ See the "const char* g_usage" string below for details. This program uses Wuffs' JSON decoder at a relatively high level, building in-memory representations of JSON 'things' (e.g. numbers, strings, objects). After the entire input has been converted, walking the tree prints the output -(in sorted order). The core conversion mechanism is to call JsonThing::parse, -which consumes a variable number of tokens (the output of Wuffs' JSON decoder). -JsonThing::parse can call itself recursively, as JSON values can nest. +(in sorted order). The wuffs_aux::DecodeJson library function converts the +lower level token stream to higher level callbacks. This .cc file deals only +with those callbacks, not with tokens per se. This approach is centered around JSON things. Each JSON thing comprises one or more JSON tokens. @@ -61,9 +61,7 @@ for a C++ compiler $CXX, such as clang++ or g++. #error "This C++ program requires -std=c++11 or later" #endif -#include -#include -#include +#include #include #include @@ -86,6 +84,8 @@ for a C++ compiler $CXX, such as clang++ or g++. // modules we use makes that process explicit. Preprocessing means that such // code simply isn't compiled. #define WUFFS_CONFIG__MODULES +#define WUFFS_CONFIG__MODULE__AUX__BASE +#define WUFFS_CONFIG__MODULE__AUX__JSON #define WUFFS_CONFIG__MODULE__BASE #define WUFFS_CONFIG__MODULE__JSON @@ -179,6 +179,8 @@ static const char* g_usage = // ---- +std::vector g_quirks; + struct { int remaining_argc; char** remaining_argv; @@ -229,7 +231,7 @@ parse_flags(int argc, char** argv) { return g_usage; } if (!strcmp(arg, "input-json-extra-comma")) { - g_flags.input_json_extra_comma = true; + g_quirks.push_back(WUFFS_JSON__QUIRK_ALLOW_EXTRA_COMMA); continue; } if (!strcmp(arg, "strict-json-pointer-syntax")) { @@ -247,152 +249,8 @@ parse_flags(int argc, char** argv) { // ---- -#define WORK_BUFFER_ARRAY_SIZE \ - WUFFS_JSON__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE - -#ifndef SRC_BUFFER_ARRAY_SIZE -#define SRC_BUFFER_ARRAY_SIZE (4 * 1024) -#endif -#ifndef TOKEN_BUFFER_ARRAY_SIZE -#define TOKEN_BUFFER_ARRAY_SIZE (1 * 1024) -#endif - -class TokenStream { - public: - struct Result { - std::string status_msg; - wuffs_base__token token; - // src_data is a sub-slice of m_src (a slice is a pointer-length pair). - // Calling TokenStream::peek or TokenStream::next may change the backing - // array's contents, so handling a TokenStream::Result may require copying - // this src_data slice's contents. - wuffs_base__slice_u8 src_data; - - Result(std::string s) - : status_msg(s), - token(wuffs_base__make_token(0)), - src_data(wuffs_base__empty_slice_u8()) {} - - Result(std::string s, wuffs_base__token t, wuffs_base__slice_u8 d) - : status_msg(s), token(t), src_data(d) {} - }; - - TokenStream(int input_file_descriptor) - : m_status(wuffs_base__make_status(nullptr)), - m_src(wuffs_base__make_io_buffer( - wuffs_base__make_slice_u8(m_src_array, SRC_BUFFER_ARRAY_SIZE), - wuffs_base__empty_io_buffer_meta())), - m_tok(wuffs_base__make_token_buffer( - wuffs_base__make_slice_token(m_tok_array, TOKEN_BUFFER_ARRAY_SIZE), - wuffs_base__empty_token_buffer_meta())), - m_input_file_descriptor(input_file_descriptor), - m_curr_token_end_src_index(0) { - m_status = - m_dec.initialize(sizeof__wuffs_json__decoder(), WUFFS_VERSION, 0); - - if (m_status.is_ok()) { - // Uncomment this line to enable the WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X - // option, discussed in a separate comment. - // m_dec.set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X, true); - - if (g_flags.input_json_extra_comma) { - m_dec.set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_EXTRA_COMMA, true); - } - } - } - - Result peek() { return peek_or_next(false); } - Result next() { return peek_or_next(true); } - - private: - Result peek_or_next(bool next) { - while (m_tok.meta.ri >= m_tok.meta.wi) { - if (m_status.repr == nullptr) { - // No-op. - } else if (m_status.repr == wuffs_base__suspension__short_read) { - if (m_curr_token_end_src_index != m_src.meta.ri) { - return Result( - "TokenStream: internal error: inconsistent src indexes"); - } - const char* z = read_src(); - m_curr_token_end_src_index = m_src.meta.ri; - if (z) { - return Result(z); - } - } else if (m_status.repr == wuffs_base__suspension__short_write) { - m_tok.compact(); - } else { - return Result(m_status.message()); - } - - m_status = - m_dec.decode_tokens(&m_tok, &m_src, - wuffs_base__make_slice_u8( - m_work_buffer_array, WORK_BUFFER_ARRAY_SIZE)); - } - - wuffs_base__token t = m_tok.data.ptr[m_tok.meta.ri]; - size_t i = m_curr_token_end_src_index; - uint64_t n = t.length(); - if ((m_src.meta.ri < i) || ((m_src.meta.ri - i) < n)) { - return Result("TokenStream: internal error: inconsistent src indexes"); - } - if (next) { - m_tok.meta.ri++; - m_curr_token_end_src_index += n; - } - return Result("", t, wuffs_base__make_slice_u8(m_src.data.ptr + i, n)); - } - - const char* // - read_src() { - if (m_src.meta.closed) { - return "main: internal error: read requested on a closed source"; - } - m_src.compact(); - if (m_src.meta.wi >= m_src.data.len) { - return "main: src buffer is full"; - } - while (true) { - ssize_t n = read(m_input_file_descriptor, m_src.writer_pointer(), - m_src.writer_length()); - if (n >= 0) { - m_src.meta.wi += n; - m_src.meta.closed = n == 0; - break; - } else if (errno != EINTR) { - return strerror(errno); - } - } - return nullptr; - } - - wuffs_base__status m_status; - wuffs_base__io_buffer m_src; - wuffs_base__token_buffer m_tok; - int m_input_file_descriptor; - // m_curr_token_end_src_index is the m_src.data.ptr index of the end of the - // current token. An invariant is that (m_curr_token_end_src_index <= - // m_src.meta.ri). - size_t m_curr_token_end_src_index; - - wuffs_base__token m_tok_array[TOKEN_BUFFER_ARRAY_SIZE]; - uint8_t m_src_array[SRC_BUFFER_ARRAY_SIZE]; -#if WORK_BUFFER_ARRAY_SIZE > 0 - uint8_t m_work_buffer_array[WORK_BUFFER_ARRAY_SIZE]; -#else - // Not all C/C++ compilers support 0-length arrays. - uint8_t m_work_buffer_array[1]; -#endif - wuffs_json__decoder m_dec; -}; - -// ---- - class JsonThing { public: - struct Result; - using Vector = std::vector; // We use a std::map in this example program to avoid dependencies outside of @@ -421,289 +279,8 @@ class JsonThing { Vector a; Map o; } value; - - static JsonThing::Result parse(TokenStream& ts); - - private: - static JsonThing::Result parse_array(TokenStream& ts); - static JsonThing::Result parse_literal(TokenStream::Result tsr); - static JsonThing::Result parse_number(TokenStream::Result tsr); - static JsonThing::Result parse_object(TokenStream& ts); - static JsonThing::Result parse_string(TokenStream& ts, - TokenStream::Result tsr); }; -struct JsonThing::Result { - std::string status_msg; - JsonThing thing; - - Result(std::string s) : status_msg(s), thing(JsonThing()) {} - - Result(std::string s, JsonThing t) : status_msg(s), thing(t) {} -}; - -JsonThing::Result // -JsonThing::parse(TokenStream& ts) { - while (true) { - TokenStream::Result tsr = ts.next(); - if (!tsr.status_msg.empty()) { - return Result(std::move(tsr.status_msg)); - } - - int64_t vbc = tsr.token.value_base_category(); - uint64_t vbd = tsr.token.value_base_detail(); - switch (vbc) { - case WUFFS_BASE__TOKEN__VBC__FILLER: - continue; - case WUFFS_BASE__TOKEN__VBC__STRUCTURE: - if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH) { - if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) { - return parse_array(ts); - } else if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT) { - return parse_object(ts); - } - } - break; - case WUFFS_BASE__TOKEN__VBC__STRING: - return parse_string(ts, tsr); - case WUFFS_BASE__TOKEN__VBC__LITERAL: - return parse_literal(tsr); - case WUFFS_BASE__TOKEN__VBC__NUMBER: - return parse_number(tsr); - } - - return Result("main: internal error: unexpected token"); - } -} - -JsonThing::Result // -JsonThing::parse_array(TokenStream& ts) { - JsonThing jt; - jt.kind = Kind::Array; - while (true) { - TokenStream::Result tsr = ts.peek(); - if (!tsr.status_msg.empty()) { - return Result(std::move(tsr.status_msg)); - } - int64_t vbc = tsr.token.value_base_category(); - uint64_t vbd = tsr.token.value_base_detail(); - if (vbc == WUFFS_BASE__TOKEN__VBC__FILLER) { - ts.next(); - continue; - } else if ((vbc == WUFFS_BASE__TOKEN__VBC__STRUCTURE) && - (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP)) { - ts.next(); - break; - } - - JsonThing::Result jtr = JsonThing::parse(ts); - if (!jtr.status_msg.empty()) { - return Result(std::move(jtr.status_msg)); - } - jt.value.a.push_back(std::move(jtr.thing)); - } - return Result("", jt); -} - -JsonThing::Result // -JsonThing::parse_literal(TokenStream::Result tsr) { - uint64_t vbd = tsr.token.value_base_detail(); - if (vbd & WUFFS_BASE__TOKEN__VBD__LITERAL__NULL) { - JsonThing jt; - jt.kind = Kind::Null; - return Result("", jt); - } else if (vbd & WUFFS_BASE__TOKEN__VBD__LITERAL__FALSE) { - JsonThing jt; - jt.kind = Kind::Bool; - jt.value.b = false; - return Result("", jt); - } else if (vbd & WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE) { - JsonThing jt; - jt.kind = Kind::Bool; - jt.value.b = true; - return Result("", jt); - } - return Result("main: internal error: unexpected token"); -} - -JsonThing::Result // -JsonThing::parse_number(TokenStream::Result tsr) { - // Parsing the number from its string representation (converting from "123" - // to 123) isn't necessary for the jsonfindptrs program, but if you're - // copy/pasting this JsonThing code, here's how to do it. - uint64_t vbd = tsr.token.value_base_detail(); - if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT) { - if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED) { - static constexpr int64_t m = 0x001FFFFFFFFFFFFF; // ((1<<53) - 1). - wuffs_base__result_i64 r = wuffs_base__parse_number_i64( - tsr.src_data, WUFFS_BASE__PARSE_NUMBER_XXX__DEFAULT_OPTIONS); - if (!r.status.is_ok()) { - return Result(r.status.message()); - } else if ((r.value < -m) || (+m < r.value)) { - return Result(wuffs_base__error__out_of_bounds); - } - JsonThing jt; - jt.kind = Kind::Int64; - jt.value.i = r.value; - return Result("", jt); - } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT) { - wuffs_base__result_f64 r = wuffs_base__parse_number_f64( - tsr.src_data, WUFFS_BASE__PARSE_NUMBER_XXX__DEFAULT_OPTIONS); - if (!r.status.is_ok()) { - return Result(r.status.message()); - } - JsonThing jt; - jt.kind = Kind::Float64; - jt.value.f = r.value; - return Result("", jt); - } - } - return Result("main: internal error: unexpected number"); -} - -JsonThing::Result // -JsonThing::parse_object(TokenStream& ts) { - JsonThing jt; - jt.kind = Kind::Object; - - std::string key; - bool have_key = false; - - while (true) { - TokenStream::Result tsr = ts.peek(); - if (!tsr.status_msg.empty()) { - return Result(std::move(tsr.status_msg)); - } - int64_t vbc = tsr.token.value_base_category(); - uint64_t vbd = tsr.token.value_base_detail(); - if (vbc == WUFFS_BASE__TOKEN__VBC__FILLER) { - ts.next(); - continue; - } else if ((vbc == WUFFS_BASE__TOKEN__VBC__STRUCTURE) && - (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP)) { - ts.next(); - break; - } - - JsonThing::Result jtr = JsonThing::parse(ts); - if (!jtr.status_msg.empty()) { - return Result(std::move(jtr.status_msg)); - } - - if (have_key) { - have_key = false; - auto iter = jt.value.o.find(key); - if (iter == jt.value.o.end()) { - jt.value.o.insert( - iter, Map::value_type(std::move(key), std::move(jtr.thing))); - } else { - return Result("main: duplicate key: " + key); - } - } else if (jtr.thing.kind == Kind::String) { - have_key = true; - key = std::move(jtr.thing.value.s); - } else { - return Result("main: internal error: unexpected non-string key"); - } - } - if (have_key) { - return Result("main: internal error: unpaired key"); - } - return Result("", jt); -} - -JsonThing::Result // -JsonThing::parse_string(TokenStream& ts, TokenStream::Result tsr) { - JsonThing jt; - jt.kind = Kind::String; - while (true) { - int64_t vbc = tsr.token.value_base_category(); - uint64_t vbd = tsr.token.value_base_detail(); - - switch (vbc) { - case WUFFS_BASE__TOKEN__VBC__STRING: { - if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP) { - // No-op. - - } else if (vbd & - WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY) { - const char* ptr = // Convert from (uint8_t*). - static_cast(static_cast(tsr.src_data.ptr)); - jt.value.s.append(ptr, tsr.src_data.len); - - } else if ( - vbd & - WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X) { - // We shouldn't get here unless we enable the - // WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X option. The jsonfindptrs - // program doesn't enable that by default, but if you're copy/pasting - // this JsonThing code and your program does enable that option, - // here's how to handle it. - // - // As per the quirk documentation, there are two options for how to - // interpret a backslash-x: as a byte or as a Unicode code point. - // This implementation chooses as a byte. - wuffs_base__slice_u8 encoded = tsr.src_data; - if (encoded.len & 3) { - return Result( - "main: internal error: \\x token length not a multiple of 4", - JsonThing()); - } - while (encoded.len) { - uint8_t decoded[64]; - const bool src_closed = true; - wuffs_base__transform__output o = wuffs_base__base_16__decode4( - wuffs_base__make_slice_u8(&decoded[0], 64), encoded, src_closed, - WUFFS_BASE__BASE_16__DEFAULT_OPTIONS); - if (o.status.is_error()) { - return Result(o.status.message(), JsonThing()); - } else if ((o.num_dst > 64) || (o.num_src > encoded.len)) { - return Result( - "main: internal error: inconsistent hexadecimal decoding", - JsonThing()); - } - const char* ptr = // Convert from (uint8_t*). - static_cast(static_cast(&decoded[0])); - jt.value.s.append(ptr, o.num_dst); - encoded.ptr += o.num_src; - encoded.len -= o.num_src; - } - - } else { - return Result( - "main: internal error: unexpected string-token conversion", - JsonThing()); - } - break; - } - - case WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT: { - uint8_t u[WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL]; - size_t n = wuffs_base__utf_8__encode( - wuffs_base__make_slice_u8(&u[0], - WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL), - vbd); - const char* ptr = // Convert from (uint8_t*). - static_cast(static_cast(&u[0])); - jt.value.s.append(ptr, n); - break; - } - - default: - return Result("main: internal error: unexpected token"); - } - - if (!tsr.token.continued()) { - break; - } - tsr = ts.next(); - if (!tsr.status_msg.empty()) { - return Result(std::move(tsr.status_msg)); - } - } - return Result("", jt); -} - // ---- std::string // @@ -776,27 +353,146 @@ print_json_pointers(JsonThing& jt, std::string s, uint32_t depth) { return ""; } +// ---- + +class Callbacks : public wuffs_aux::DecodeJsonCallbacks { + public: + struct Entry { + Entry(JsonThing&& jt) + : thing(std::move(jt)), has_map_key(false), map_key() {} + + JsonThing thing; + bool has_map_key; + std::string map_key; + }; + + Callbacks() = default; + + std::string Append(JsonThing&& jt) { + if (m_stack.empty()) { + m_stack.push_back(Entry(std::move(jt))); + return ""; + } + Entry& top = m_stack.back(); + switch (top.thing.kind) { + case JsonThing::Kind::Array: + top.thing.value.a.push_back(std::move(jt)); + return ""; + case JsonThing::Kind::Object: + if (top.has_map_key) { + top.has_map_key = false; + auto iter = top.thing.value.o.find(top.map_key); + if (iter != top.thing.value.o.end()) { + return "main: duplicate key: " + top.map_key; + } + top.thing.value.o.insert( + iter, JsonThing::Map::value_type(std::move(top.map_key), + std::move(jt))); + return ""; + } else if (jt.kind == JsonThing::Kind::String) { + top.has_map_key = true; + top.map_key = std::move(jt.value.s); + return ""; + } + return "main: internal error: non-string map key"; + } + return "main: internal error: non-container stack entry"; + } + + virtual std::string AppendNull() { + JsonThing jt; + jt.kind = JsonThing::Kind::Null; + return Append(std::move(jt)); + } + + virtual std::string AppendBool(bool val) { + JsonThing jt; + jt.kind = JsonThing::Kind::Bool; + jt.value.b = val; + return Append(std::move(jt)); + } + + virtual std::string AppendI64(int64_t val) { + JsonThing jt; + jt.kind = JsonThing::Kind::Int64; + jt.value.i = val; + return Append(std::move(jt)); + } + + virtual std::string AppendF64(double val) { + JsonThing jt; + jt.kind = JsonThing::Kind::Float64; + jt.value.f = val; + return Append(std::move(jt)); + } + + virtual std::string AppendString(std::string&& val) { + JsonThing jt; + jt.kind = JsonThing::Kind::String; + jt.value.s = std::move(val); + return Append(std::move(jt)); + } + + virtual std::string Push(uint32_t flags) { + if (flags & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) { + JsonThing jt; + jt.kind = JsonThing::Kind::Array; + m_stack.push_back(std::move(jt)); + return ""; + } else if (flags & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT) { + JsonThing jt; + jt.kind = JsonThing::Kind::Object; + m_stack.push_back(std::move(jt)); + return ""; + } + return "main: internal error: bad push"; + } + + virtual std::string Pop(uint32_t flags) { + if (m_stack.empty()) { + return "main: internal error: bad pop"; + } + JsonThing jt = std::move(m_stack.back().thing); + m_stack.pop_back(); + return Append(std::move(jt)); + } + + virtual void Done(wuffs_aux::DecodeJsonResult& result, + wuffs_aux::sync_io::Input& input, + wuffs_aux::IOBuffer& buffer) { + if (!result.error_message.empty()) { + return; + } else if (m_stack.size() != 1) { + result.error_message = "main: internal error: bad depth"; + return; + } + result.error_message = print_json_pointers(m_stack.back().thing, "", 0); + } + + private: + std::vector m_stack; +}; + +// ---- + std::string // main1(int argc, char** argv) { TRY(parse_flags(argc, argv)); - int input_file_descriptor = 0; // A 0 default means stdin. + FILE* in = stdin; if (g_flags.remaining_argc > 1) { return g_usage; } else if (g_flags.remaining_argc == 1) { - const char* arg = g_flags.remaining_argv[0]; - input_file_descriptor = open(arg, O_RDONLY); - if (input_file_descriptor < 0) { - return std::string("main: cannot read ") + arg + ": " + strerror(errno); + in = fopen(g_flags.remaining_argv[0], "r"); + if (!in) { + return std::string("main: cannot read input file"); } } - TokenStream ts(input_file_descriptor); - JsonThing::Result jtr = JsonThing::parse(ts); - if (!jtr.status_msg.empty()) { - return jtr.status_msg; - } - return print_json_pointers(jtr.thing, "", 0); + return wuffs_aux::DecodeJson( + Callbacks(), wuffs_aux::sync_io::FileInput(in), + wuffs_base__make_slice_u32(g_quirks.data(), g_quirks.size())) + .error_message; } // ---- diff --git a/internal/cgen/aux/base.cc b/internal/cgen/aux/base.cc new file mode 100644 index 00000000..73e643ac --- /dev/null +++ b/internal/cgen/aux/base.cc @@ -0,0 +1,85 @@ +// After editing this file, run "go generate" in the ../data directory. + +// Copyright 2020 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ---------------- Auxiliary - Base + +// Auxiliary code is discussed at +// https://github.com/google/wuffs/blob/master/doc/note/auxiliary-code.md + +#if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__AUX__BASE) + +namespace wuffs_aux { + +namespace sync_io { + +// -------- + +IOBuffer* // +Input::BringsItsOwnIOBuffer() { + return nullptr; +} + +// -------- + +FileInput::FileInput(FILE* f) : m_f(f) {} + +std::string // +FileInput::CopyIn(IOBuffer* dst) { + if (!m_f) { + return "wuffs_aux::sync_io::FileInput: nullptr file"; + } else if (dst && !dst->meta.closed) { + size_t n = fread(dst->writer_pointer(), 1, dst->writer_length(), m_f); + dst->meta.wi += n; + dst->meta.closed = feof(m_f); + if (ferror(m_f)) { + return "wuffs_aux::sync_io::FileInput: error reading file"; + } + } + return ""; +} + +// -------- + +MemoryInput::MemoryInput(const uint8_t* ptr, size_t len) + : m_io(wuffs_base__ptr_u8__reader(const_cast(ptr), len, true)) {} + +IOBuffer* // +MemoryInput::BringsItsOwnIOBuffer() { + return &m_io; +} + +std::string // +MemoryInput::CopyIn(IOBuffer* dst) { + if (dst && !dst->meta.closed && (dst != &m_io)) { + size_t nd = dst->writer_length(); + size_t ns = m_io.reader_length(); + size_t n = (nd < ns) ? nd : ns; + memcpy(dst->writer_pointer(), m_io.reader_pointer(), n); + m_io.meta.ri += n; + dst->meta.wi += n; + dst->meta.closed = m_io.reader_length() == 0; + } + return ""; +} + +// -------- + +} // namespace sync_io + +} // namespace wuffs_aux + +#endif // !defined(WUFFS_CONFIG__MODULES) || + // defined(WUFFS_CONFIG__MODULE__AUX__BASE) diff --git a/internal/cgen/aux/base.hh b/internal/cgen/aux/base.hh new file mode 100644 index 00000000..276ad8ea --- /dev/null +++ b/internal/cgen/aux/base.hh @@ -0,0 +1,88 @@ +// After editing this file, run "go generate" in the ../data directory. + +// Copyright 2020 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ---------------- Auxiliary - Base + +// Auxiliary code is discussed at +// https://github.com/google/wuffs/blob/master/doc/note/auxiliary-code.md + +#if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__AUX__BASE) + +#include + +#include + +namespace wuffs_aux { + +using IOBuffer = wuffs_base__io_buffer; + +namespace sync_io { + +// -------- + +class Input { + public: + virtual IOBuffer* BringsItsOwnIOBuffer(); + virtual std::string CopyIn(IOBuffer* dst) = 0; +}; + +// -------- + +// FileInput is an Input that reads from a file source. +// +// It does not take responsibility for closing the file when done. +class FileInput : public Input { + public: + FileInput(FILE* f); + + virtual std::string CopyIn(IOBuffer* dst); + + private: + FILE* m_f; + + // Delete the copy and assign constructors. + FileInput(const FileInput&) = delete; + FileInput& operator=(const FileInput&) = delete; +}; + +// -------- + +// MemoryInput is an Input that reads from an in-memory source. +// +// It does not take responsibility for freeing the memory when done. +class MemoryInput : public Input { + public: + MemoryInput(const uint8_t* ptr, size_t len); + + virtual IOBuffer* BringsItsOwnIOBuffer(); + virtual std::string CopyIn(IOBuffer* dst); + + private: + IOBuffer m_io; + + // Delete the copy and assign constructors. + MemoryInput(const MemoryInput&) = delete; + MemoryInput& operator=(const MemoryInput&) = delete; +}; + +// -------- + +} // namespace sync_io + +} // namespace wuffs_aux + +#endif // !defined(WUFFS_CONFIG__MODULES) || + // defined(WUFFS_CONFIG__MODULE__AUX__BASE) diff --git a/internal/cgen/aux/json.cc b/internal/cgen/aux/json.cc new file mode 100644 index 00000000..f2edd02a --- /dev/null +++ b/internal/cgen/aux/json.cc @@ -0,0 +1,259 @@ +// After editing this file, run "go generate" in the ../data directory. + +// Copyright 2020 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ---------------- Auxiliary - JSON + +#if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__AUX__JSON) + +#include + +namespace wuffs_aux { + +DecodeJsonResult::DecodeJsonResult(std::string&& error_message0, + uint64_t cursor_position0) + : error_message(std::move(error_message0)), + cursor_position(cursor_position0) {} + +DecodeJsonResult // +DecodeJson(DecodeJsonCallbacks&& callbacks, + sync_io::Input&& input, + wuffs_base__slice_u32 quirks) { + // Prepare the wuffs_base__io_buffer and the resultant error_message. + wuffs_base__io_buffer* io_buf = input.BringsItsOwnIOBuffer(); + wuffs_base__io_buffer fallback_io_buf = wuffs_base__empty_io_buffer(); + std::unique_ptr fallback_io_array(nullptr); + if (!io_buf) { + fallback_io_array = std::unique_ptr(new uint8_t[4096]); + fallback_io_buf = wuffs_base__ptr_u8__writer(fallback_io_array.get(), 4096); + io_buf = &fallback_io_buf; + } + size_t cursor_index = 0; + std::string ret_error_message; + std::string io_error_message; + + do { + // Prepare the low-level JSON decoder. + wuffs_json__decoder::unique_ptr dec = wuffs_json__decoder::alloc(); + if (!dec) { + ret_error_message = "wuffs_aux::JsonDecoder: out of memory"; + goto done; + } + for (size_t i = 0; i < quirks.len; i++) { + dec->set_quirk_enabled(quirks.ptr[i], true); + } + + // Prepare the wuffs_base__tok_buffer. + wuffs_base__token tok_array[256]; + wuffs_base__token_buffer tok_buf = + wuffs_base__slice_token__writer(wuffs_base__make_slice_token( + &tok_array[0], (sizeof(tok_array) / sizeof(tok_array[0])))); + wuffs_base__status tok_status = wuffs_base__make_status(nullptr); + + // Prepare other state. + uint32_t depth = 0; + std::string str; + + // Loop, doing these two things: + // 1. Get the next token. + // 2. Process that token. + while (true) { + // 1. Get the next token. + + while (tok_buf.meta.ri >= tok_buf.meta.wi) { + if (tok_status.repr == nullptr) { + // No-op. + } else if (tok_status.repr == wuffs_base__suspension__short_write) { + tok_buf.compact(); + } else if (tok_status.repr == wuffs_base__suspension__short_read) { + // Read from input to io_buf. + if (!io_error_message.empty()) { + ret_error_message = std::move(io_error_message); + goto done; + } else if (cursor_index != io_buf->meta.ri) { + ret_error_message = + "wuffs_aux::JsonDecoder: internal error: bad cursor_index"; + goto done; + } else if (io_buf->meta.closed) { + ret_error_message = + "wuffs_aux::JsonDecoder: internal error: io_buf is closed"; + goto done; + } + io_buf->compact(); + if (io_buf->meta.wi >= io_buf->data.len) { + ret_error_message = + "wuffs_aux::JsonDecoder: internal error: io_buf is full"; + goto done; + } + cursor_index = io_buf->meta.ri; + io_error_message = input.CopyIn(io_buf); + } else { + ret_error_message = tok_status.message(); + goto done; + } + + if (WUFFS_JSON__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE != 0) { + ret_error_message = + "wuffs_aux::JsonDecoder: internal error: bad WORKBUF_LEN"; + goto done; + } + wuffs_base__slice_u8 work_buf = wuffs_base__empty_slice_u8(); + tok_status = dec->decode_tokens(&tok_buf, io_buf, work_buf); + } + + wuffs_base__token token = tok_buf.data.ptr[tok_buf.meta.ri++]; + uint64_t token_len = token.length(); + if ((io_buf->meta.ri < cursor_index) || + ((io_buf->meta.ri - cursor_index) < token_len)) { + ret_error_message = + "wuffs_aux::JsonDecoder: internal error: bad token indexes"; + goto done; + } + uint8_t* token_ptr = io_buf->data.ptr + cursor_index; + cursor_index += token_len; + + // 2. Process that token. + + int64_t vbc = token.value_base_category(); + uint64_t vbd = token.value_base_detail(); + switch (vbc) { + case WUFFS_BASE__TOKEN__VBC__FILLER: + continue; + + case WUFFS_BASE__TOKEN__VBC__STRUCTURE: { + if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH) { + ret_error_message = callbacks.Push(static_cast(vbd)); + if (!ret_error_message.empty()) { + goto done; + } + depth++; + continue; + } + ret_error_message = callbacks.Pop(static_cast(vbd)); + depth--; + goto parsed_a_value; + } + + case WUFFS_BASE__TOKEN__VBC__STRING: { + if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP) { + // No-op. + } else if (vbd & + WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY) { + const char* ptr = // Convert from (uint8_t*). + static_cast(static_cast(token_ptr)); + str.append(ptr, token_len); + } else { + goto fail; + } + if (token.continued()) { + continue; + } + ret_error_message = callbacks.AppendString(std::move(str)); + str.clear(); + goto parsed_a_value; + } + + case WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT: { + uint8_t u[WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL]; + size_t n = wuffs_base__utf_8__encode( + wuffs_base__make_slice_u8( + &u[0], WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL), + static_cast(vbd)); + const char* ptr = // Convert from (uint8_t*). + static_cast(static_cast(&u[0])); + str.append(ptr, n); + if (token.continued()) { + continue; + } + goto fail; + } + + case WUFFS_BASE__TOKEN__VBC__LITERAL: { + ret_error_message = + (vbd & WUFFS_BASE__TOKEN__VBD__LITERAL__NULL) + ? callbacks.AppendNull() + : callbacks.AppendBool(vbd & + WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE); + goto parsed_a_value; + } + + case WUFFS_BASE__TOKEN__VBC__NUMBER: { + if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT) { + if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED) { + wuffs_base__result_i64 r = wuffs_base__parse_number_i64( + wuffs_base__make_slice_u8(token_ptr, token_len), + WUFFS_BASE__PARSE_NUMBER_XXX__DEFAULT_OPTIONS); + if (r.status.is_ok()) { + ret_error_message = callbacks.AppendI64(r.value); + goto parsed_a_value; + } + } + if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT) { + wuffs_base__result_f64 r = wuffs_base__parse_number_f64( + wuffs_base__make_slice_u8(token_ptr, token_len), + WUFFS_BASE__PARSE_NUMBER_XXX__DEFAULT_OPTIONS); + if (r.status.is_ok()) { + ret_error_message = callbacks.AppendF64(r.value); + goto parsed_a_value; + } + } + } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_NEG_INF) { + ret_error_message = callbacks.AppendF64( + wuffs_base__ieee_754_bit_representation__from_u64_to_f64( + 0xFFF0000000000000ul)); + goto parsed_a_value; + } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_POS_INF) { + ret_error_message = callbacks.AppendF64( + wuffs_base__ieee_754_bit_representation__from_u64_to_f64( + 0x7FF0000000000000ul)); + goto parsed_a_value; + } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_NEG_NAN) { + ret_error_message = callbacks.AppendF64( + wuffs_base__ieee_754_bit_representation__from_u64_to_f64( + 0xFFFFFFFFFFFFFFFFul)); + goto parsed_a_value; + } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_POS_NAN) { + ret_error_message = callbacks.AppendF64( + wuffs_base__ieee_754_bit_representation__from_u64_to_f64( + 0x7FFFFFFFFFFFFFFFul)); + goto parsed_a_value; + } + } + } + + fail: + ret_error_message = + "wuffs_aux::JsonDecoder: internal error: unexpected token"; + goto done; + + parsed_a_value: + if (!ret_error_message.empty() || (depth == 0)) { + goto done; + } + } + } while (false); + +done: + DecodeJsonResult result( + std::move(ret_error_message), + wuffs_base__u64__sat_add(io_buf->meta.pos, cursor_index)); + callbacks.Done(result, input, *io_buf); + return result; +} + +} // namespace wuffs_aux + +#endif // !defined(WUFFS_CONFIG__MODULES) || + // defined(WUFFS_CONFIG__MODULE__AUX__JSON) diff --git a/internal/cgen/aux/json.hh b/internal/cgen/aux/json.hh new file mode 100644 index 00000000..cab47bfe --- /dev/null +++ b/internal/cgen/aux/json.hh @@ -0,0 +1,83 @@ +// After editing this file, run "go generate" in the ../data directory. + +// Copyright 2020 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ---------------- Auxiliary - JSON + +#if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__AUX__JSON) + +namespace wuffs_aux { + +struct DecodeJsonResult { + DecodeJsonResult(std::string&& error_message0, uint64_t cursor_position0); + + std::string error_message; + uint64_t cursor_position; +}; + +class DecodeJsonCallbacks { + public: + // AppendXxx are called for leaf nodes: literals, numbers and strings. For + // strings, the Callbacks implementation is responsible for tracking map keys + // versus other values. + + virtual std::string AppendNull() = 0; + virtual std::string AppendBool(bool val) = 0; + virtual std::string AppendI64(int64_t val) = 0; + virtual std::string AppendF64(double val) = 0; + virtual std::string AppendString(std::string&& val) = 0; + + // Push and Pop are called for container nodes: JSON arrays (lists) and JSON + // objects (dictionaries). + // + // The flags bits combine exactly one of: + // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE + // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST + // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT + // and exactly one of: + // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE + // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST + // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT + + virtual std::string Push(uint32_t flags) = 0; + virtual std::string Pop(uint32_t flags) = 0; + + // Done is always the last Callback method called by DecodeJson, whether or + // not parsing the input as JSON encountered an error. Even when successful, + // trailing data may remain in input and buffer. See "Unintuitive JSON + // Parsing" (https://nullprogram.com/blog/2019/12/28/) which discusses JSON + // parsing and when it stops. + // + // Do not keep a reference to buffer or buffer.data.ptr after Done returns, + // as DecodeJson may then de-allocate the backing array. + virtual void Done(DecodeJsonResult& result, + sync_io::Input& input, + IOBuffer& buffer) = 0; +}; + +// DecodeJson calls callbacks based on the JSON-formatted data in input. +// +// On success, the returned error_message is empty and cursor_position counts +// the number of bytes consumed. On failure, error_message is non-empty and +// cursor_position is the location of the error. That error may be a content +// error (invalid JSON) or an input error (e.g. network failure). +DecodeJsonResult DecodeJson(DecodeJsonCallbacks&& callbacks, + sync_io::Input&& input, + wuffs_base__slice_u32 quirks); + +} // namespace wuffs_aux + +#endif // !defined(WUFFS_CONFIG__MODULES) || + // defined(WUFFS_CONFIG__MODULE__AUX__JSON) diff --git a/internal/cgen/data/data.go b/internal/cgen/data/data.go index 5023301a..e09a71f5 100644 --- a/internal/cgen/data/data.go +++ b/internal/cgen/data/data.go @@ -599,6 +599,56 @@ const BaseUTF8SubmoduleC = "" + "mize this by manually inlining the\n // wuffs_base__utf_8__next calls.\n size_t original_len = s_len;\n while (s_len > 0) {\n wuffs_base__utf_8__next__output o = wuffs_base__utf_8__next(s_ptr, s_len);\n if ((o.code_point > 0x7F) && (o.byte_length == 1)) {\n break;\n }\n s_ptr += o.byte_length;\n s_len -= o.byte_length;\n }\n return original_len - s_len;\n}\n\nWUFFS_BASE__MAYBE_STATIC size_t //\nwuffs_base__ascii__longest_valid_prefix(const uint8_t* s_ptr, size_t s_len) {\n // TODO: possibly optimize this by checking 4 or 8 bytes at a time.\n const uint8_t* original_ptr = s_ptr;\n const uint8_t* p = s_ptr;\n const uint8_t* q = s_ptr + s_len;\n for (; (p != q) && ((*p & 0x80) == 0); p++) {\n }\n return (size_t)(p - original_ptr);\n}\n" + "" +const AuxBaseCc = "" + + "// ---------------- Auxiliary - Base\n\n// Auxiliary code is discussed at\n// https://github.com/google/wuffs/blob/master/doc/note/auxiliary-code.md\n\n#if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__AUX__BASE)\n\nnamespace wuffs_aux {\n\nnamespace sync_io {\n\n" + + "" + + "// --------\n\nIOBuffer* //\nInput::BringsItsOwnIOBuffer() {\n return nullptr;\n}\n\n" + + "" + + "// --------\n\nFileInput::FileInput(FILE* f) : m_f(f) {}\n\nstd::string //\nFileInput::CopyIn(IOBuffer* dst) {\n if (!m_f) {\n return \"wuffs_aux::sync_io::FileInput: nullptr file\";\n } else if (dst && !dst->meta.closed) {\n size_t n = fread(dst->writer_pointer(), 1, dst->writer_length(), m_f);\n dst->meta.wi += n;\n dst->meta.closed = feof(m_f);\n if (ferror(m_f)) {\n return \"wuffs_aux::sync_io::FileInput: error reading file\";\n }\n }\n return \"\";\n}\n\n" + + "" + + "// --------\n\nMemoryInput::MemoryInput(const uint8_t* ptr, size_t len)\n : m_io(wuffs_base__ptr_u8__reader(const_cast(ptr), len, true)) {}\n\nIOBuffer* //\nMemoryInput::BringsItsOwnIOBuffer() {\n return &m_io;\n}\n\nstd::string //\nMemoryInput::CopyIn(IOBuffer* dst) {\n if (dst && !dst->meta.closed && (dst != &m_io)) {\n size_t nd = dst->writer_length();\n size_t ns = m_io.reader_length();\n size_t n = (nd < ns) ? nd : ns;\n memcpy(dst->writer_pointer(), m_io.reader_pointer(), n);\n m_io.meta.ri += n;\n dst->meta.wi += n;\n dst->meta.closed = m_io.reader_length() == 0;\n }\n return \"\";\n}\n\n" + + "" + + "// --------\n\n} // namespace sync_io\n\n} // namespace wuffs_aux\n\n#endif // !defined(WUFFS_CONFIG__MODULES) ||\n // defined(WUFFS_CONFIG__MODULE__AUX__BASE)\n" + + "" + +const AuxBaseHh = "" + + "// ---------------- Auxiliary - Base\n\n// Auxiliary code is discussed at\n// https://github.com/google/wuffs/blob/master/doc/note/auxiliary-code.md\n\n#if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__AUX__BASE)\n\n#include \n\n#include \n\nnamespace wuffs_aux {\n\nusing IOBuffer = wuffs_base__io_buffer;\n\nnamespace sync_io {\n\n" + + "" + + "// --------\n\nclass Input {\n public:\n virtual IOBuffer* BringsItsOwnIOBuffer();\n virtual std::string CopyIn(IOBuffer* dst) = 0;\n};\n\n" + + "" + + "// --------\n\n// FileInput is an Input that reads from a file source.\n//\n// It does not take responsibility for closing the file when done.\nclass FileInput : public Input {\n public:\n FileInput(FILE* f);\n\n virtual std::string CopyIn(IOBuffer* dst);\n\n private:\n FILE* m_f;\n\n // Delete the copy and assign constructors.\n FileInput(const FileInput&) = delete;\n FileInput& operator=(const FileInput&) = delete;\n};\n\n" + + "" + + "// --------\n\n// MemoryInput is an Input that reads from an in-memory source.\n//\n// It does not take responsibility for freeing the memory when done.\nclass MemoryInput : public Input {\n public:\n MemoryInput(const uint8_t* ptr, size_t len);\n\n virtual IOBuffer* BringsItsOwnIOBuffer();\n virtual std::string CopyIn(IOBuffer* dst);\n\n private:\n IOBuffer m_io;\n\n // Delete the copy and assign constructors.\n MemoryInput(const MemoryInput&) = delete;\n MemoryInput& operator=(const MemoryInput&) = delete;\n};\n\n" + + "" + + "// --------\n\n} // namespace sync_io\n\n} // namespace wuffs_aux\n\n#endif // !defined(WUFFS_CONFIG__MODULES) ||\n // defined(WUFFS_CONFIG__MODULE__AUX__BASE)\n" + + "" + +const AuxJsonCc = "" + + "// ---------------- Auxiliary - JSON\n\n#if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__AUX__JSON)\n\n#include \n\nnamespace wuffs_aux {\n\nDecodeJsonResult::DecodeJsonResult(std::string&& error_message0,\n uint64_t cursor_position0)\n : error_message(std::move(error_message0)),\n cursor_position(cursor_position0) {}\n\nDecodeJsonResult //\nDecodeJson(DecodeJsonCallbacks&& callbacks,\n sync_io::Input&& input,\n wuffs_base__slice_u32 quirks) {\n // Prepare the wuffs_base__io_buffer and the resultant error_message.\n wuffs_base__io_buffer* io_buf = input.BringsItsOwnIOBuffer();\n wuffs_base__io_buffer fallback_io_buf = wuffs_base__empty_io_buffer();\n std::unique_ptr fallback_io_array(nullptr);\n if (!io_buf) {\n fallback_io_array = std::unique_ptr(new uint8_t[4096]);\n fallback_io_buf = wuffs_base__ptr_u8__writer(fallback_io_array.get(), 4096);\n io_buf = &fallback_io_buf;\n }\n size_t cursor_index = 0;\n s" + + "td::string ret_error_message;\n std::string io_error_message;\n\n do {\n // Prepare the low-level JSON decoder.\n wuffs_json__decoder::unique_ptr dec = wuffs_json__decoder::alloc();\n if (!dec) {\n ret_error_message = \"wuffs_aux::JsonDecoder: out of memory\";\n goto done;\n }\n for (size_t i = 0; i < quirks.len; i++) {\n dec->set_quirk_enabled(quirks.ptr[i], true);\n }\n\n // Prepare the wuffs_base__tok_buffer.\n wuffs_base__token tok_array[256];\n wuffs_base__token_buffer tok_buf =\n wuffs_base__slice_token__writer(wuffs_base__make_slice_token(\n &tok_array[0], (sizeof(tok_array) / sizeof(tok_array[0]))));\n wuffs_base__status tok_status = wuffs_base__make_status(nullptr);\n\n // Prepare other state.\n uint32_t depth = 0;\n std::string str;\n\n // Loop, doing these two things:\n // 1. Get the next token.\n // 2. Process that token.\n while (true) {\n // 1. Get the next token.\n\n while (tok_buf.meta.ri >= tok_buf.meta.wi) {\n if (tok_sta" + + "tus.repr == nullptr) {\n // No-op.\n } else if (tok_status.repr == wuffs_base__suspension__short_write) {\n tok_buf.compact();\n } else if (tok_status.repr == wuffs_base__suspension__short_read) {\n // Read from input to io_buf.\n if (!io_error_message.empty()) {\n ret_error_message = std::move(io_error_message);\n goto done;\n } else if (cursor_index != io_buf->meta.ri) {\n ret_error_message =\n \"wuffs_aux::JsonDecoder: internal error: bad cursor_index\";\n goto done;\n } else if (io_buf->meta.closed) {\n ret_error_message =\n \"wuffs_aux::JsonDecoder: internal error: io_buf is closed\";\n goto done;\n }\n io_buf->compact();\n if (io_buf->meta.wi >= io_buf->data.len) {\n ret_error_message =\n \"wuffs_aux::JsonDecoder: internal error: io_buf is full\";\n goto done;\n }\n cursor_index = io_buf->me" + + "ta.ri;\n io_error_message = input.CopyIn(io_buf);\n } else {\n ret_error_message = tok_status.message();\n goto done;\n }\n\n if (WUFFS_JSON__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE != 0) {\n ret_error_message =\n \"wuffs_aux::JsonDecoder: internal error: bad WORKBUF_LEN\";\n goto done;\n }\n wuffs_base__slice_u8 work_buf = wuffs_base__empty_slice_u8();\n tok_status = dec->decode_tokens(&tok_buf, io_buf, work_buf);\n }\n\n wuffs_base__token token = tok_buf.data.ptr[tok_buf.meta.ri++];\n uint64_t token_len = token.length();\n if ((io_buf->meta.ri < cursor_index) ||\n ((io_buf->meta.ri - cursor_index) < token_len)) {\n ret_error_message =\n \"wuffs_aux::JsonDecoder: internal error: bad token indexes\";\n goto done;\n }\n uint8_t* token_ptr = io_buf->data.ptr + cursor_index;\n cursor_index += token_len;\n\n // 2. Process that token.\n\n int64_t vbc = token.value_base_c" + + "ategory();\n uint64_t vbd = token.value_base_detail();\n switch (vbc) {\n case WUFFS_BASE__TOKEN__VBC__FILLER:\n continue;\n\n case WUFFS_BASE__TOKEN__VBC__STRUCTURE: {\n if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH) {\n ret_error_message = callbacks.Push(static_cast(vbd));\n if (!ret_error_message.empty()) {\n goto done;\n }\n depth++;\n continue;\n }\n ret_error_message = callbacks.Pop(static_cast(vbd));\n depth--;\n goto parsed_a_value;\n }\n\n case WUFFS_BASE__TOKEN__VBC__STRING: {\n if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP) {\n // No-op.\n } else if (vbd &\n WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY) {\n const char* ptr = // Convert from (uint8_t*).\n static_cast(static_cast(token_ptr));\n str.append(ptr" + + ", token_len);\n } else {\n goto fail;\n }\n if (token.continued()) {\n continue;\n }\n ret_error_message = callbacks.AppendString(std::move(str));\n str.clear();\n goto parsed_a_value;\n }\n\n case WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT: {\n uint8_t u[WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL];\n size_t n = wuffs_base__utf_8__encode(\n wuffs_base__make_slice_u8(\n &u[0], WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL),\n static_cast(vbd));\n const char* ptr = // Convert from (uint8_t*).\n static_cast(static_cast(&u[0]));\n str.append(ptr, n);\n if (token.continued()) {\n continue;\n }\n goto fail;\n }\n\n case WUFFS_BASE__TOKEN__VBC__LITERAL: {\n ret_error_message =\n (vbd & WUFFS_BASE__TOKEN__VBD__LITERAL__NULL)\n ? callbacks.AppendN" + + "ull()\n : callbacks.AppendBool(vbd &\n WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE);\n goto parsed_a_value;\n }\n\n case WUFFS_BASE__TOKEN__VBC__NUMBER: {\n if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT) {\n if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED) {\n wuffs_base__result_i64 r = wuffs_base__parse_number_i64(\n wuffs_base__make_slice_u8(token_ptr, token_len),\n WUFFS_BASE__PARSE_NUMBER_XXX__DEFAULT_OPTIONS);\n if (r.status.is_ok()) {\n ret_error_message = callbacks.AppendI64(r.value);\n goto parsed_a_value;\n }\n }\n if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT) {\n wuffs_base__result_f64 r = wuffs_base__parse_number_f64(\n wuffs_base__make_slice_u8(token_ptr, token_len),\n WUFFS_BASE__PARSE_NUMBER_XXX__DEFAULT_OPTIO" + + "NS);\n if (r.status.is_ok()) {\n ret_error_message = callbacks.AppendF64(r.value);\n goto parsed_a_value;\n }\n }\n } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_NEG_INF) {\n ret_error_message = callbacks.AppendF64(\n wuffs_base__ieee_754_bit_representation__from_u64_to_f64(\n 0xFFF0000000000000ul));\n goto parsed_a_value;\n } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_POS_INF) {\n ret_error_message = callbacks.AppendF64(\n wuffs_base__ieee_754_bit_representation__from_u64_to_f64(\n 0x7FF0000000000000ul));\n goto parsed_a_value;\n } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_NEG_NAN) {\n ret_error_message = callbacks.AppendF64(\n wuffs_base__ieee_754_bit_representation__from_u64_to_f64(\n 0xFFFFFFFFFFFFFFFFul));\n goto parsed_a_value;\n " + + " } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_POS_NAN) {\n ret_error_message = callbacks.AppendF64(\n wuffs_base__ieee_754_bit_representation__from_u64_to_f64(\n 0x7FFFFFFFFFFFFFFFul));\n goto parsed_a_value;\n }\n }\n }\n\n fail:\n ret_error_message =\n \"wuffs_aux::JsonDecoder: internal error: unexpected token\";\n goto done;\n\n parsed_a_value:\n if (!ret_error_message.empty() || (depth == 0)) {\n goto done;\n }\n }\n } while (false);\n\ndone:\n DecodeJsonResult result(\n std::move(ret_error_message),\n wuffs_base__u64__sat_add(io_buf->meta.pos, cursor_index));\n callbacks.Done(result, input, *io_buf);\n return result;\n}\n\n} // namespace wuffs_aux\n\n#endif // !defined(WUFFS_CONFIG__MODULES) ||\n // defined(WUFFS_CONFIG__MODULE__AUX__JSON)\n" + + "" + +const AuxJsonHh = "" + + "// ---------------- Auxiliary - JSON\n\n#if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__AUX__JSON)\n\nnamespace wuffs_aux {\n\nstruct DecodeJsonResult {\n DecodeJsonResult(std::string&& error_message0, uint64_t cursor_position0);\n\n std::string error_message;\n uint64_t cursor_position;\n};\n\nclass DecodeJsonCallbacks {\n public:\n // AppendXxx are called for leaf nodes: literals, numbers and strings. For\n // strings, the Callbacks implementation is responsible for tracking map keys\n // versus other values.\n\n virtual std::string AppendNull() = 0;\n virtual std::string AppendBool(bool val) = 0;\n virtual std::string AppendI64(int64_t val) = 0;\n virtual std::string AppendF64(double val) = 0;\n virtual std::string AppendString(std::string&& val) = 0;\n\n // Push and Pop are called for container nodes: JSON arrays (lists) and JSON\n // objects (dictionaries).\n //\n // The flags bits combine exactly one of:\n // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE\n // - WUFFS_BASE__TOKEN__VBD__STRUCTU" + + "RE__FROM_LIST\n // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT\n // and exactly one of:\n // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE\n // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST\n // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT\n\n virtual std::string Push(uint32_t flags) = 0;\n virtual std::string Pop(uint32_t flags) = 0;\n\n // Done is always the last Callback method called by DecodeJson, whether or\n // not parsing the input as JSON encountered an error. Even when successful,\n // trailing data may remain in input and buffer. See \"Unintuitive JSON\n // Parsing\" (https://nullprogram.com/blog/2019/12/28/) which discusses JSON\n // parsing and when it stops.\n //\n // Do not keep a reference to buffer or buffer.data.ptr after Done returns,\n // as DecodeJson may then de-allocate the backing array.\n virtual void Done(DecodeJsonResult& result,\n sync_io::Input& input,\n IOBuffer& buffer) = 0;\n};\n\n// DecodeJson calls callbacks based on the JSON-formatted data i" + + "n input.\n//\n// On success, the returned error_message is empty and cursor_position counts\n// the number of bytes consumed. On failure, error_message is non-empty and\n// cursor_position is the location of the error. That error may be a content\n// error (invalid JSON) or an input error (e.g. network failure).\nDecodeJsonResult DecodeJson(DecodeJsonCallbacks&& callbacks,\n sync_io::Input&& input,\n wuffs_base__slice_u32 quirks);\n\n} // namespace wuffs_aux\n\n#endif // !defined(WUFFS_CONFIG__MODULES) ||\n // defined(WUFFS_CONFIG__MODULE__AUX__JSON)\n" + + "" + +var AuxNonBaseCcFiles = []string{ + AuxJsonCc, +} + +var AuxNonBaseHhFiles = []string{ + AuxJsonHh, +} + const BaseCopyright = "" + "// Copyright 2017 The Wuffs Authors.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// https://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n" + "" diff --git a/internal/cgen/data/gen.go b/internal/cgen/data/gen.go index ded89368..6f286fac 100644 --- a/internal/cgen/data/gen.go +++ b/internal/cgen/data/gen.go @@ -26,6 +26,7 @@ import ( "go/format" "io/ioutil" "os" + "strings" ) const columns = 1024 @@ -58,7 +59,7 @@ func main1() error { out.WriteString("package data\n") out.WriteString("\n") - if err := genBase(out); err != nil { + if err := genData(out); err != nil { return err } @@ -69,7 +70,7 @@ func main1() error { return ioutil.WriteFile("data.go", formatted, 0644) } -func genBase(out *bytes.Buffer) error { +func genData(out *bytes.Buffer) error { files := []struct { filename, varname string }{ @@ -95,6 +96,11 @@ func genBase(out *bytes.Buffer) error { {"../base/intconv-submodule.c", "BaseIntConvSubmoduleC"}, {"../base/pixconv-submodule.c", "BasePixConvSubmoduleC"}, {"../base/utf8-submodule.c", "BaseUTF8SubmoduleC"}, + + {"../aux/base.cc", "AuxBaseCc"}, + {"../aux/base.hh", "AuxBaseHh"}, + {"../aux/json.cc", "AuxJsonCc"}, + {"../aux/json.hh", "AuxJsonHh"}, } prefixAfterEditing := []byte("// After editing this file,") @@ -130,6 +136,26 @@ func genBase(out *bytes.Buffer) error { out.WriteString("\"\"\n\n") } + fmt.Fprintf(out, "var AuxNonBaseCcFiles = []string{\n") + for _, f := range files { + if strings.HasPrefix(f.varname, "Aux") && + strings.HasSuffix(f.varname, "Cc") && + (f.varname != "AuxBaseCc") { + fmt.Fprintf(out, "%s,\n", f.varname) + } + } + fmt.Fprintf(out, "}\n\n") + + fmt.Fprintf(out, "var AuxNonBaseHhFiles = []string{\n") + for _, f := range files { + if strings.HasPrefix(f.varname, "Aux") && + strings.HasSuffix(f.varname, "Hh") && + (f.varname != "AuxBaseHh") { + fmt.Fprintf(out, "%s,\n", f.varname) + } + } + fmt.Fprintf(out, "}\n\n") + fmt.Fprintf(out, "const BaseCopyright = \"\" +\n") writeStringConst(out, copyright) out.WriteString("\"\"\n\n") diff --git a/internal/cgen/data/placeholder.go b/internal/cgen/data/placeholder.go index 83661d69..645c5ff3 100644 --- a/internal/cgen/data/placeholder.go +++ b/internal/cgen/data/placeholder.go @@ -14,7 +14,8 @@ //go:generate go run gen.go -// Package data contains the files in the sibling "../base" directory: the -// hand-written C code that is the 'runtime library' for Wuffs-transpiled-to-C. -// In this Go package, those files' contents are provided as string constants. +// Package data contains the files in the sibling "../base" and "../aux" +// directories: the hand-written C and C++ code that is the 'runtime library' +// for Wuffs-transpiled-to-C. In this Go package, those files' contents are +// provided as string constants. package data diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 0b7a5d03..bb12c67c 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -8281,6 +8281,151 @@ struct wuffs_zlib__decoder__struct { } // extern "C" #endif +#if defined(__cplusplus) && (__cplusplus >= 201103L) + +// ---------------- Auxiliary - Base + +// Auxiliary code is discussed at +// https://github.com/google/wuffs/blob/master/doc/note/auxiliary-code.md + +#if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__AUX__BASE) + +#include + +#include + +namespace wuffs_aux { + +using IOBuffer = wuffs_base__io_buffer; + +namespace sync_io { + +// -------- + +class Input { + public: + virtual IOBuffer* BringsItsOwnIOBuffer(); + virtual std::string CopyIn(IOBuffer* dst) = 0; +}; + +// -------- + +// FileInput is an Input that reads from a file source. +// +// It does not take responsibility for closing the file when done. +class FileInput : public Input { + public: + FileInput(FILE* f); + + virtual std::string CopyIn(IOBuffer* dst); + + private: + FILE* m_f; + + // Delete the copy and assign constructors. + FileInput(const FileInput&) = delete; + FileInput& operator=(const FileInput&) = delete; +}; + +// -------- + +// MemoryInput is an Input that reads from an in-memory source. +// +// It does not take responsibility for freeing the memory when done. +class MemoryInput : public Input { + public: + MemoryInput(const uint8_t* ptr, size_t len); + + virtual IOBuffer* BringsItsOwnIOBuffer(); + virtual std::string CopyIn(IOBuffer* dst); + + private: + IOBuffer m_io; + + // Delete the copy and assign constructors. + MemoryInput(const MemoryInput&) = delete; + MemoryInput& operator=(const MemoryInput&) = delete; +}; + +// -------- + +} // namespace sync_io + +} // namespace wuffs_aux + +#endif // !defined(WUFFS_CONFIG__MODULES) || + // defined(WUFFS_CONFIG__MODULE__AUX__BASE) + +// ---------------- Auxiliary - JSON + +#if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__AUX__JSON) + +namespace wuffs_aux { + +struct DecodeJsonResult { + DecodeJsonResult(std::string&& error_message0, uint64_t cursor_position0); + + std::string error_message; + uint64_t cursor_position; +}; + +class DecodeJsonCallbacks { + public: + // AppendXxx are called for leaf nodes: literals, numbers and strings. For + // strings, the Callbacks implementation is responsible for tracking map keys + // versus other values. + + virtual std::string AppendNull() = 0; + virtual std::string AppendBool(bool val) = 0; + virtual std::string AppendI64(int64_t val) = 0; + virtual std::string AppendF64(double val) = 0; + virtual std::string AppendString(std::string&& val) = 0; + + // Push and Pop are called for container nodes: JSON arrays (lists) and JSON + // objects (dictionaries). + // + // The flags bits combine exactly one of: + // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE + // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST + // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT + // and exactly one of: + // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE + // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST + // - WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT + + virtual std::string Push(uint32_t flags) = 0; + virtual std::string Pop(uint32_t flags) = 0; + + // Done is always the last Callback method called by DecodeJson, whether or + // not parsing the input as JSON encountered an error. Even when successful, + // trailing data may remain in input and buffer. See "Unintuitive JSON + // Parsing" (https://nullprogram.com/blog/2019/12/28/) which discusses JSON + // parsing and when it stops. + // + // Do not keep a reference to buffer or buffer.data.ptr after Done returns, + // as DecodeJson may then de-allocate the backing array. + virtual void Done(DecodeJsonResult& result, + sync_io::Input& input, + IOBuffer& buffer) = 0; +}; + +// DecodeJson calls callbacks based on the JSON-formatted data in input. +// +// On success, the returned error_message is empty and cursor_position counts +// the number of bytes consumed. On failure, error_message is non-empty and +// cursor_position is the location of the error. That error may be a content +// error (invalid JSON) or an input error (e.g. network failure). +DecodeJsonResult DecodeJson(DecodeJsonCallbacks&& callbacks, + sync_io::Input&& input, + wuffs_base__slice_u32 quirks); + +} // namespace wuffs_aux + +#endif // !defined(WUFFS_CONFIG__MODULES) || + // defined(WUFFS_CONFIG__MODULE__AUX__JSON) + +#endif // defined(__cplusplus) && (__cplusplus >= 201103L) + // WUFFS C HEADER ENDS HERE. #ifdef WUFFS_IMPLEMENTATION @@ -28545,6 +28690,324 @@ wuffs_zlib__decoder__transform_io( #endif // !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__ZLIB) +#if defined(__cplusplus) && (__cplusplus >= 201103L) + +// ---------------- Auxiliary - Base + +// Auxiliary code is discussed at +// https://github.com/google/wuffs/blob/master/doc/note/auxiliary-code.md + +#if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__AUX__BASE) + +namespace wuffs_aux { + +namespace sync_io { + +// -------- + +IOBuffer* // +Input::BringsItsOwnIOBuffer() { + return nullptr; +} + +// -------- + +FileInput::FileInput(FILE* f) : m_f(f) {} + +std::string // +FileInput::CopyIn(IOBuffer* dst) { + if (!m_f) { + return "wuffs_aux::sync_io::FileInput: nullptr file"; + } else if (dst && !dst->meta.closed) { + size_t n = fread(dst->writer_pointer(), 1, dst->writer_length(), m_f); + dst->meta.wi += n; + dst->meta.closed = feof(m_f); + if (ferror(m_f)) { + return "wuffs_aux::sync_io::FileInput: error reading file"; + } + } + return ""; +} + +// -------- + +MemoryInput::MemoryInput(const uint8_t* ptr, size_t len) + : m_io(wuffs_base__ptr_u8__reader(const_cast(ptr), len, true)) {} + +IOBuffer* // +MemoryInput::BringsItsOwnIOBuffer() { + return &m_io; +} + +std::string // +MemoryInput::CopyIn(IOBuffer* dst) { + if (dst && !dst->meta.closed && (dst != &m_io)) { + size_t nd = dst->writer_length(); + size_t ns = m_io.reader_length(); + size_t n = (nd < ns) ? nd : ns; + memcpy(dst->writer_pointer(), m_io.reader_pointer(), n); + m_io.meta.ri += n; + dst->meta.wi += n; + dst->meta.closed = m_io.reader_length() == 0; + } + return ""; +} + +// -------- + +} // namespace sync_io + +} // namespace wuffs_aux + +#endif // !defined(WUFFS_CONFIG__MODULES) || + // defined(WUFFS_CONFIG__MODULE__AUX__BASE) + +// ---------------- Auxiliary - JSON + +#if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__AUX__JSON) + +#include + +namespace wuffs_aux { + +DecodeJsonResult::DecodeJsonResult(std::string&& error_message0, + uint64_t cursor_position0) + : error_message(std::move(error_message0)), + cursor_position(cursor_position0) {} + +DecodeJsonResult // +DecodeJson(DecodeJsonCallbacks&& callbacks, + sync_io::Input&& input, + wuffs_base__slice_u32 quirks) { + // Prepare the wuffs_base__io_buffer and the resultant error_message. + wuffs_base__io_buffer* io_buf = input.BringsItsOwnIOBuffer(); + wuffs_base__io_buffer fallback_io_buf = wuffs_base__empty_io_buffer(); + std::unique_ptr fallback_io_array(nullptr); + if (!io_buf) { + fallback_io_array = std::unique_ptr(new uint8_t[4096]); + fallback_io_buf = wuffs_base__ptr_u8__writer(fallback_io_array.get(), 4096); + io_buf = &fallback_io_buf; + } + size_t cursor_index = 0; + std::string ret_error_message; + std::string io_error_message; + + do { + // Prepare the low-level JSON decoder. + wuffs_json__decoder::unique_ptr dec = wuffs_json__decoder::alloc(); + if (!dec) { + ret_error_message = "wuffs_aux::JsonDecoder: out of memory"; + goto done; + } + for (size_t i = 0; i < quirks.len; i++) { + dec->set_quirk_enabled(quirks.ptr[i], true); + } + + // Prepare the wuffs_base__tok_buffer. + wuffs_base__token tok_array[256]; + wuffs_base__token_buffer tok_buf = + wuffs_base__slice_token__writer(wuffs_base__make_slice_token( + &tok_array[0], (sizeof(tok_array) / sizeof(tok_array[0])))); + wuffs_base__status tok_status = wuffs_base__make_status(nullptr); + + // Prepare other state. + uint32_t depth = 0; + std::string str; + + // Loop, doing these two things: + // 1. Get the next token. + // 2. Process that token. + while (true) { + // 1. Get the next token. + + while (tok_buf.meta.ri >= tok_buf.meta.wi) { + if (tok_status.repr == nullptr) { + // No-op. + } else if (tok_status.repr == wuffs_base__suspension__short_write) { + tok_buf.compact(); + } else if (tok_status.repr == wuffs_base__suspension__short_read) { + // Read from input to io_buf. + if (!io_error_message.empty()) { + ret_error_message = std::move(io_error_message); + goto done; + } else if (cursor_index != io_buf->meta.ri) { + ret_error_message = + "wuffs_aux::JsonDecoder: internal error: bad cursor_index"; + goto done; + } else if (io_buf->meta.closed) { + ret_error_message = + "wuffs_aux::JsonDecoder: internal error: io_buf is closed"; + goto done; + } + io_buf->compact(); + if (io_buf->meta.wi >= io_buf->data.len) { + ret_error_message = + "wuffs_aux::JsonDecoder: internal error: io_buf is full"; + goto done; + } + cursor_index = io_buf->meta.ri; + io_error_message = input.CopyIn(io_buf); + } else { + ret_error_message = tok_status.message(); + goto done; + } + + if (WUFFS_JSON__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE != 0) { + ret_error_message = + "wuffs_aux::JsonDecoder: internal error: bad WORKBUF_LEN"; + goto done; + } + wuffs_base__slice_u8 work_buf = wuffs_base__empty_slice_u8(); + tok_status = dec->decode_tokens(&tok_buf, io_buf, work_buf); + } + + wuffs_base__token token = tok_buf.data.ptr[tok_buf.meta.ri++]; + uint64_t token_len = token.length(); + if ((io_buf->meta.ri < cursor_index) || + ((io_buf->meta.ri - cursor_index) < token_len)) { + ret_error_message = + "wuffs_aux::JsonDecoder: internal error: bad token indexes"; + goto done; + } + uint8_t* token_ptr = io_buf->data.ptr + cursor_index; + cursor_index += token_len; + + // 2. Process that token. + + int64_t vbc = token.value_base_category(); + uint64_t vbd = token.value_base_detail(); + switch (vbc) { + case WUFFS_BASE__TOKEN__VBC__FILLER: + continue; + + case WUFFS_BASE__TOKEN__VBC__STRUCTURE: { + if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH) { + ret_error_message = callbacks.Push(static_cast(vbd)); + if (!ret_error_message.empty()) { + goto done; + } + depth++; + continue; + } + ret_error_message = callbacks.Pop(static_cast(vbd)); + depth--; + goto parsed_a_value; + } + + case WUFFS_BASE__TOKEN__VBC__STRING: { + if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP) { + // No-op. + } else if (vbd & + WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY) { + const char* ptr = // Convert from (uint8_t*). + static_cast(static_cast(token_ptr)); + str.append(ptr, token_len); + } else { + goto fail; + } + if (token.continued()) { + continue; + } + ret_error_message = callbacks.AppendString(std::move(str)); + str.clear(); + goto parsed_a_value; + } + + case WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT: { + uint8_t u[WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL]; + size_t n = wuffs_base__utf_8__encode( + wuffs_base__make_slice_u8( + &u[0], WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL), + static_cast(vbd)); + const char* ptr = // Convert from (uint8_t*). + static_cast(static_cast(&u[0])); + str.append(ptr, n); + if (token.continued()) { + continue; + } + goto fail; + } + + case WUFFS_BASE__TOKEN__VBC__LITERAL: { + ret_error_message = + (vbd & WUFFS_BASE__TOKEN__VBD__LITERAL__NULL) + ? callbacks.AppendNull() + : callbacks.AppendBool(vbd & + WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE); + goto parsed_a_value; + } + + case WUFFS_BASE__TOKEN__VBC__NUMBER: { + if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT) { + if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED) { + wuffs_base__result_i64 r = wuffs_base__parse_number_i64( + wuffs_base__make_slice_u8(token_ptr, token_len), + WUFFS_BASE__PARSE_NUMBER_XXX__DEFAULT_OPTIONS); + if (r.status.is_ok()) { + ret_error_message = callbacks.AppendI64(r.value); + goto parsed_a_value; + } + } + if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT) { + wuffs_base__result_f64 r = wuffs_base__parse_number_f64( + wuffs_base__make_slice_u8(token_ptr, token_len), + WUFFS_BASE__PARSE_NUMBER_XXX__DEFAULT_OPTIONS); + if (r.status.is_ok()) { + ret_error_message = callbacks.AppendF64(r.value); + goto parsed_a_value; + } + } + } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_NEG_INF) { + ret_error_message = callbacks.AppendF64( + wuffs_base__ieee_754_bit_representation__from_u64_to_f64( + 0xFFF0000000000000ul)); + goto parsed_a_value; + } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_POS_INF) { + ret_error_message = callbacks.AppendF64( + wuffs_base__ieee_754_bit_representation__from_u64_to_f64( + 0x7FF0000000000000ul)); + goto parsed_a_value; + } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_NEG_NAN) { + ret_error_message = callbacks.AppendF64( + wuffs_base__ieee_754_bit_representation__from_u64_to_f64( + 0xFFFFFFFFFFFFFFFFul)); + goto parsed_a_value; + } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_POS_NAN) { + ret_error_message = callbacks.AppendF64( + wuffs_base__ieee_754_bit_representation__from_u64_to_f64( + 0x7FFFFFFFFFFFFFFFul)); + goto parsed_a_value; + } + } + } + + fail: + ret_error_message = + "wuffs_aux::JsonDecoder: internal error: unexpected token"; + goto done; + + parsed_a_value: + if (!ret_error_message.empty() || (depth == 0)) { + goto done; + } + } + } while (false); + +done: + DecodeJsonResult result( + std::move(ret_error_message), + wuffs_base__u64__sat_add(io_buf->meta.pos, cursor_index)); + callbacks.Done(result, input, *io_buf); + return result; +} + +} // namespace wuffs_aux + +#endif // !defined(WUFFS_CONFIG__MODULES) || + // defined(WUFFS_CONFIG__MODULE__AUX__JSON) + +#endif // defined(__cplusplus) && (__cplusplus >= 201103L) + #endif // WUFFS_IMPLEMENTATION #ifdef __clang__