std: add compact_retaining, history_retain_length

This commit is contained in:
Nigel Tao committed 2023-10-05 13:24:44 +11:00
1 parent 0f91f23bb6
commit 5fb82e7d0a
33 files changed
+1088 -226

No files matched your search

+6 -6
View File
@@ -102,7 +102,7 @@ Input::BringsItsOwnIOBuffer() {
FileInput::FileInput(FILE* f) : m_f(f) {}
std::string //
FileInput::CopyIn(IOBuffer* dst) {
FileInput::CopyIn(IOBuffer* dst, uint64_t history_retain_length) {
if (!m_f) {
return "wuffs_aux::sync_io::FileInput: nullptr file";
} else if (!dst) {
@@ -110,7 +110,7 @@ FileInput::CopyIn(IOBuffer* dst) {
} else if (dst->meta.closed) {
return "wuffs_aux::sync_io::FileInput: end of file";
} else {
dst->compact();
dst->compact_retaining(history_retain_length);
size_t n = fread(dst->writer_pointer(), 1, dst->writer_length(), m_f);
dst->meta.wi += n;
dst->meta.closed = feof(m_f);
@@ -138,7 +138,7 @@ MemoryInput::BringsItsOwnIOBuffer() {
}
std::string //
MemoryInput::CopyIn(IOBuffer* dst) {
MemoryInput::CopyIn(IOBuffer* dst, uint64_t history_retain_length) {
if (!dst) {
return "wuffs_aux::sync_io::MemoryInput: nullptr IOBuffer";
} else if (dst->meta.closed) {
@@ -148,7 +148,7 @@ MemoryInput::CopyIn(IOBuffer* dst) {
// to it.
return "wuffs_aux::sync_io::MemoryInput: overlapping buffers";
} else {
dst->compact();
dst->compact_retaining(history_retain_length);
size_t nd = dst->writer_length();
size_t ns = m_io.reader_length();
size_t n = (nd < ns) ? nd : ns;
@@ -203,7 +203,7 @@ AdvanceIOBufferTo(const ErrorMessages& error_messages,
if (!input.BringsItsOwnIOBuffer()) {
io_buf.compact();
}
std::string error_message = input.CopyIn(&io_buf);
std::string error_message = input.CopyIn(&io_buf, 0);
if (!error_message.empty()) {
return error_message;
}
@@ -286,7 +286,7 @@ HandleMetadata(
} else if (!input.BringsItsOwnIOBuffer()) {
io_buf.compact();
}
std::string error_message = input.CopyIn(&io_buf);
std::string error_message = input.CopyIn(&io_buf, 0);
if (!error_message.empty()) {
return error_message;
}
+3 -3
View File
@@ -98,7 +98,7 @@ class Input {
virtual ~Input();
virtual IOBuffer* BringsItsOwnIOBuffer();
virtual std::string CopyIn(IOBuffer* dst) = 0;
virtual std::string CopyIn(IOBuffer* dst, uint64_t history_retain_length) = 0;
};
// --------
@@ -110,7 +110,7 @@ class FileInput : public Input {
public:
FileInput(FILE* f);
virtual std::string CopyIn(IOBuffer* dst);
virtual std::string CopyIn(IOBuffer* dst, uint64_t history_retain_length);
private:
FILE* m_f;
@@ -131,7 +131,7 @@ class MemoryInput : public Input {
MemoryInput(const uint8_t* ptr, size_t len);
virtual IOBuffer* BringsItsOwnIOBuffer();
virtual std::string CopyIn(IOBuffer* dst);
virtual std::string CopyIn(IOBuffer* dst, uint64_t history_retain_length);
private:
IOBuffer m_io;
+2 -2
View File
@@ -115,14 +115,14 @@ DecodeCbor(DecodeCborCallbacks& callbacks,
"wuffs_aux::DecodeCbor: internal error: io_buf is closed";
goto done;
}
io_buf->compact();
io_buf->compact_retaining(dec->history_retain_length());
if (io_buf->meta.wi >= io_buf->data.len) {
ret_error_message =
"wuffs_aux::DecodeCbor: internal error: io_buf is full";
goto done;
}
cursor_index = io_buf->meta.ri;
io_error_message = input.CopyIn(io_buf);
io_error_message = input.CopyIn(io_buf, dec->history_retain_length());
} else {
ret_error_message = tok_status.message();
goto done;
+9 -5
View File
@@ -344,7 +344,7 @@ redirect:
fourcc = 0;
break;
}
std::string error_message = input.CopyIn(&io_buf);
std::string error_message = input.CopyIn(&io_buf, 0);
if (!error_message.empty()) {
return DecodeImageResult(std::move(error_message));
}
@@ -440,7 +440,8 @@ redirect:
} else if (io_buf.meta.closed) {
return DecodeImageResult(DecodeImage_UnexpectedEndOfFile);
} else {
std::string error_message = input.CopyIn(&io_buf);
std::string error_message =
input.CopyIn(&io_buf, image_decoder->history_retain_length());
if (!error_message.empty()) {
return DecodeImageResult(std::move(error_message));
}
@@ -522,7 +523,8 @@ redirect:
} else if (io_buf.meta.closed) {
return DecodeImageResult(DecodeImage_UnexpectedEndOfFile);
} else {
std::string error_message = input.CopyIn(&io_buf);
std::string error_message =
input.CopyIn(&io_buf, image_decoder->history_retain_length());
if (!error_message.empty()) {
return DecodeImageResult(std::move(error_message));
}
@@ -551,7 +553,8 @@ redirect:
message = DecodeImage_UnexpectedEndOfFile;
break;
} else {
std::string error_message = input.CopyIn(&io_buf);
std::string error_message =
input.CopyIn(&io_buf, image_decoder->history_retain_length());
if (!error_message.empty()) {
message = std::move(error_message);
break;
@@ -579,7 +582,8 @@ redirect:
} else if (io_buf.meta.closed) {
return DecodeImageResult(DecodeImage_UnexpectedEndOfFile);
} else {
std::string error_message = input.CopyIn(&io_buf);
std::string error_message =
input.CopyIn(&io_buf, image_decoder->history_retain_length());
if (!error_message.empty()) {
return DecodeImageResult(std::move(error_message));
}
+52 -52
View File
@@ -58,58 +58,58 @@ DecodeJsonArgJsonPointer::DefaultValue() {
// --------
#define WUFFS_AUX__DECODE_JSON__GET_THE_NEXT_TOKEN \
while (tok_buf.meta.ri >= tok_buf.meta.wi) { \
if (tok_status.repr == nullptr) { \
goto done; \
} else if (tok_status.repr == wuffs_base__suspension__short_write) { \
tok_buf.compact(); \
} else if (tok_status.repr == wuffs_base__suspension__short_read) { \
if (!io_error_message.empty()) { \
ret_error_message = std::move(io_error_message); \
goto done; \
} else if (cursor_index != io_buf->meta.ri) { \
ret_error_message = \
"wuffs_aux::DecodeJson: internal error: bad cursor_index"; \
goto done; \
} else if (io_buf->meta.closed) { \
ret_error_message = \
"wuffs_aux::DecodeJson: internal error: io_buf is closed"; \
goto done; \
} \
io_buf->compact(); \
if (io_buf->meta.wi >= io_buf->data.len) { \
ret_error_message = \
"wuffs_aux::DecodeJson: internal error: io_buf is full"; \
goto done; \
} \
cursor_index = io_buf->meta.ri; \
io_error_message = input.CopyIn(io_buf); \
} else { \
ret_error_message = tok_status.message(); \
goto done; \
} \
tok_status = \
dec->decode_tokens(&tok_buf, io_buf, wuffs_base__empty_slice_u8()); \
if ((tok_buf.meta.ri > tok_buf.meta.wi) || \
(tok_buf.meta.wi > tok_buf.data.len) || \
(io_buf->meta.ri > io_buf->meta.wi) || \
(io_buf->meta.wi > io_buf->data.len)) { \
ret_error_message = \
"wuffs_aux::DecodeJson: internal error: bad buffer indexes"; \
goto done; \
} \
} \
wuffs_base__token token = tok_buf.data.ptr[tok_buf.meta.ri++]; \
uint64_t token_len = token.length(); \
if ((io_buf->meta.ri < cursor_index) || \
((io_buf->meta.ri - cursor_index) < token_len)) { \
ret_error_message = \
"wuffs_aux::DecodeJson: internal error: bad token indexes"; \
goto done; \
} \
uint8_t* token_ptr = io_buf->data.ptr + cursor_index; \
(void)(token_ptr); \
#define WUFFS_AUX__DECODE_JSON__GET_THE_NEXT_TOKEN \
while (tok_buf.meta.ri >= tok_buf.meta.wi) { \
if (tok_status.repr == nullptr) { \
goto done; \
} else if (tok_status.repr == wuffs_base__suspension__short_write) { \
tok_buf.compact(); \
} else if (tok_status.repr == wuffs_base__suspension__short_read) { \
if (!io_error_message.empty()) { \
ret_error_message = std::move(io_error_message); \
goto done; \
} else if (cursor_index != io_buf->meta.ri) { \
ret_error_message = \
"wuffs_aux::DecodeJson: internal error: bad cursor_index"; \
goto done; \
} else if (io_buf->meta.closed) { \
ret_error_message = \
"wuffs_aux::DecodeJson: internal error: io_buf is closed"; \
goto done; \
} \
io_buf->compact_retaining(dec->history_retain_length()); \
if (io_buf->meta.wi >= io_buf->data.len) { \
ret_error_message = \
"wuffs_aux::DecodeJson: internal error: io_buf is full"; \
goto done; \
} \
cursor_index = io_buf->meta.ri; \
io_error_message = input.CopyIn(io_buf, dec->history_retain_length()); \
} else { \
ret_error_message = tok_status.message(); \
goto done; \
} \
tok_status = \
dec->decode_tokens(&tok_buf, io_buf, wuffs_base__empty_slice_u8()); \
if ((tok_buf.meta.ri > tok_buf.meta.wi) || \
(tok_buf.meta.wi > tok_buf.data.len) || \
(io_buf->meta.ri > io_buf->meta.wi) || \
(io_buf->meta.wi > io_buf->data.len)) { \
ret_error_message = \
"wuffs_aux::DecodeJson: internal error: bad buffer indexes"; \
goto done; \
} \
} \
wuffs_base__token token = tok_buf.data.ptr[tok_buf.meta.ri++]; \
uint64_t token_len = token.length(); \
if ((io_buf->meta.ri < cursor_index) || \
((io_buf->meta.ri - cursor_index) < token_len)) { \
ret_error_message = \
"wuffs_aux::DecodeJson: internal error: bad token indexes"; \
goto done; \
} \
uint8_t* token_ptr = io_buf->data.ptr + cursor_index; \
(void)(token_ptr); \
cursor_index += static_cast<size_t>(token_len)
// --------
+51 -4
View File
@@ -36,6 +36,7 @@ typedef struct wuffs_base__io_buffer__struct {
#ifdef __cplusplus
inline bool is_valid() const;
inline void compact();
inline void compact_retaining(uint64_t history_retain_length);
inline size_t reader_length() const;
inline uint8_t* reader_pointer() const;
inline uint64_t reader_position() const;
@@ -160,14 +161,55 @@ wuffs_base__io_buffer__compact(wuffs_base__io_buffer* buf) {
return;
}
buf->meta.pos = wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri);
size_t n = buf->meta.wi - buf->meta.ri;
if (n != 0) {
memmove(buf->data.ptr, buf->data.ptr + buf->meta.ri, n);
size_t new_wi = buf->meta.wi - buf->meta.ri;
if (new_wi != 0) {
memmove(buf->data.ptr, buf->data.ptr + buf->meta.ri, new_wi);
}
buf->meta.wi = n;
buf->meta.wi = new_wi;
buf->meta.ri = 0;
}
// wuffs_base__io_buffer__compact_retaining moves any written but unread bytes
// closer to the start of the buffer. It retains H bytes of history (the most
// recently read bytes), where H is min(buf->meta.ri, history_retain_length).
// A postcondition is that buf->meta.ri == H.
//
// wuffs_base__io_buffer__compact_retaining(0) is equivalent to
// wuffs_base__io_buffer__compact().
//
// For example, if buf started like this:
//
// +--- ri = 3
// v
// abcdefgh?? len = 10, pos = 900
// ^
// +--- wi = 8
//
// Then, depending on history_retain_length, the resultant buf would be:
//
// HRL = 0 defgh????? ri = 0 wi = 5 pos = 903
// HRL = 1 cdefgh???? ri = 1 wi = 6 pos = 902
// HRL = 2 bcdefgh??? ri = 2 wi = 7 pos = 901
// HRL = 3 abcdefgh?? ri = 3 wi = 8 pos = 900
// HRL = 4+ abcdefgh?? ri = 3 wi = 8 pos = 900
static inline void //
wuffs_base__io_buffer__compact_retaining(wuffs_base__io_buffer* buf,
uint64_t history_retain_length) {
if (!buf || (buf->meta.ri == 0)) {
return;
}
size_t old_ri = buf->meta.ri;
size_t new_ri = (size_t)(wuffs_base__u64__min(old_ri, history_retain_length));
size_t memmove_start = old_ri - new_ri;
buf->meta.pos = wuffs_base__u64__sat_add(buf->meta.pos, memmove_start);
size_t new_wi = buf->meta.wi - memmove_start;
if ((new_wi != 0) && (memmove_start != 0)) {
memmove(buf->data.ptr, buf->data.ptr + memmove_start, new_wi);
}
buf->meta.wi = new_wi;
buf->meta.ri = new_ri;
}
static inline size_t //
wuffs_base__io_buffer__reader_length(const wuffs_base__io_buffer* buf) {
return buf ? buf->meta.wi - buf->meta.ri : 0;
@@ -224,6 +266,11 @@ wuffs_base__io_buffer::compact() {
wuffs_base__io_buffer__compact(this);
}
inline void //
wuffs_base__io_buffer::compact_retaining(uint64_t history_retain_length) {
wuffs_base__io_buffer__compact_retaining(this, history_retain_length);
}
inline size_t //
wuffs_base__io_buffer::reader_length() const {
return wuffs_base__io_buffer__reader_length(this);
+52 -4
View File
@@ -321,6 +321,7 @@ typedef struct wuffs_base__token_buffer__struct {
#ifdef __cplusplus
inline bool is_valid() const;
inline void compact();
inline void compact_retaining(uint64_t history_retain_length);
inline uint64_t reader_length() const;
inline wuffs_base__token* reader_pointer() const;
inline wuffs_base__slice_token reader_slice() const;
@@ -421,15 +422,57 @@ wuffs_base__token_buffer__compact(wuffs_base__token_buffer* buf) {
return;
}
buf->meta.pos = wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri);
size_t n = buf->meta.wi - buf->meta.ri;
if (n != 0) {
size_t new_wi = buf->meta.wi - buf->meta.ri;
if (new_wi != 0) {
memmove(buf->data.ptr, buf->data.ptr + buf->meta.ri,
n * sizeof(wuffs_base__token));
new_wi * sizeof(wuffs_base__token));
}
buf->meta.wi = n;
buf->meta.wi = new_wi;
buf->meta.ri = 0;
}
// wuffs_base__token_buffer__compact_retaining moves any written but unread
// tokens closer to the start of the buffer. It retains H tokens of history
// (the most recently read tokens), where H is min(buf->meta.ri,
// history_retain_length). A postcondition is that buf->meta.ri == H.
//
// wuffs_base__token_buffer__compact_retaining(0) is equivalent to
// wuffs_base__token_buffer__compact().
//
// For example, if buf started like this:
//
// +--- ri = 3
// v
// abcdefgh?? len = 10, pos = 900
// ^
// +--- wi = 8
//
// Then, depending on history_retain_length, the resultant buf would be:
//
// HRL = 0 defgh????? ri = 0 wi = 5 pos = 903
// HRL = 1 cdefgh???? ri = 1 wi = 6 pos = 902
// HRL = 2 bcdefgh??? ri = 2 wi = 7 pos = 901
// HRL = 3 abcdefgh?? ri = 3 wi = 8 pos = 900
// HRL = 4+ abcdefgh?? ri = 3 wi = 8 pos = 900
static inline void //
wuffs_base__token_buffer__compact_retaining(wuffs_base__token_buffer* buf,
uint64_t history_retain_length) {
if (!buf || (buf->meta.ri == 0)) {
return;
}
size_t old_ri = buf->meta.ri;
size_t new_ri = (size_t)(wuffs_base__u64__min(old_ri, history_retain_length));
size_t memmove_start = old_ri - new_ri;
buf->meta.pos = wuffs_base__u64__sat_add(buf->meta.pos, memmove_start);
size_t new_wi = buf->meta.wi - memmove_start;
if ((new_wi != 0) && (memmove_start != 0)) {
memmove(buf->data.ptr, buf->data.ptr + memmove_start,
new_wi * sizeof(wuffs_base__token));
}
buf->meta.wi = new_wi;
buf->meta.ri = new_ri;
}
static inline uint64_t //
wuffs_base__token_buffer__reader_length(const wuffs_base__token_buffer* buf) {
return buf ? buf->meta.wi - buf->meta.ri : 0;
@@ -488,6 +531,11 @@ wuffs_base__token_buffer::compact() {
wuffs_base__token_buffer__compact(this);
}
inline void //
wuffs_base__token_buffer::compact_retaining(uint64_t history_retain_length) {
wuffs_base__token_buffer__compact_retaining(this, history_retain_length);
}
inline uint64_t //
wuffs_base__token_buffer::reader_length() const {
return wuffs_base__token_buffer__reader_length(this);