diff --git a/std/brotli/README.md b/std/brotli/README.md index 28ff1e1b..0a5869f5 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -120,7 +120,27 @@ Notes: across `transform_io` calls. When the C code runs out of input, it first tries to flush its output (`WriteRingBuffer`) and that checks for an overrun metablock. A Wuffs `$short read` suspension corresponds to the C code's - running out of input. + running out of input. With the whole input in one call, the two always + agree. With split input, there is one known difference: when the C code + runs out of input in the middle of a command's, distance's or block + switch's extra bits, it rolls back to the start of that (partially read) + item and copies the unused input bytes into an internal buffer + (`BrotliDecoderState.buffer`). On the next call, it re-reads the item from + that buffer, appending one new input byte at a time, and every time that + buffer runs dry it runs out of input again (and checks for an overrun + metablock again), at points where the Wuffs decoder just reads on. So, for + an invalid stream whose overrun happens just after such a resumption, + libbrotlidec can report `BLOCK_LENGTH_1` (`"#bad block length"`) where this + package reports `"#bad metablock length"` (`BLOCK_LENGTH_2`). Mirroring + that would need mirroring the C code's bit reader buffering (including its + fast path's prefetching, which decides how many bytes are rolled back), so + this is a deliberate deviation. For example, the 36-byte stream `09 83 04 + 00 00 36 e0 28 0a 61 d0 09 07 93 e3 b6 38 e4 02 00 b0 dc 1a a5 6f 3f 00 20 + 00 e2 b0 00 6a dd 07 00`, split after bytes 24, 25 and 30, gives + `BLOCK_LENGTH_1` from libbrotlidec v1.2.1 and `"#bad metablock length"` + here (and `BLOCK_LENGTH_2` from both when given in one piece), as checked + by `test_mimic_brotli_decode_split_block_length`. Differential fuzzing + (see `script/manual-test-brotli-fuzz.c`) found no other difference. - `"#bad block switch"` needs more than `(1 << 24)` commands in one metablock. It is implemented but not tested. - The remaining libbrotlidec error codes do not apply: `DICTIONARY_NOT_SET` diff --git a/test/c/std/brotli.c b/test/c/std/brotli.c index fdfd7a56..1f6847cc 100644 --- a/test/c/std/brotli.c +++ b/test/c/std/brotli.c @@ -633,6 +633,83 @@ test_mimic_brotli_decode_malformed() { return NULL; } +// g_brotli_split_block_length is an invalid stream (found by differential +// fuzzing) whose last metablock is overrun. In one piece, libbrotlidec +// reports BLOCK_LENGTH_2 (and Wuffs "#bad metablock length"). Split after +// bytes 24, 25 and 30, libbrotlidec reports BLOCK_LENGTH_1, because of its +// internal input buffer, while Wuffs still reports "#bad metablock length". +// This is a documented difference (see "Status Mapping" in +// std/brotli/README.md). +static const uint8_t g_brotli_split_block_length[36] = { + 0x09, 0x83, 0x04, 0x00, 0x00, 0x36, 0xE0, 0x28, 0x0A, 0x61, 0xD0, 0x09, + 0x07, 0x93, 0xE3, 0xB6, 0x38, 0xE4, 0x02, 0x00, 0xB0, 0xDC, 0x1A, 0xA5, + 0x6F, 0x3F, 0x00, 0x20, 0x00, 0xE2, 0xB0, 0x00, 0x6A, 0xDD, 0x07, 0x00, +}; + +const char* // +test_mimic_brotli_decode_split_block_length() { + CHECK_FOCUS(__func__); + static const size_t ends[2][4] = {{36, 0, 0, 0}, {24, 25, 30, 36}}; + static const int want_c[2] = {-10, -9}; + for (int split = 0; split < 2; split++) { + // libbrotlidec. + BrotliDecoderState* state = BrotliDecoderCreateInstance(NULL, NULL, NULL); + if (!state) { + RETURN_FAIL("BrotliDecoderCreateInstance failed"); + } + size_t pos = 0; + size_t total_out = 0; + for (int k = 0; (k < 4) && ends[split][k]; k++) { + size_t avail_in = ends[split][k] - pos; + const uint8_t* next_in = g_brotli_split_block_length + pos; + size_t avail_out = g_have_slice_u8.len - total_out; + uint8_t* next_out = g_have_slice_u8.ptr + total_out; + BrotliDecoderResult r = BrotliDecoderDecompressStream( + state, &avail_in, &next_in, &avail_out, &next_out, NULL); + pos = ends[split][k] - avail_in; + total_out = g_have_slice_u8.len - avail_out; + if (r != BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { + break; + } + } + int code = (int)BrotliDecoderGetErrorCode(state); + BrotliDecoderDestroyInstance(state); + if (code != want_c[split]) { + RETURN_FAIL("split=%d: libbrotlidec error code: have %d, want %d", split, + code, want_c[split]); + } + + // Wuffs. + wuffs_brotli__decoder dec; + CHECK_STATUS("initialize", + wuffs_brotli__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED)); + wuffs_base__io_buffer dst = ((wuffs_base__io_buffer){ + .data = g_have_slice_u8, + }); + wuffs_base__status status = wuffs_base__make_status(NULL); + pos = 0; + for (int k = 0; (k < 4) && ends[split][k]; k++) { + wuffs_base__io_buffer src = wuffs_base__ptr_u8__reader( + (uint8_t*)g_brotli_split_block_length, ends[split][k], + ends[split][k] == 36); + src.meta.ri = pos; + status = wuffs_brotli__decoder__transform_io(&dec, &dst, &src, + g_work_slice_u8); + pos = src.meta.ri; + if (status.repr != wuffs_base__suspension__short_read) { + break; + } + } + if (status.repr != wuffs_brotli__error__bad_metablock_length) { + RETURN_FAIL("split=%d: Wuffs status: have \"%s\", want \"%s\"", split, + status.repr, wuffs_brotli__error__bad_metablock_length); + } + } + return NULL; +} + #endif // WUFFS_MIMIC // ---------------- Encoder Tests @@ -4390,6 +4467,7 @@ proc g_tests[] = { test_mimic_brotli_decode_differential, test_mimic_brotli_decode_malformed, test_mimic_brotli_decode_romeo, + test_mimic_brotli_decode_split_block_length, // Large window and dictionaries. test_mimic_brotli_decode_ext_differential,