diff --git a/script/manual-test-brotli-fuzz.c b/script/manual-test-brotli-fuzz.c index 619e5c9a..5d4eace9 100644 --- a/script/manual-test-brotli-fuzz.c +++ b/script/manual-test-brotli-fuzz.c @@ -62,7 +62,8 @@ // gcc -O2 -DBROTLI_EXPERIMENTAL -I$BROTLI/c/include -c $f // -o /tmp/bd/$(basename $(dirname $f))_$(basename $f .c).o; done // ar rcs /tmp/bd/libbrotlidecexp.a /tmp/bd/*.o -// gcc -O2 -std=c99 -I$BROTLI/c/include script/manual-test-brotli-fuzz.c +// gcc -O2 -std=c99 -I$BROTLI/c/include -I$BROTLI/c +// script/manual-test-brotli-fuzz.c // $BROTLI_BUILD/libbrotlienc.a /tmp/bd/libbrotlidecexp.a -lm // -o /tmp/brotli-fuzz // /tmp/brotli-fuzz -dec -seed=1 -n=100000 @@ -88,6 +89,9 @@ #include "brotli/decode.h" #include "brotli/encode.h" #include "brotli/shared_dictionary.h" +// libbrotlienc's internal state, to recognize a known C bug (see +// c_state_corrupted). This needs -I$BROTLI/c as well as -I$BROTLI/c/include. +#include "enc/state.h" #define WUFFS_IMPLEMENTATION #define WUFFS_CONFIG__MODULES @@ -930,6 +934,28 @@ cleanup: } // encode_c runs the plan with libbrotlienc. +// g_c_state_corrupted is encode_c's error when libbrotlienc's +// BrotliEncoderCompressStream fails because of a C bug: in ProcessMetadata, +// when dst has no room, the metadata body goes through tiny_buf_ (16 bytes) +// and pushing it out leaves next_out_ pointing just past tiny_buf_, at +// remaining_metadata_bytes_. Back in the processing state, qualities 0 and 1 +// compress in place (into the caller's dst) without resetting next_out_, so +// a later flush's InjectBytePaddingBlock writes the padding bits through the +// stale pointer, over remaining_metadata_bytes_. Its output is still right, +// but the next BrotliEncoderCompressStream call sees a metadata block in +// progress and fails. A C build that sets next_out_ to NULL when the +// metadata workflow ends produces the same bytes as Wuffs. The corruption +// shows as the processing state with remaining_metadata_bytes_ not +// BROTLI_UINT32_MAX, which never happens otherwise. +static const char g_c_state_corrupted[] = + "C state corrupted (stale next_out_ after metadata)"; + +static bool // +c_state_corrupted(BrotliEncoderState* state) { + return (state->stream_state_ == BROTLI_STREAM_PROCESSING) && + (state->remaining_metadata_bytes_ != 0xFFFFFFFFu); +} + static const char* // encode_c(const enc_config* c, const dict_set* dicts, const uint8_t* src_ptr, uint8_t* out, size_t out_cap, size_t* out_len) { @@ -982,7 +1008,8 @@ encode_c(const enc_config* c, const dict_set* dicts, const uint8_t* src_ptr, uint8_t* next_out = out + *out_len; if (!BrotliEncoderCompressStream(state, op, &avail_in, &next_in, &avail_out, &next_out, NULL)) { - ret = "BrotliEncoderCompressStream failed"; + ret = c_state_corrupted(state) ? g_c_state_corrupted + : "BrotliEncoderCompressStream failed"; goto cleanup; } *out_len += avail_out0 - avail_out; @@ -2098,6 +2125,7 @@ static uint64_t g_enc_by_quality[12]; static uint64_t g_enc_bytes_in; static uint64_t g_enc_both_rejected; static uint64_t g_enc_both_flint_failed; +static uint64_t g_enc_c_corrupted; static uint64_t g_enc_skipped; static uint64_t g_enc_roundtrips; static uint64_t g_enc_with_dicts; @@ -2469,10 +2497,23 @@ run_enc(uint64_t iters, double secs) { (have_err == wuffs_base__error__bad_call_sequence) && !strcmp(want_err, "BrotliEncoderCompressStream failed") && (have_len == want_len) && !memcmp(have, want, have_len); + // See g_c_state_corrupted: C's output must be a prefix of Wuffs'. + bool c_corrupted = !have_err && (want_err == g_c_state_corrupted) && + (want_len <= have_len) && + !memcmp(have, want, want_len); if (both_rejected) { g_enc_both_rejected++; } else if (both_flint_failed) { g_enc_both_flint_failed++; + } else if (c_corrupted) { + g_enc_c_corrupted++; + if (g_enc_c_corrupted <= 5) { + static char desc[1 << 16]; + describe_enc(desc, sizeof desc, g_c_state_corrupted, &c, &ds, len, + src_len); + save_failure("enc-known", desc, src, src_len, &ds); + g_num_failures--; + } } else { char what[512]; snprintf(what, sizeof what, "errors: wuffs \"%s\", C \"%s\"", @@ -2529,9 +2570,10 @@ run_enc(uint64_t iters, double secs) { printf("encoder campaign: seed=%" PRIu64 " q=%u-%u configs=%" PRIu64 " failures=%" PRIu64 " skipped(workbuf>3GiB)=%" PRIu64 " both_rejected=%" PRIu64 " both_flint_failed=%" PRIu64 - " roundtrips=%" PRIu64 "\n", + " c_state_corrupted=%" PRIu64 " roundtrips=%" PRIu64 "\n", g_seed, g_qmin, g_qmax, g_enc_configs, g_num_failures, g_enc_skipped, - g_enc_both_rejected, g_enc_both_flint_failed, g_enc_roundtrips); + g_enc_both_rejected, g_enc_both_flint_failed, g_enc_c_corrupted, + g_enc_roundtrips); printf(" by quality:"); for (int q = 0; q < 12; q++) { printf(" q%d=%" PRIu64, q, g_enc_by_quality[q]); @@ -2558,10 +2600,13 @@ run_enc(uint64_t iters, double secs) { // return the index of the first step that fails (or num_steps). A C step // fails if BrotliEncoderCompressStream returns BROTLI_FALSE, a Wuffs step if // request_flush, request_emit_metadata or transform_io returns an error. +static bool g_steps_c_corrupted; + static size_t // run_steps_c(const enc_config* c, const uint8_t* src, uint8_t* out, size_t out_cap, size_t* out_len) { *out_len = 0; + g_steps_c_corrupted = false; BrotliEncoderState* state = BrotliEncoderCreateInstance(NULL, NULL, NULL); for (size_t j = 0; j < c->num_params; j++) { BrotliEncoderSetParameter(state, (BrotliEncoderParameter)(c->keys[j]), @@ -2582,6 +2627,7 @@ run_steps_c(const enc_config* c, const uint8_t* src, uint8_t* out, uint8_t* next_out = out + *out_len; if (!BrotliEncoderCompressStream(state, op, &avail_in, &next_in, &avail_out, &next_out, NULL)) { + g_steps_c_corrupted = c_state_corrupted(state); failed = true; break; } @@ -2690,6 +2736,7 @@ run_steps_wuffs(const enc_config* c, const uint8_t* src_ptr, size_t src_len, static uint64_t g_api_checks; static uint64_t g_api_both_failed_meta; // Too-long metadata. static uint64_t g_api_both_failed_seq; // Bad call sequence. +static uint64_t g_api_c_corrupted; // See g_c_state_corrupted. static void // api_fail(const char* fmt, const char* a, uint64_t x, uint64_t y) { @@ -2819,6 +2866,12 @@ api_compare_steps(const char* what, const enc_config* c, const uint8_t* src, bad = true; } } + if (bad && g_steps_c_corrupted && (cf < wf) && (want_len <= have_len) && + !memcmp(have, want, want_len)) { + // See g_c_state_corrupted. + g_api_c_corrupted++; + return; + } if (bad) { static char desc[1 << 16]; char what2[512]; @@ -3111,7 +3164,7 @@ run_api(uint64_t iters, double secs) { " failures=%" PRIu64 "\n", g_seed, fixed, it, g_api_checks, g_num_failures); printf(" both rejected: too-long metadata=%" PRIu64 - ", bad call sequence=%" PRIu64 "\n", - g_api_both_failed_meta, g_api_both_failed_seq); + ", bad call sequence=%" PRIu64 "; C state corrupted=%" PRIu64 "\n", + g_api_both_failed_meta, g_api_both_failed_seq, g_api_c_corrupted); return g_num_failures ? 1 : 0; } diff --git a/std/brotli/README.md b/std/brotli/README.md index e63f318b..115c6318 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -644,6 +644,21 @@ returns `"#bad call sequence"` there, after writing the same bytes as the C code. The differential fuzzer (`script/manual-test-brotli-fuzz.c -enc`) checks that both encoders fail at the same point with equal output. +One libbrotlienc bug is not mirrored. When `ProcessMetadata` has no room in +`dst` for a metadata block's body, it copies up to 16 bytes into `tiny_buf_` +and points `next_out_` at it. Pushing those out leaves `next_out_` just past +`tiny_buf_` (at `remaining_metadata_bytes_`), and nothing resets it when the +metadata block ends. At qualities 0 and 1, `BrotliEncoderCompressStreamFast` +compresses into the caller's buffer without touching `next_out_`, so a later +flush's `InjectBytePaddingBlock` writes its padding bits through the stale +pointer, over `remaining_metadata_bytes_`. The bytes written are still right, +but every later `BrotliEncoderCompressStream` call fails, as if a metadata +block were in progress. The Wuffs encoder writes the same bytes and carries +on, which is what libbrotlienc does once `ProcessMetadata` sets `next_out_` to +`NULL` when the metadata block ends. The +`test_mimic_brotli_encode_metadata_stale_pointer` test in +`test/c/std/brotli.c` shows it at quality 1. + ### Encoder Large Window diff --git a/test/c/std/brotli.c b/test/c/std/brotli.c index 060c7549..ebfd3af7 100644 --- a/test/c/std/brotli.c +++ b/test/c/std/brotli.c @@ -2219,6 +2219,96 @@ test_mimic_brotli_encode_parity_q10_params() { return NULL; } +// test_mimic_brotli_encode_metadata_stale_pointer checks a libbrotlienc bug +// that Wuffs does not mirror. At quality 1, a metadata block whose body goes +// through the C code's 16-byte tiny_buf_ (dst has no room) leaves next_out_ +// just past tiny_buf_, at remaining_metadata_bytes_. Qualities 0 and 1 then +// compress in place without resetting next_out_, so the next flush's +// InjectBytePaddingBlock writes over remaining_metadata_bytes_ (its output is +// still right) and the next BrotliEncoderCompressStream call fails. Wuffs +// produces the same bytes and carries on, as libbrotlienc does when +// ProcessMetadata sets next_out_ to NULL when it ends. See "Encoder Operation +// Mapping" in std/brotli/README.md. +const char* // +test_mimic_brotli_encode_metadata_stale_pointer() { + CHECK_FOCUS(__func__); + uint8_t src[300]; + static const char fox[] = "the quick brown fox jumps over the lazy dog "; + for (int i = 0; i < 300; i++) { + src[i] = (uint8_t)(fox[i % 44] ^ (i / 97)); + } + // PROCESS 100 bytes, EMIT_METADATA 32 bytes (with 1 byte of dst per call), + // PROCESS 100 bytes, FLUSH, FINISH 68 bytes. + static const uint32_t ops[5] = { + BROTLI_ENC_OP_PROCESS, BROTLI_ENC_OP_EMIT_METADATA, + BROTLI_ENC_OP_PROCESS, BROTLI_ENC_OP_FLUSH, BROTLI_ENC_OP_FINISH, + }; + static const size_t lens[5] = {100, 32, 100, 0, 68}; + + // libbrotlienc. + BrotliEncoderState* state = BrotliEncoderCreateInstance(NULL, NULL, NULL); + if (!state) { + RETURN_FAIL("BrotliEncoderCreateInstance failed"); + } + BrotliEncoderSetParameter(state, BROTLI_PARAM_QUALITY, 1); + BrotliEncoderSetParameter(state, BROTLI_PARAM_LGWIN, 16); + const uint8_t* next_in = src; + size_t want_len = 0; + int failed_step = -1; + for (int i = 0; (i < 5) && (failed_step < 0); i++) { + size_t avail_in = lens[i]; + while (true) { + size_t room = (ops[i] == BROTLI_ENC_OP_EMIT_METADATA) ? 1 : 1000; + size_t avail_out = room; + uint8_t* next_out = g_want_slice_u8.ptr + want_len; + if (!BrotliEncoderCompressStream(state, (BrotliEncoderOperation)ops[i], + &avail_in, &next_in, &avail_out, + &next_out, NULL)) { + failed_step = i; + break; + } + want_len += room - avail_out; + if ((ops[i] == BROTLI_ENC_OP_FINISH) ? BrotliEncoderIsFinished(state) + : ((avail_in == 0) && + !BrotliEncoderHasMoreOutput( + state))) { + break; + } + } + } + BrotliEncoderDestroyInstance(state); + if (failed_step != 4) { + RETURN_FAIL("libbrotlienc: failed step: have %d, want 4", failed_step); + } + + // Wuffs. + wuffs_base__io_buffer have = ((wuffs_base__io_buffer){ + .data = g_have_slice_u8, + }); + wuffs_base__io_buffer in = wuffs_base__ptr_u8__reader(src, 300, false); + uint32_t keys[2] = {BROTLI_ENC_PARAM_QUALITY, BROTLI_ENC_PARAM_LGWIN}; + uint32_t values[2] = {1, 16}; + CHECK_STRING( + wuffs_brotli_encode_plan(&have, &in, keys, values, 2, ops, lens, 5, 1)); + if ((have.meta.wi <= want_len) || + memcmp(have.data.ptr, g_want_slice_u8.ptr, want_len)) { + RETURN_FAIL("Wuffs output (%zu bytes) does not extend C's (%zu bytes)", + have.meta.wi, want_len); + } + + // The Wuffs output decodes to the data (without the metadata). + size_t decoded_len = g_work_slice_u8.len; + if (BrotliDecoderDecompress(have.meta.wi, have.data.ptr, &decoded_len, + g_work_slice_u8.ptr) != BROTLI_DECODER_RESULT_SUCCESS) { + RETURN_FAIL("BrotliDecoderDecompress failed"); + } else if ((decoded_len != 268) || + memcmp(g_work_slice_u8.ptr, src, 100) || + memcmp(g_work_slice_u8.ptr + 100, src + 132, 168)) { + RETURN_FAIL("round trip mismatch"); + } + return NULL; +} + const char* // do_test_mimic_brotli_encode_file(const char* name, const uint8_t* data, @@ -4541,6 +4631,7 @@ proc g_tests[] = { test_mimic_brotli_encode_dictionary_rejections, test_mimic_brotli_encode_large_window_far, test_mimic_brotli_encode_large_window_files, + test_mimic_brotli_encode_metadata_stale_pointer, test_mimic_brotli_encode_parity_corpus, test_mimic_brotli_encode_parity_grid, test_mimic_brotli_encode_parity_metadata,