From 11e73576bb54dea193f1ba6bc3e29ba5146fd11d Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 20 Mar 2019 21:20:14 -0700 Subject: [PATCH 1/3] [regression] Add more streaming tests * Test all of the `ZSTD_initCStream*()` variants. * Fix a typo in the zstdcli method. --- tests/regression/config.c | 2 +- tests/regression/method.c | 217 ++++++++++++++++++------- tests/regression/results.csv | 306 ++++++++++++++++++++++++++++++++++- 3 files changed, 465 insertions(+), 60 deletions(-) diff --git a/tests/regression/config.c b/tests/regression/config.c index 262cb605d..b82482f46 100644 --- a/tests/regression/config.c +++ b/tests/regression/config.c @@ -97,7 +97,7 @@ static param_value_t mt_advanced_param_values[] = { static config_t mt_advanced = { .name = "multithreaded with advanced params", - .cli_args = "-T2 --no-compressed-literals", + .cli_args = "-T2 --no-compress-literals", .param_values = PARAM_VALUES(mt_advanced_param_values), }; diff --git a/tests/regression/method.c b/tests/regression/method.c index 16701c0b4..1e84021c3 100644 --- a/tests/regression/method.c +++ b/tests/regression/method.c @@ -432,77 +432,158 @@ out: return result; } -static result_t old_streaming_compress( - method_state_t* base, - config_t const* config) { - buffer_state_t* state = container_of(base, buffer_state_t, base); - - if (buffer_state_bad(state, config)) - return result_error(result_error_system_error); - - int const level = config_get_level(config); - if (level == CONFIG_NO_LEVEL) - return result_error(result_error_skip); - - ZSTD_CStream* zcs = ZSTD_createCStream(); - result_t result; - if (zcs == NULL) { - result = result_error(result_error_compression_error); - goto out; - } +static int init_cstream( + buffer_state_t* state, + ZSTD_CStream* zcs, + config_t const* config, + int const advanced, + ZSTD_CDict** cdict) +{ size_t zret; - if (config->use_dictionary) { - zret = ZSTD_initCStream_usingDict( - zcs, state->dictionary.data, state->dictionary.size, level); + if (advanced) { + ZSTD_parameters const params = config_get_zstd_params(config, 0, 0); + ZSTD_CDict* dict = NULL; + if (cdict) { + *cdict = ZSTD_createCDict_advanced( + state->dictionary.data, + state->dictionary.size, + ZSTD_dlm_byRef, + ZSTD_dct_auto, + params.cParams, + ZSTD_defaultCMem); + if (!*cdict) { + return 1; + } + zret = ZSTD_initCStream_usingCDict_advanced( + zcs, *cdict, params.fParams, ZSTD_CONTENTSIZE_UNKNOWN); + } else { + zret = ZSTD_initCStream_advanced( + zcs, + state->dictionary.data, + state->dictionary.size, + params, + ZSTD_CONTENTSIZE_UNKNOWN); + } } else { - zret = ZSTD_initCStream(zcs, level); + int const level = config_get_level(config); + if (cdict) { + *cdict = ZSTD_createCDict( + state->dictionary.data, + state->dictionary.size, + level); + if (!*cdict) { + return 1; + } + zret = ZSTD_initCStream_usingCDict(zcs, *cdict); + } else if (config->use_dictionary) { + zret = ZSTD_initCStream_usingDict( + zcs, state->dictionary.data, state->dictionary.size, level); + } else { + zret = ZSTD_initCStream(zcs, level); + } } if (ZSTD_isError(zret)) { - result = result_error(result_error_compression_error); - goto out; + return 1; + } + return 0; +} + +static result_t old_streaming_compress_internal( + method_state_t* base, + config_t const* config, + int const advanced, + int const cdict) { + buffer_state_t* state = container_of(base, buffer_state_t, base); + + if (buffer_state_bad(state, config)) + return result_error(result_error_system_error); + + + ZSTD_CStream* zcs = ZSTD_createCStream(); + ZSTD_CDict* cd = NULL; + result_t result; + if (zcs == NULL) { + result = result_error(result_error_compression_error); + goto out; + } + if (init_cstream(state, zcs, config, advanced, cdict ? &cd : NULL)) { + result = result_error(result_error_compression_error); + goto out; + } + + result_data_t data = {.total_size = 0}; + for (size_t i = 0; i < state->inputs.size; ++i) { + data_buffer_t input = state->inputs.buffers[i]; + size_t zret = ZSTD_resetCStream( + zcs, + config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN : input.size); + if (ZSTD_isError(zret)) { + result = result_error(result_error_compression_error); + goto out; } - result_data_t data = {.total_size = 0}; - for (size_t i = 0; i < state->inputs.size; ++i) { - data_buffer_t input = state->inputs.buffers[i]; - zret = ZSTD_resetCStream( - zcs, - config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN - : input.size); + while (input.size > 0) { + ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)}; + input.data += in.size; + input.size -= in.size; + ZSTD_EndDirective const op = + input.size > 0 ? ZSTD_e_continue : ZSTD_e_end; + zret = 0; + while (in.pos < in.size || (op == ZSTD_e_end && zret != 0)) { + ZSTD_outBuffer out = {state->compressed.data, + MIN(state->compressed.capacity, 1024)}; + if (op == ZSTD_e_continue || in.pos < in.size) + zret = ZSTD_compressStream(zcs, &out, &in); + else + zret = ZSTD_endStream(zcs, &out); if (ZSTD_isError(zret)) { - result = result_error(result_error_compression_error); - goto out; - } - - while (input.size > 0) { - ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)}; - input.data += in.size; - input.size -= in.size; - ZSTD_EndDirective const op = - input.size > 0 ? ZSTD_e_continue : ZSTD_e_end; - zret = 0; - while (in.pos < in.size || (op == ZSTD_e_end && zret != 0)) { - ZSTD_outBuffer out = {state->compressed.data, - MIN(state->compressed.capacity, 1024)}; - if (op == ZSTD_e_continue || in.pos < in.size) - zret = ZSTD_compressStream(zcs, &out, &in); - else - zret = ZSTD_endStream(zcs, &out); - if (ZSTD_isError(zret)) { - result = result_error(result_error_compression_error); - goto out; - } - data.total_size += out.pos; - } + result = result_error(result_error_compression_error); + goto out; } + data.total_size += out.pos; + } } + } - result = result_data(data); + result = result_data(data); out: ZSTD_freeCStream(zcs); + ZSTD_freeCDict(cd); return result; } +static result_t old_streaming_compress( + method_state_t* base, + config_t const* config) +{ + return old_streaming_compress_internal( + base, config, /* advanced */ 0, /* cdict */ 0); +} + +static result_t old_streaming_compress_advanced( + method_state_t* base, + config_t const* config) +{ + return old_streaming_compress_internal( + base, config, /* advanced */ 1, /* cdict */ 0); +} + +static result_t old_streaming_compress_cdict( + method_state_t* base, + config_t const* config) +{ + return old_streaming_compress_internal( + base, config, /* advanced */ 0, /* cdict */ 1); +} + +static result_t old_streaming_compress_cdict_advanced( + method_state_t* base, + config_t const* config) +{ + return old_streaming_compress_internal( + base, config, /* advanced */ 1, /* cdict */ 1); +} + method_t const simple = { .name = "compress simple", .create = buffer_state_create, @@ -545,6 +626,27 @@ method_t const old_streaming = { .destroy = buffer_state_destroy, }; +method_t const old_streaming_advanced = { + .name = "old streaming advanced", + .create = buffer_state_create, + .compress = old_streaming_compress, + .destroy = buffer_state_destroy, +}; + +method_t const old_streaming_cdict = { + .name = "old streaming cdcit", + .create = buffer_state_create, + .compress = old_streaming_compress, + .destroy = buffer_state_destroy, +}; + +method_t const old_streaming_advanced_cdict = { + .name = "old streaming advanced cdict", + .create = buffer_state_create, + .compress = old_streaming_compress, + .destroy = buffer_state_destroy, +}; + method_t const cli = { .name = "zstdcli", .create = method_state_create, @@ -560,6 +662,9 @@ static method_t const* g_methods[] = { &advanced_one_pass_small_out, &advanced_streaming, &old_streaming, + &old_streaming_advanced, + &old_streaming_cdict, + &old_streaming_advanced_cdict, NULL, }; diff --git a/tests/regression/results.csv b/tests/regression/results.csv index a24553b01..f5c639320 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -104,7 +104,7 @@ silesia, explicit params, zstdcli, silesia, uncompressed literals, zstdcli, 5155472 silesia, uncompressed literals optimal, zstdcli, 4325475 silesia, huffman literals, zstdcli, 5341405 -silesia, multithreaded with advanced params, zstdcli, compression error +silesia, multithreaded with advanced params, zstdcli, 5155472 silesia.tar, level -5, zstdcli, 7161160 silesia.tar, level -3, zstdcli, 6789865 silesia.tar, level -1, zstdcli, 6196433 @@ -130,7 +130,7 @@ silesia.tar, explicit params, zstdcli, silesia.tar, uncompressed literals, zstdcli, 5158134 silesia.tar, uncompressed literals optimal, zstdcli, 4321098 silesia.tar, huffman literals, zstdcli, 5358479 -silesia.tar, multithreaded with advanced params, zstdcli, compression error +silesia.tar, multithreaded with advanced params, zstdcli, 5158134 github, level -5, zstdcli, 234744 github, level -5 with dict, zstdcli, 48718 github, level -3, zstdcli, 222611 @@ -169,7 +169,7 @@ github, explicit params, zstdcli, github, uncompressed literals, zstdcli, 169004 github, uncompressed literals optimal, zstdcli, 158824 github, huffman literals, zstdcli, 145457 -github, multithreaded with advanced params, zstdcli, compression error +github, multithreaded with advanced params, zstdcli, 169004 silesia, level -5, advanced one pass, 7152294 silesia, level -3, advanced one pass, 6789969 silesia, level -1, advanced one pass, 6191548 @@ -461,9 +461,17 @@ silesia, level 13, old stre silesia, level 16, old streaming, 4377391 silesia, level 19, old streaming, 4293262 silesia, no source size, old streaming, 4862341 +silesia, long distance mode, old streaming, 12000408 +silesia, multithreaded, old streaming, 12000408 +silesia, multithreaded long distance mode, old streaming, 12000408 +silesia, small window log, old streaming, 12000408 +silesia, small hash log, old streaming, 12000408 +silesia, small chain log, old streaming, 12000408 +silesia, explicit params, old streaming, 12000408 silesia, uncompressed literals, old streaming, 4862377 silesia, uncompressed literals optimal, old streaming, 4293262 silesia, huffman literals, old streaming, 6191549 +silesia, multithreaded with advanced params, old streaming, 12000408 silesia.tar, level -5, old streaming, 7160440 silesia.tar, level -3, old streaming, 6789026 silesia.tar, level -1, old streaming, 6195465 @@ -479,9 +487,17 @@ silesia.tar, level 13, old stre silesia.tar, level 16, old streaming, 4381277 silesia.tar, level 19, old streaming, 4281514 silesia.tar, no source size, old streaming, 4875006 +silesia.tar, long distance mode, old streaming, 12022046 +silesia.tar, multithreaded, old streaming, 12022046 +silesia.tar, multithreaded long distance mode, old streaming, 12022046 +silesia.tar, small window log, old streaming, 12022046 +silesia.tar, small hash log, old streaming, 12022046 +silesia.tar, small chain log, old streaming, 12022046 +silesia.tar, explicit params, old streaming, 12022046 silesia.tar, uncompressed literals, old streaming, 4875010 silesia.tar, uncompressed literals optimal, old streaming, 4281514 silesia.tar, huffman literals, old streaming, 6195465 +silesia.tar, multithreaded with advanced params, old streaming, 12022046 github, level -5, old streaming, 232744 github, level -5 with dict, old streaming, 46718 github, level -3, old streaming, 220611 @@ -511,6 +527,290 @@ github, level 16 with dict, old stre github, level 19, old streaming, 133717 github, level 19 with dict, old streaming, 37576 github, no source size, old streaming, 141003 +github, long distance mode, old streaming, 412933 +github, multithreaded, old streaming, 412933 +github, multithreaded long distance mode, old streaming, 412933 +github, small window log, old streaming, 412933 +github, small hash log, old streaming, 412933 +github, small chain log, old streaming, 412933 +github, explicit params, old streaming, 412933 github, uncompressed literals, old streaming, 136397 github, uncompressed literals optimal, old streaming, 133717 github, huffman literals, old streaming, 176575 +github, multithreaded with advanced params, old streaming, 412933 +silesia, level -5, old streaming advanced, 7152294 +silesia, level -3, old streaming advanced, 6789973 +silesia, level -1, old streaming advanced, 6191549 +silesia, level 0, old streaming advanced, 4862377 +silesia, level 1, old streaming advanced, 5318036 +silesia, level 3, old streaming advanced, 4862377 +silesia, level 4, old streaming advanced, 4800629 +silesia, level 5, old streaming advanced, 4710178 +silesia, level 6, old streaming advanced, 4659996 +silesia, level 7, old streaming advanced, 4596234 +silesia, level 9, old streaming advanced, 4543862 +silesia, level 13, old streaming advanced, 4482073 +silesia, level 16, old streaming advanced, 4377391 +silesia, level 19, old streaming advanced, 4293262 +silesia, no source size, old streaming advanced, 4862341 +silesia, long distance mode, old streaming advanced, 12000408 +silesia, multithreaded, old streaming advanced, 12000408 +silesia, multithreaded long distance mode, old streaming advanced, 12000408 +silesia, small window log, old streaming advanced, 12000408 +silesia, small hash log, old streaming advanced, 12000408 +silesia, small chain log, old streaming advanced, 12000408 +silesia, explicit params, old streaming advanced, 12000408 +silesia, uncompressed literals, old streaming advanced, 4862377 +silesia, uncompressed literals optimal, old streaming advanced, 4293262 +silesia, huffman literals, old streaming advanced, 6191549 +silesia, multithreaded with advanced params, old streaming advanced, 12000408 +silesia.tar, level -5, old streaming advanced, 7160440 +silesia.tar, level -3, old streaming advanced, 6789026 +silesia.tar, level -1, old streaming advanced, 6195465 +silesia.tar, level 0, old streaming advanced, 4875010 +silesia.tar, level 1, old streaming advanced, 5339701 +silesia.tar, level 3, old streaming advanced, 4875010 +silesia.tar, level 4, old streaming advanced, 4813507 +silesia.tar, level 5, old streaming advanced, 4722240 +silesia.tar, level 6, old streaming advanced, 4672203 +silesia.tar, level 7, old streaming advanced, 4606658 +silesia.tar, level 9, old streaming advanced, 4554105 +silesia.tar, level 13, old streaming advanced, 4491703 +silesia.tar, level 16, old streaming advanced, 4381277 +silesia.tar, level 19, old streaming advanced, 4281514 +silesia.tar, no source size, old streaming advanced, 4875006 +silesia.tar, long distance mode, old streaming advanced, 12022046 +silesia.tar, multithreaded, old streaming advanced, 12022046 +silesia.tar, multithreaded long distance mode, old streaming advanced, 12022046 +silesia.tar, small window log, old streaming advanced, 12022046 +silesia.tar, small hash log, old streaming advanced, 12022046 +silesia.tar, small chain log, old streaming advanced, 12022046 +silesia.tar, explicit params, old streaming advanced, 12022046 +silesia.tar, uncompressed literals, old streaming advanced, 4875010 +silesia.tar, uncompressed literals optimal, old streaming advanced, 4281514 +silesia.tar, huffman literals, old streaming advanced, 6195465 +silesia.tar, multithreaded with advanced params, old streaming advanced, 12022046 +github, level -5, old streaming advanced, 232744 +github, level -5 with dict, old streaming advanced, 46718 +github, level -3, old streaming advanced, 220611 +github, level -3 with dict, old streaming advanced, 45395 +github, level -1, old streaming advanced, 176575 +github, level -1 with dict, old streaming advanced, 43170 +github, level 0, old streaming advanced, 136397 +github, level 0 with dict, old streaming advanced, 41170 +github, level 1, old streaming advanced, 143457 +github, level 1 with dict, old streaming advanced, 41682 +github, level 3, old streaming advanced, 136397 +github, level 3 with dict, old streaming advanced, 41170 +github, level 4, old streaming advanced, 136144 +github, level 4 with dict, old streaming advanced, 41306 +github, level 5, old streaming advanced, 135106 +github, level 5 with dict, old streaming advanced, 38938 +github, level 6, old streaming advanced, 135108 +github, level 6 with dict, old streaming advanced, 38632 +github, level 7, old streaming advanced, 135108 +github, level 7 with dict, old streaming advanced, 38766 +github, level 9, old streaming advanced, 135108 +github, level 9 with dict, old streaming advanced, 39326 +github, level 13, old streaming advanced, 133717 +github, level 13 with dict, old streaming advanced, 39716 +github, level 16, old streaming advanced, 133717 +github, level 16 with dict, old streaming advanced, 37577 +github, level 19, old streaming advanced, 133717 +github, level 19 with dict, old streaming advanced, 37576 +github, no source size, old streaming advanced, 141003 +github, long distance mode, old streaming advanced, 412933 +github, multithreaded, old streaming advanced, 412933 +github, multithreaded long distance mode, old streaming advanced, 412933 +github, small window log, old streaming advanced, 412933 +github, small hash log, old streaming advanced, 412933 +github, small chain log, old streaming advanced, 412933 +github, explicit params, old streaming advanced, 412933 +github, uncompressed literals, old streaming advanced, 136397 +github, uncompressed literals optimal, old streaming advanced, 133717 +github, huffman literals, old streaming advanced, 176575 +github, multithreaded with advanced params, old streaming advanced, 412933 +silesia, level -5, old streaming cdcit, 7152294 +silesia, level -3, old streaming cdcit, 6789973 +silesia, level -1, old streaming cdcit, 6191549 +silesia, level 0, old streaming cdcit, 4862377 +silesia, level 1, old streaming cdcit, 5318036 +silesia, level 3, old streaming cdcit, 4862377 +silesia, level 4, old streaming cdcit, 4800629 +silesia, level 5, old streaming cdcit, 4710178 +silesia, level 6, old streaming cdcit, 4659996 +silesia, level 7, old streaming cdcit, 4596234 +silesia, level 9, old streaming cdcit, 4543862 +silesia, level 13, old streaming cdcit, 4482073 +silesia, level 16, old streaming cdcit, 4377391 +silesia, level 19, old streaming cdcit, 4293262 +silesia, no source size, old streaming cdcit, 4862341 +silesia, long distance mode, old streaming cdcit, 12000408 +silesia, multithreaded, old streaming cdcit, 12000408 +silesia, multithreaded long distance mode, old streaming cdcit, 12000408 +silesia, small window log, old streaming cdcit, 12000408 +silesia, small hash log, old streaming cdcit, 12000408 +silesia, small chain log, old streaming cdcit, 12000408 +silesia, explicit params, old streaming cdcit, 12000408 +silesia, uncompressed literals, old streaming cdcit, 4862377 +silesia, uncompressed literals optimal, old streaming cdcit, 4293262 +silesia, huffman literals, old streaming cdcit, 6191549 +silesia, multithreaded with advanced params, old streaming cdcit, 12000408 +silesia.tar, level -5, old streaming cdcit, 7160440 +silesia.tar, level -3, old streaming cdcit, 6789026 +silesia.tar, level -1, old streaming cdcit, 6195465 +silesia.tar, level 0, old streaming cdcit, 4875010 +silesia.tar, level 1, old streaming cdcit, 5339701 +silesia.tar, level 3, old streaming cdcit, 4875010 +silesia.tar, level 4, old streaming cdcit, 4813507 +silesia.tar, level 5, old streaming cdcit, 4722240 +silesia.tar, level 6, old streaming cdcit, 4672203 +silesia.tar, level 7, old streaming cdcit, 4606658 +silesia.tar, level 9, old streaming cdcit, 4554105 +silesia.tar, level 13, old streaming cdcit, 4491703 +silesia.tar, level 16, old streaming cdcit, 4381277 +silesia.tar, level 19, old streaming cdcit, 4281514 +silesia.tar, no source size, old streaming cdcit, 4875006 +silesia.tar, long distance mode, old streaming cdcit, 12022046 +silesia.tar, multithreaded, old streaming cdcit, 12022046 +silesia.tar, multithreaded long distance mode, old streaming cdcit, 12022046 +silesia.tar, small window log, old streaming cdcit, 12022046 +silesia.tar, small hash log, old streaming cdcit, 12022046 +silesia.tar, small chain log, old streaming cdcit, 12022046 +silesia.tar, explicit params, old streaming cdcit, 12022046 +silesia.tar, uncompressed literals, old streaming cdcit, 4875010 +silesia.tar, uncompressed literals optimal, old streaming cdcit, 4281514 +silesia.tar, huffman literals, old streaming cdcit, 6195465 +silesia.tar, multithreaded with advanced params, old streaming cdcit, 12022046 +github, level -5, old streaming cdcit, 232744 +github, level -5 with dict, old streaming cdcit, 46718 +github, level -3, old streaming cdcit, 220611 +github, level -3 with dict, old streaming cdcit, 45395 +github, level -1, old streaming cdcit, 176575 +github, level -1 with dict, old streaming cdcit, 43170 +github, level 0, old streaming cdcit, 136397 +github, level 0 with dict, old streaming cdcit, 41170 +github, level 1, old streaming cdcit, 143457 +github, level 1 with dict, old streaming cdcit, 41682 +github, level 3, old streaming cdcit, 136397 +github, level 3 with dict, old streaming cdcit, 41170 +github, level 4, old streaming cdcit, 136144 +github, level 4 with dict, old streaming cdcit, 41306 +github, level 5, old streaming cdcit, 135106 +github, level 5 with dict, old streaming cdcit, 38938 +github, level 6, old streaming cdcit, 135108 +github, level 6 with dict, old streaming cdcit, 38632 +github, level 7, old streaming cdcit, 135108 +github, level 7 with dict, old streaming cdcit, 38766 +github, level 9, old streaming cdcit, 135108 +github, level 9 with dict, old streaming cdcit, 39326 +github, level 13, old streaming cdcit, 133717 +github, level 13 with dict, old streaming cdcit, 39716 +github, level 16, old streaming cdcit, 133717 +github, level 16 with dict, old streaming cdcit, 37577 +github, level 19, old streaming cdcit, 133717 +github, level 19 with dict, old streaming cdcit, 37576 +github, no source size, old streaming cdcit, 141003 +github, long distance mode, old streaming cdcit, 412933 +github, multithreaded, old streaming cdcit, 412933 +github, multithreaded long distance mode, old streaming cdcit, 412933 +github, small window log, old streaming cdcit, 412933 +github, small hash log, old streaming cdcit, 412933 +github, small chain log, old streaming cdcit, 412933 +github, explicit params, old streaming cdcit, 412933 +github, uncompressed literals, old streaming cdcit, 136397 +github, uncompressed literals optimal, old streaming cdcit, 133717 +github, huffman literals, old streaming cdcit, 176575 +github, multithreaded with advanced params, old streaming cdcit, 412933 +silesia, level -5, old streaming advanced cdict, 7152294 +silesia, level -3, old streaming advanced cdict, 6789973 +silesia, level -1, old streaming advanced cdict, 6191549 +silesia, level 0, old streaming advanced cdict, 4862377 +silesia, level 1, old streaming advanced cdict, 5318036 +silesia, level 3, old streaming advanced cdict, 4862377 +silesia, level 4, old streaming advanced cdict, 4800629 +silesia, level 5, old streaming advanced cdict, 4710178 +silesia, level 6, old streaming advanced cdict, 4659996 +silesia, level 7, old streaming advanced cdict, 4596234 +silesia, level 9, old streaming advanced cdict, 4543862 +silesia, level 13, old streaming advanced cdict, 4482073 +silesia, level 16, old streaming advanced cdict, 4377391 +silesia, level 19, old streaming advanced cdict, 4293262 +silesia, no source size, old streaming advanced cdict, 4862341 +silesia, long distance mode, old streaming advanced cdict, 12000408 +silesia, multithreaded, old streaming advanced cdict, 12000408 +silesia, multithreaded long distance mode, old streaming advanced cdict, 12000408 +silesia, small window log, old streaming advanced cdict, 12000408 +silesia, small hash log, old streaming advanced cdict, 12000408 +silesia, small chain log, old streaming advanced cdict, 12000408 +silesia, explicit params, old streaming advanced cdict, 12000408 +silesia, uncompressed literals, old streaming advanced cdict, 4862377 +silesia, uncompressed literals optimal, old streaming advanced cdict, 4293262 +silesia, huffman literals, old streaming advanced cdict, 6191549 +silesia, multithreaded with advanced params, old streaming advanced cdict, 12000408 +silesia.tar, level -5, old streaming advanced cdict, 7160440 +silesia.tar, level -3, old streaming advanced cdict, 6789026 +silesia.tar, level -1, old streaming advanced cdict, 6195465 +silesia.tar, level 0, old streaming advanced cdict, 4875010 +silesia.tar, level 1, old streaming advanced cdict, 5339701 +silesia.tar, level 3, old streaming advanced cdict, 4875010 +silesia.tar, level 4, old streaming advanced cdict, 4813507 +silesia.tar, level 5, old streaming advanced cdict, 4722240 +silesia.tar, level 6, old streaming advanced cdict, 4672203 +silesia.tar, level 7, old streaming advanced cdict, 4606658 +silesia.tar, level 9, old streaming advanced cdict, 4554105 +silesia.tar, level 13, old streaming advanced cdict, 4491703 +silesia.tar, level 16, old streaming advanced cdict, 4381277 +silesia.tar, level 19, old streaming advanced cdict, 4281514 +silesia.tar, no source size, old streaming advanced cdict, 4875006 +silesia.tar, long distance mode, old streaming advanced cdict, 12022046 +silesia.tar, multithreaded, old streaming advanced cdict, 12022046 +silesia.tar, multithreaded long distance mode, old streaming advanced cdict, 12022046 +silesia.tar, small window log, old streaming advanced cdict, 12022046 +silesia.tar, small hash log, old streaming advanced cdict, 12022046 +silesia.tar, small chain log, old streaming advanced cdict, 12022046 +silesia.tar, explicit params, old streaming advanced cdict, 12022046 +silesia.tar, uncompressed literals, old streaming advanced cdict, 4875010 +silesia.tar, uncompressed literals optimal, old streaming advanced cdict, 4281514 +silesia.tar, huffman literals, old streaming advanced cdict, 6195465 +silesia.tar, multithreaded with advanced params, old streaming advanced cdict, 12022046 +github, level -5, old streaming advanced cdict, 232744 +github, level -5 with dict, old streaming advanced cdict, 46718 +github, level -3, old streaming advanced cdict, 220611 +github, level -3 with dict, old streaming advanced cdict, 45395 +github, level -1, old streaming advanced cdict, 176575 +github, level -1 with dict, old streaming advanced cdict, 43170 +github, level 0, old streaming advanced cdict, 136397 +github, level 0 with dict, old streaming advanced cdict, 41170 +github, level 1, old streaming advanced cdict, 143457 +github, level 1 with dict, old streaming advanced cdict, 41682 +github, level 3, old streaming advanced cdict, 136397 +github, level 3 with dict, old streaming advanced cdict, 41170 +github, level 4, old streaming advanced cdict, 136144 +github, level 4 with dict, old streaming advanced cdict, 41306 +github, level 5, old streaming advanced cdict, 135106 +github, level 5 with dict, old streaming advanced cdict, 38938 +github, level 6, old streaming advanced cdict, 135108 +github, level 6 with dict, old streaming advanced cdict, 38632 +github, level 7, old streaming advanced cdict, 135108 +github, level 7 with dict, old streaming advanced cdict, 38766 +github, level 9, old streaming advanced cdict, 135108 +github, level 9 with dict, old streaming advanced cdict, 39326 +github, level 13, old streaming advanced cdict, 133717 +github, level 13 with dict, old streaming advanced cdict, 39716 +github, level 16, old streaming advanced cdict, 133717 +github, level 16 with dict, old streaming advanced cdict, 37577 +github, level 19, old streaming advanced cdict, 133717 +github, level 19 with dict, old streaming advanced cdict, 37576 +github, no source size, old streaming advanced cdict, 141003 +github, long distance mode, old streaming advanced cdict, 412933 +github, multithreaded, old streaming advanced cdict, 412933 +github, multithreaded long distance mode, old streaming advanced cdict, 412933 +github, small window log, old streaming advanced cdict, 412933 +github, small hash log, old streaming advanced cdict, 412933 +github, small chain log, old streaming advanced cdict, 412933 +github, explicit params, old streaming advanced cdict, 412933 +github, uncompressed literals, old streaming advanced cdict, 136397 +github, uncompressed literals optimal, old streaming advanced cdict, 133717 +github, huffman literals, old streaming advanced cdict, 176575 +github, multithreaded with advanced params, old streaming advanced cdict, 412933 From e55da9e9638ef504f93a0ed448a47d0a8d745657 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 13 Mar 2019 14:05:18 -0700 Subject: [PATCH 2/3] Wrap the new advanced api completely --- lib/compress/zstd_compress.c | 112 ++++++++++++++------------ lib/compress/zstd_compress_internal.h | 7 -- lib/compress/zstdmt_compress.c | 2 +- tests/zstreamtest.c | 15 ++-- 4 files changed, 70 insertions(+), 66 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 987c7b90a..62dab8003 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3769,13 +3769,17 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx, /* ZSTD_resetCStream(): * pledgedSrcSize == 0 means "unknown" */ -size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) { - ZSTD_CCtx_params params = zcs->requestedParams; + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize); - if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; - params.fParams.contentSizeFlag = 1; - return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); + return 0; } /*! ZSTD_initCStream_internal() : @@ -3787,31 +3791,18 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_initCStream_internal"); - params.cParams = ZSTD_getCParamsFromCCtxParams(¶ms, pledgedSrcSize, dictSize); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + zcs->requestedParams = params; assert(!((dict) && (cdict))); /* either dict or cdict, not both */ - - if (dict && dictSize >= 8) { - DEBUGLOG(4, "loading dictionary of size %u", (unsigned)dictSize); - RETURN_ERROR_IF( - zcs->staticSize, memory_allocation, - "static CCtx: incompatible with internal cdict creation"); - ZSTD_freeCDict(zcs->cdictLocal); - zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, - ZSTD_dlm_byCopy, ZSTD_dct_auto, - params.cParams, zcs->customMem); - zcs->cdict = zcs->cdictLocal; - RETURN_ERROR_IF(zcs->cdictLocal == NULL, memory_allocation); + if (dict) { + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); } else { - if (cdict) { - params.cParams = ZSTD_getCParamsFromCDict(cdict); /* cParams are enforced from cdict; it includes windowLog */ - } - ZSTD_freeCDict(zcs->cdictLocal); - zcs->cdictLocal = NULL; - zcs->cdict = cdict; + /* Dictionary is cleared if !cdict */ + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) ); } - - return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize); + return 0; } /* ZSTD_initCStream_usingCDict_advanced() : @@ -3822,23 +3813,20 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced"); - RETURN_ERROR_IF(!cdict, dictionary_wrong, - "cannot handle NULL cdict (does not know what to do)"); - { ZSTD_CCtx_params params = zcs->requestedParams; - params.cParams = ZSTD_getCParamsFromCDict(cdict); - params.fParams = fParams; - return ZSTD_initCStream_internal(zcs, - NULL, 0, cdict, - params, pledgedSrcSize); - } + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); + zcs->requestedParams.fParams = fParams; + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) ); + return 0; } /* note : cdict must outlive compression session */ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) { - ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, 0 /* checksum */, 0 /* hideDictID */ }; DEBUGLOG(4, "ZSTD_initCStream_usingCDict"); - return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); /* note : will check that cdict != NULL */ + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) ); + return 0; } @@ -3848,33 +3836,53 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) * dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize) + ZSTD_parameters params, unsigned long long pss) { - DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u", - (unsigned)pledgedSrcSize, params.fParams.contentSizeFlag); + /* for compatibility with older programs relying on this behavior. + * Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. + * This line will be removed in the future. + */ + U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_advanced"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) ); - if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0)) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* for compatibility with older programs relying on this behavior. Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. This line will be removed in the future. */ zcs->requestedParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); - return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL /*cdict*/, zcs->requestedParams, pledgedSrcSize); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); + return 0; } size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel); - return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, zcs->requestedParams, ZSTD_CONTENTSIZE_UNKNOWN); + DEBUGLOG(4, "ZSTD_initCStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) ); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); + return 0; } size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss) { - U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; /* temporary : 0 interpreted as "unknown" during transition period. Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. `0` will be interpreted as "empty" in the future */ - ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel); - return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, zcs->requestedParams, pledgedSrcSize); + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_srcSize"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); + return 0; } size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) { DEBUGLOG(4, "ZSTD_initCStream"); - return ZSTD_initCStream_srcSize(zcs, compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) ); + return 0; } /*====== Compression ======*/ @@ -3898,10 +3906,10 @@ static size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, * internal function for all *compressStream*() variants * non-static, because can be called from zstdmt_compress.c * @return : hint size for next input */ -size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective const flushMode) +static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective const flushMode) { const char* const istart = (const char*)input->src; const char* const iend = istart + input->size; diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index a828de3b4..c90034d4f 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -808,13 +808,6 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, void ZSTD_resetSeqStore(seqStore_t* ssPtr); -/*! ZSTD_compressStream_generic() : - * Private use only. To be called from zstdmt_compress.c in single-thread mode. */ -size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective const flushMode); - /*! ZSTD_getCParamsFromCDict() : * as the name implies */ ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict); diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index dab3a9508..7e2c78984 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -1967,7 +1967,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, assert(input->pos <= input->size); if (mtctx->singleBlockingThread) { /* delegate to single-thread (synchronous) */ - return ZSTD_compressStream_generic(mtctx->cctxPool->cctx[0], output, input, endOp); + return ZSTD_compressStream2(mtctx->cctxPool->cctx[0], output, input, endOp); } if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) { diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index b099e2870..e26375e07 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -495,7 +495,7 @@ static int basicUnitTests(U32 seed, double compressibility) /* _srcSize compression test */ DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); - ZSTD_initCStream_srcSize(zc, 1, CNBufferSize); + CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) ); outBuff.dst = (char*)(compressedBuffer); outBuff.size = compressedBufferSize; outBuff.pos = 0; @@ -503,11 +503,14 @@ static int basicUnitTests(U32 seed, double compressibility) inBuff.size = CNBufferSize; inBuff.pos = 0; CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); - if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ - { size_t const r = ZSTD_endStream(zc, &outBuff); - if (r != 0) goto _output_error; } /* error, or some data not flushed */ - { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos); - if ((size_t)origSize != CNBufferSize) goto _output_error; } /* exact original size must be present */ + CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed"); + { size_t const r = ZSTD_endStream(zc, &outBuff); + CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r); + } + { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos); + CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!"); + CHECK((size_t)origSize != CNBufferSize, "Exact original size must be present (got %llu)", origSize); + } DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); /* wrong _srcSize compression test */ From 20f9ff7e5361916560e95336c89e8dc8564ba230 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 15 Mar 2019 18:03:31 -0700 Subject: [PATCH 3/3] Update documentation to tell how to replace the old streaming API with the new one. --- lib/zstd.h | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 74 insertions(+), 5 deletions(-) diff --git a/lib/zstd.h b/lib/zstd.h index f6332589d..c1c0fe744 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -347,9 +347,24 @@ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /*===== Streaming compression functions =====*/ +/** + * Equivalent to: + * + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + */ ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +/** + * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). + * NOTE: The return value is different. ZSTD_compressStream() returns a hint for + * the next read size (if non-zero and not an error). ZSTD_compressStream2() + * returns the number of bytes left to flush (if non-zero and not an error). + */ ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ @@ -1542,14 +1557,68 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( ********************************************************************/ /*===== Advanced Streaming compression functions =====*/ -ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */ -ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/ +/**! ZSTD_initCStream_srcSize() : + * This function is deprecated, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * + * pledgedSrcSize must be correct. If it is not known at init time, use + * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + * "0" also disables frame content size field. It may be enabled in the future. + */ +ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); +/**! ZSTD_initCStream_usingDict() : + * This function is deprecated, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * Creates of an internal CDict (incompatible with static CCtx), except if + * dict == NULL or dictSize < 8, in which case no dict is used. + * Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if + * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + */ +ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); +/**! ZSTD_initCStream_advanced() : + * This function is deprecated, and is approximately equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. + */ ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */ -ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ -ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */ + ZSTD_parameters params, unsigned long long pledgedSrcSize); +/**! ZSTD_initCStream_usingCDict() : + * This function is deprecated, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * note : cdict will just be referenced, and must outlive compression session + */ +ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); +/**! ZSTD_initCStream_usingCDict_advanced() : + * This function is deprecated, and is approximately equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. + */ +ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /*! ZSTD_resetCStream() : + * This function is deprecated, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * * start a new frame, using same parameters from previous frame. * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. * Note that zcs must be init at least once before using ZSTD_resetCStream().