Merge pull request #1551 from terrelln/stream-wrap
Refactor old stream API to completely wrap the advanced API
This commit is contained in:
Nick Terrell
@@ -3769,13 +3769,17 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx,
|
||||
|
||||
/* ZSTD_resetCStream():
|
||||
* pledgedSrcSize == 0 means "unknown" */
|
||||
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
|
||||
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss)
|
||||
{
|
||||
ZSTD_CCtx_params params = zcs->requestedParams;
|
||||
/* temporary : 0 interpreted as "unknown" during transition period.
|
||||
* Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN.
|
||||
* 0 will be interpreted as "empty" in the future.
|
||||
*/
|
||||
U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;
|
||||
DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize);
|
||||
if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize);
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) );
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*! ZSTD_initCStream_internal() :
|
||||
@@ -3787,31 +3791,18 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
|
||||
ZSTD_CCtx_params params, unsigned long long pledgedSrcSize)
|
||||
{
|
||||
DEBUGLOG(4, "ZSTD_initCStream_internal");
|
||||
params.cParams = ZSTD_getCParamsFromCCtxParams(¶ms, pledgedSrcSize, dictSize);
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) );
|
||||
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
|
||||
zcs->requestedParams = params;
|
||||
assert(!((dict) && (cdict))); /* either dict or cdict, not both */
|
||||
|
||||
if (dict && dictSize >= 8) {
|
||||
DEBUGLOG(4, "loading dictionary of size %u", (unsigned)dictSize);
|
||||
RETURN_ERROR_IF(
|
||||
zcs->staticSize, memory_allocation,
|
||||
"static CCtx: incompatible with internal cdict creation");
|
||||
ZSTD_freeCDict(zcs->cdictLocal);
|
||||
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
|
||||
ZSTD_dlm_byCopy, ZSTD_dct_auto,
|
||||
params.cParams, zcs->customMem);
|
||||
zcs->cdict = zcs->cdictLocal;
|
||||
RETURN_ERROR_IF(zcs->cdictLocal == NULL, memory_allocation);
|
||||
if (dict) {
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) );
|
||||
} else {
|
||||
if (cdict) {
|
||||
params.cParams = ZSTD_getCParamsFromCDict(cdict); /* cParams are enforced from cdict; it includes windowLog */
|
||||
}
|
||||
ZSTD_freeCDict(zcs->cdictLocal);
|
||||
zcs->cdictLocal = NULL;
|
||||
zcs->cdict = cdict;
|
||||
/* Dictionary is cleared if !cdict */
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) );
|
||||
}
|
||||
|
||||
return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ZSTD_initCStream_usingCDict_advanced() :
|
||||
@@ -3822,23 +3813,20 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
|
||||
unsigned long long pledgedSrcSize)
|
||||
{
|
||||
DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced");
|
||||
RETURN_ERROR_IF(!cdict, dictionary_wrong,
|
||||
"cannot handle NULL cdict (does not know what to do)");
|
||||
{ ZSTD_CCtx_params params = zcs->requestedParams;
|
||||
params.cParams = ZSTD_getCParamsFromCDict(cdict);
|
||||
params.fParams = fParams;
|
||||
return ZSTD_initCStream_internal(zcs,
|
||||
NULL, 0, cdict,
|
||||
params, pledgedSrcSize);
|
||||
}
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) );
|
||||
zcs->requestedParams.fParams = fParams;
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) );
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* note : cdict must outlive compression session */
|
||||
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
|
||||
{
|
||||
ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, 0 /* checksum */, 0 /* hideDictID */ };
|
||||
DEBUGLOG(4, "ZSTD_initCStream_usingCDict");
|
||||
return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); /* note : will check that cdict != NULL */
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) );
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -3848,33 +3836,53 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
|
||||
* dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */
|
||||
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize)
|
||||
ZSTD_parameters params, unsigned long long pss)
|
||||
{
|
||||
DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u",
|
||||
(unsigned)pledgedSrcSize, params.fParams.contentSizeFlag);
|
||||
/* for compatibility with older programs relying on this behavior.
|
||||
* Users should now specify ZSTD_CONTENTSIZE_UNKNOWN.
|
||||
* This line will be removed in the future.
|
||||
*/
|
||||
U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;
|
||||
DEBUGLOG(4, "ZSTD_initCStream_advanced");
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) );
|
||||
FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) );
|
||||
if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0)) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* for compatibility with older programs relying on this behavior. Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. This line will be removed in the future. */
|
||||
zcs->requestedParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
|
||||
return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL /*cdict*/, zcs->requestedParams, pledgedSrcSize);
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) );
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel);
|
||||
return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, zcs->requestedParams, ZSTD_CONTENTSIZE_UNKNOWN);
|
||||
DEBUGLOG(4, "ZSTD_initCStream_usingDict");
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) );
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) );
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss)
|
||||
{
|
||||
U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; /* temporary : 0 interpreted as "unknown" during transition period. Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. `0` will be interpreted as "empty" in the future */
|
||||
ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel);
|
||||
return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, zcs->requestedParams, pledgedSrcSize);
|
||||
/* temporary : 0 interpreted as "unknown" during transition period.
|
||||
* Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN.
|
||||
* 0 will be interpreted as "empty" in the future.
|
||||
*/
|
||||
U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;
|
||||
DEBUGLOG(4, "ZSTD_initCStream_srcSize");
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) );
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) );
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) );
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
|
||||
{
|
||||
DEBUGLOG(4, "ZSTD_initCStream");
|
||||
return ZSTD_initCStream_srcSize(zcs, compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN);
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) );
|
||||
FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) );
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*====== Compression ======*/
|
||||
@@ -3898,10 +3906,10 @@ static size_t ZSTD_limitCopy(void* dst, size_t dstCapacity,
|
||||
* internal function for all *compressStream*() variants
|
||||
* non-static, because can be called from zstdmt_compress.c
|
||||
* @return : hint size for next input */
|
||||
size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
||||
ZSTD_outBuffer* output,
|
||||
ZSTD_inBuffer* input,
|
||||
ZSTD_EndDirective const flushMode)
|
||||
static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
||||
ZSTD_outBuffer* output,
|
||||
ZSTD_inBuffer* input,
|
||||
ZSTD_EndDirective const flushMode)
|
||||
{
|
||||
const char* const istart = (const char*)input->src;
|
||||
const char* const iend = istart + input->size;
|
||||
|
||||
@@ -808,13 +808,6 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
|
||||
|
||||
void ZSTD_resetSeqStore(seqStore_t* ssPtr);
|
||||
|
||||
/*! ZSTD_compressStream_generic() :
|
||||
* Private use only. To be called from zstdmt_compress.c in single-thread mode. */
|
||||
size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
||||
ZSTD_outBuffer* output,
|
||||
ZSTD_inBuffer* input,
|
||||
ZSTD_EndDirective const flushMode);
|
||||
|
||||
/*! ZSTD_getCParamsFromCDict() :
|
||||
* as the name implies */
|
||||
ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);
|
||||
|
||||
@@ -1967,7 +1967,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
|
||||
assert(input->pos <= input->size);
|
||||
|
||||
if (mtctx->singleBlockingThread) { /* delegate to single-thread (synchronous) */
|
||||
return ZSTD_compressStream_generic(mtctx->cctxPool->cctx[0], output, input, endOp);
|
||||
return ZSTD_compressStream2(mtctx->cctxPool->cctx[0], output, input, endOp);
|
||||
}
|
||||
|
||||
if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) {
|
||||
|
||||
+74
-5
@@ -347,9 +347,24 @@ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
|
||||
ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
|
||||
|
||||
/*===== Streaming compression functions =====*/
|
||||
/**
|
||||
* Equivalent to:
|
||||
*
|
||||
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
||||
* ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
|
||||
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
|
||||
/**
|
||||
* Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
|
||||
* NOTE: The return value is different. ZSTD_compressStream() returns a hint for
|
||||
* the next read size (if non-zero and not an error). ZSTD_compressStream2()
|
||||
* returns the number of bytes left to flush (if non-zero and not an error).
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
||||
/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
|
||||
ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
|
||||
/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
|
||||
ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
|
||||
@@ -1542,14 +1557,68 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
|
||||
********************************************************************/
|
||||
|
||||
/*===== Advanced Streaming compression functions =====*/
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
|
||||
/**! ZSTD_initCStream_srcSize() :
|
||||
* This function is deprecated, and equivalent to:
|
||||
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
||||
* ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
|
||||
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
|
||||
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
|
||||
*
|
||||
* pledgedSrcSize must be correct. If it is not known at init time, use
|
||||
* ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
|
||||
* "0" also disables frame content size field. It may be enabled in the future.
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);
|
||||
/**! ZSTD_initCStream_usingDict() :
|
||||
* This function is deprecated, and is equivalent to:
|
||||
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
||||
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
|
||||
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
|
||||
*
|
||||
* Creates of an internal CDict (incompatible with static CCtx), except if
|
||||
* dict == NULL or dictSize < 8, in which case no dict is used.
|
||||
* Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
|
||||
* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
|
||||
/**! ZSTD_initCStream_advanced() :
|
||||
* This function is deprecated, and is approximately equivalent to:
|
||||
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
||||
* ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is
|
||||
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
|
||||
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
|
||||
*
|
||||
* pledgedSrcSize must be correct. If srcSize is not known at init time, use
|
||||
* value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize);
|
||||
/**! ZSTD_initCStream_usingCDict() :
|
||||
* This function is deprecated, and equivalent to:
|
||||
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
||||
* ZSTD_CCtx_refCDict(zcs, cdict);
|
||||
*
|
||||
* note : cdict will just be referenced, and must outlive compression session
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
|
||||
/**! ZSTD_initCStream_usingCDict_advanced() :
|
||||
* This function is deprecated, and is approximately equivalent to:
|
||||
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
||||
* ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is
|
||||
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
|
||||
* ZSTD_CCtx_refCDict(zcs, cdict);
|
||||
*
|
||||
* same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
|
||||
* pledgedSrcSize must be correct. If srcSize is not known at init time, use
|
||||
* value ZSTD_CONTENTSIZE_UNKNOWN.
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);
|
||||
|
||||
/*! ZSTD_resetCStream() :
|
||||
* This function is deprecated, and is equivalent to:
|
||||
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
||||
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
|
||||
*
|
||||
* start a new frame, using same parameters from previous frame.
|
||||
* This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
|
||||
* Note that zcs must be init at least once before using ZSTD_resetCStream().
|
||||
|
||||
@@ -97,7 +97,7 @@ static param_value_t mt_advanced_param_values[] = {
|
||||
|
||||
static config_t mt_advanced = {
|
||||
.name = "multithreaded with advanced params",
|
||||
.cli_args = "-T2 --no-compressed-literals",
|
||||
.cli_args = "-T2 --no-compress-literals",
|
||||
.param_values = PARAM_VALUES(mt_advanced_param_values),
|
||||
};
|
||||
|
||||
|
||||
+161
-56
@@ -432,77 +432,158 @@ out:
|
||||
return result;
|
||||
}
|
||||
|
||||
static result_t old_streaming_compress(
|
||||
method_state_t* base,
|
||||
config_t const* config) {
|
||||
buffer_state_t* state = container_of(base, buffer_state_t, base);
|
||||
|
||||
if (buffer_state_bad(state, config))
|
||||
return result_error(result_error_system_error);
|
||||
|
||||
int const level = config_get_level(config);
|
||||
if (level == CONFIG_NO_LEVEL)
|
||||
return result_error(result_error_skip);
|
||||
|
||||
ZSTD_CStream* zcs = ZSTD_createCStream();
|
||||
result_t result;
|
||||
if (zcs == NULL) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
static int init_cstream(
|
||||
buffer_state_t* state,
|
||||
ZSTD_CStream* zcs,
|
||||
config_t const* config,
|
||||
int const advanced,
|
||||
ZSTD_CDict** cdict)
|
||||
{
|
||||
size_t zret;
|
||||
if (config->use_dictionary) {
|
||||
zret = ZSTD_initCStream_usingDict(
|
||||
zcs, state->dictionary.data, state->dictionary.size, level);
|
||||
if (advanced) {
|
||||
ZSTD_parameters const params = config_get_zstd_params(config, 0, 0);
|
||||
ZSTD_CDict* dict = NULL;
|
||||
if (cdict) {
|
||||
*cdict = ZSTD_createCDict_advanced(
|
||||
state->dictionary.data,
|
||||
state->dictionary.size,
|
||||
ZSTD_dlm_byRef,
|
||||
ZSTD_dct_auto,
|
||||
params.cParams,
|
||||
ZSTD_defaultCMem);
|
||||
if (!*cdict) {
|
||||
return 1;
|
||||
}
|
||||
zret = ZSTD_initCStream_usingCDict_advanced(
|
||||
zcs, *cdict, params.fParams, ZSTD_CONTENTSIZE_UNKNOWN);
|
||||
} else {
|
||||
zret = ZSTD_initCStream_advanced(
|
||||
zcs,
|
||||
state->dictionary.data,
|
||||
state->dictionary.size,
|
||||
params,
|
||||
ZSTD_CONTENTSIZE_UNKNOWN);
|
||||
}
|
||||
} else {
|
||||
zret = ZSTD_initCStream(zcs, level);
|
||||
int const level = config_get_level(config);
|
||||
if (cdict) {
|
||||
*cdict = ZSTD_createCDict(
|
||||
state->dictionary.data,
|
||||
state->dictionary.size,
|
||||
level);
|
||||
if (!*cdict) {
|
||||
return 1;
|
||||
}
|
||||
zret = ZSTD_initCStream_usingCDict(zcs, *cdict);
|
||||
} else if (config->use_dictionary) {
|
||||
zret = ZSTD_initCStream_usingDict(
|
||||
zcs, state->dictionary.data, state->dictionary.size, level);
|
||||
} else {
|
||||
zret = ZSTD_initCStream(zcs, level);
|
||||
}
|
||||
}
|
||||
if (ZSTD_isError(zret)) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static result_t old_streaming_compress_internal(
|
||||
method_state_t* base,
|
||||
config_t const* config,
|
||||
int const advanced,
|
||||
int const cdict) {
|
||||
buffer_state_t* state = container_of(base, buffer_state_t, base);
|
||||
|
||||
if (buffer_state_bad(state, config))
|
||||
return result_error(result_error_system_error);
|
||||
|
||||
|
||||
ZSTD_CStream* zcs = ZSTD_createCStream();
|
||||
ZSTD_CDict* cd = NULL;
|
||||
result_t result;
|
||||
if (zcs == NULL) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
if (init_cstream(state, zcs, config, advanced, cdict ? &cd : NULL)) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
|
||||
result_data_t data = {.total_size = 0};
|
||||
for (size_t i = 0; i < state->inputs.size; ++i) {
|
||||
data_buffer_t input = state->inputs.buffers[i];
|
||||
size_t zret = ZSTD_resetCStream(
|
||||
zcs,
|
||||
config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN : input.size);
|
||||
if (ZSTD_isError(zret)) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
|
||||
result_data_t data = {.total_size = 0};
|
||||
for (size_t i = 0; i < state->inputs.size; ++i) {
|
||||
data_buffer_t input = state->inputs.buffers[i];
|
||||
zret = ZSTD_resetCStream(
|
||||
zcs,
|
||||
config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN
|
||||
: input.size);
|
||||
while (input.size > 0) {
|
||||
ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)};
|
||||
input.data += in.size;
|
||||
input.size -= in.size;
|
||||
ZSTD_EndDirective const op =
|
||||
input.size > 0 ? ZSTD_e_continue : ZSTD_e_end;
|
||||
zret = 0;
|
||||
while (in.pos < in.size || (op == ZSTD_e_end && zret != 0)) {
|
||||
ZSTD_outBuffer out = {state->compressed.data,
|
||||
MIN(state->compressed.capacity, 1024)};
|
||||
if (op == ZSTD_e_continue || in.pos < in.size)
|
||||
zret = ZSTD_compressStream(zcs, &out, &in);
|
||||
else
|
||||
zret = ZSTD_endStream(zcs, &out);
|
||||
if (ZSTD_isError(zret)) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
|
||||
while (input.size > 0) {
|
||||
ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)};
|
||||
input.data += in.size;
|
||||
input.size -= in.size;
|
||||
ZSTD_EndDirective const op =
|
||||
input.size > 0 ? ZSTD_e_continue : ZSTD_e_end;
|
||||
zret = 0;
|
||||
while (in.pos < in.size || (op == ZSTD_e_end && zret != 0)) {
|
||||
ZSTD_outBuffer out = {state->compressed.data,
|
||||
MIN(state->compressed.capacity, 1024)};
|
||||
if (op == ZSTD_e_continue || in.pos < in.size)
|
||||
zret = ZSTD_compressStream(zcs, &out, &in);
|
||||
else
|
||||
zret = ZSTD_endStream(zcs, &out);
|
||||
if (ZSTD_isError(zret)) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
data.total_size += out.pos;
|
||||
}
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
data.total_size += out.pos;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result = result_data(data);
|
||||
result = result_data(data);
|
||||
out:
|
||||
ZSTD_freeCStream(zcs);
|
||||
ZSTD_freeCDict(cd);
|
||||
return result;
|
||||
}
|
||||
|
||||
static result_t old_streaming_compress(
|
||||
method_state_t* base,
|
||||
config_t const* config)
|
||||
{
|
||||
return old_streaming_compress_internal(
|
||||
base, config, /* advanced */ 0, /* cdict */ 0);
|
||||
}
|
||||
|
||||
static result_t old_streaming_compress_advanced(
|
||||
method_state_t* base,
|
||||
config_t const* config)
|
||||
{
|
||||
return old_streaming_compress_internal(
|
||||
base, config, /* advanced */ 1, /* cdict */ 0);
|
||||
}
|
||||
|
||||
static result_t old_streaming_compress_cdict(
|
||||
method_state_t* base,
|
||||
config_t const* config)
|
||||
{
|
||||
return old_streaming_compress_internal(
|
||||
base, config, /* advanced */ 0, /* cdict */ 1);
|
||||
}
|
||||
|
||||
static result_t old_streaming_compress_cdict_advanced(
|
||||
method_state_t* base,
|
||||
config_t const* config)
|
||||
{
|
||||
return old_streaming_compress_internal(
|
||||
base, config, /* advanced */ 1, /* cdict */ 1);
|
||||
}
|
||||
|
||||
method_t const simple = {
|
||||
.name = "compress simple",
|
||||
.create = buffer_state_create,
|
||||
@@ -545,6 +626,27 @@ method_t const old_streaming = {
|
||||
.destroy = buffer_state_destroy,
|
||||
};
|
||||
|
||||
method_t const old_streaming_advanced = {
|
||||
.name = "old streaming advanced",
|
||||
.create = buffer_state_create,
|
||||
.compress = old_streaming_compress,
|
||||
.destroy = buffer_state_destroy,
|
||||
};
|
||||
|
||||
method_t const old_streaming_cdict = {
|
||||
.name = "old streaming cdcit",
|
||||
.create = buffer_state_create,
|
||||
.compress = old_streaming_compress,
|
||||
.destroy = buffer_state_destroy,
|
||||
};
|
||||
|
||||
method_t const old_streaming_advanced_cdict = {
|
||||
.name = "old streaming advanced cdict",
|
||||
.create = buffer_state_create,
|
||||
.compress = old_streaming_compress,
|
||||
.destroy = buffer_state_destroy,
|
||||
};
|
||||
|
||||
method_t const cli = {
|
||||
.name = "zstdcli",
|
||||
.create = method_state_create,
|
||||
@@ -560,6 +662,9 @@ static method_t const* g_methods[] = {
|
||||
&advanced_one_pass_small_out,
|
||||
&advanced_streaming,
|
||||
&old_streaming,
|
||||
&old_streaming_advanced,
|
||||
&old_streaming_cdict,
|
||||
&old_streaming_advanced_cdict,
|
||||
NULL,
|
||||
};
|
||||
|
||||
|
||||
@@ -104,7 +104,7 @@ silesia, explicit params, zstdcli,
|
||||
silesia, uncompressed literals, zstdcli, 5155472
|
||||
silesia, uncompressed literals optimal, zstdcli, 4325475
|
||||
silesia, huffman literals, zstdcli, 5341405
|
||||
silesia, multithreaded with advanced params, zstdcli, compression error
|
||||
silesia, multithreaded with advanced params, zstdcli, 5155472
|
||||
silesia.tar, level -5, zstdcli, 7161160
|
||||
silesia.tar, level -3, zstdcli, 6789865
|
||||
silesia.tar, level -1, zstdcli, 6196433
|
||||
@@ -130,7 +130,7 @@ silesia.tar, explicit params, zstdcli,
|
||||
silesia.tar, uncompressed literals, zstdcli, 5158134
|
||||
silesia.tar, uncompressed literals optimal, zstdcli, 4321098
|
||||
silesia.tar, huffman literals, zstdcli, 5358479
|
||||
silesia.tar, multithreaded with advanced params, zstdcli, compression error
|
||||
silesia.tar, multithreaded with advanced params, zstdcli, 5158134
|
||||
github, level -5, zstdcli, 234744
|
||||
github, level -5 with dict, zstdcli, 48718
|
||||
github, level -3, zstdcli, 222611
|
||||
@@ -169,7 +169,7 @@ github, explicit params, zstdcli,
|
||||
github, uncompressed literals, zstdcli, 169004
|
||||
github, uncompressed literals optimal, zstdcli, 158824
|
||||
github, huffman literals, zstdcli, 145457
|
||||
github, multithreaded with advanced params, zstdcli, compression error
|
||||
github, multithreaded with advanced params, zstdcli, 169004
|
||||
silesia, level -5, advanced one pass, 7152294
|
||||
silesia, level -3, advanced one pass, 6789969
|
||||
silesia, level -1, advanced one pass, 6191548
|
||||
@@ -461,9 +461,17 @@ silesia, level 13, old stre
|
||||
silesia, level 16, old streaming, 4377391
|
||||
silesia, level 19, old streaming, 4293262
|
||||
silesia, no source size, old streaming, 4862341
|
||||
silesia, long distance mode, old streaming, 12000408
|
||||
silesia, multithreaded, old streaming, 12000408
|
||||
silesia, multithreaded long distance mode, old streaming, 12000408
|
||||
silesia, small window log, old streaming, 12000408
|
||||
silesia, small hash log, old streaming, 12000408
|
||||
silesia, small chain log, old streaming, 12000408
|
||||
silesia, explicit params, old streaming, 12000408
|
||||
silesia, uncompressed literals, old streaming, 4862377
|
||||
silesia, uncompressed literals optimal, old streaming, 4293262
|
||||
silesia, huffman literals, old streaming, 6191549
|
||||
silesia, multithreaded with advanced params, old streaming, 12000408
|
||||
silesia.tar, level -5, old streaming, 7160440
|
||||
silesia.tar, level -3, old streaming, 6789026
|
||||
silesia.tar, level -1, old streaming, 6195465
|
||||
@@ -479,9 +487,17 @@ silesia.tar, level 13, old stre
|
||||
silesia.tar, level 16, old streaming, 4381277
|
||||
silesia.tar, level 19, old streaming, 4281514
|
||||
silesia.tar, no source size, old streaming, 4875006
|
||||
silesia.tar, long distance mode, old streaming, 12022046
|
||||
silesia.tar, multithreaded, old streaming, 12022046
|
||||
silesia.tar, multithreaded long distance mode, old streaming, 12022046
|
||||
silesia.tar, small window log, old streaming, 12022046
|
||||
silesia.tar, small hash log, old streaming, 12022046
|
||||
silesia.tar, small chain log, old streaming, 12022046
|
||||
silesia.tar, explicit params, old streaming, 12022046
|
||||
silesia.tar, uncompressed literals, old streaming, 4875010
|
||||
silesia.tar, uncompressed literals optimal, old streaming, 4281514
|
||||
silesia.tar, huffman literals, old streaming, 6195465
|
||||
silesia.tar, multithreaded with advanced params, old streaming, 12022046
|
||||
github, level -5, old streaming, 232744
|
||||
github, level -5 with dict, old streaming, 46718
|
||||
github, level -3, old streaming, 220611
|
||||
@@ -511,6 +527,290 @@ github, level 16 with dict, old stre
|
||||
github, level 19, old streaming, 133717
|
||||
github, level 19 with dict, old streaming, 37576
|
||||
github, no source size, old streaming, 141003
|
||||
github, long distance mode, old streaming, 412933
|
||||
github, multithreaded, old streaming, 412933
|
||||
github, multithreaded long distance mode, old streaming, 412933
|
||||
github, small window log, old streaming, 412933
|
||||
github, small hash log, old streaming, 412933
|
||||
github, small chain log, old streaming, 412933
|
||||
github, explicit params, old streaming, 412933
|
||||
github, uncompressed literals, old streaming, 136397
|
||||
github, uncompressed literals optimal, old streaming, 133717
|
||||
github, huffman literals, old streaming, 176575
|
||||
github, multithreaded with advanced params, old streaming, 412933
|
||||
silesia, level -5, old streaming advanced, 7152294
|
||||
silesia, level -3, old streaming advanced, 6789973
|
||||
silesia, level -1, old streaming advanced, 6191549
|
||||
silesia, level 0, old streaming advanced, 4862377
|
||||
silesia, level 1, old streaming advanced, 5318036
|
||||
silesia, level 3, old streaming advanced, 4862377
|
||||
silesia, level 4, old streaming advanced, 4800629
|
||||
silesia, level 5, old streaming advanced, 4710178
|
||||
silesia, level 6, old streaming advanced, 4659996
|
||||
silesia, level 7, old streaming advanced, 4596234
|
||||
silesia, level 9, old streaming advanced, 4543862
|
||||
silesia, level 13, old streaming advanced, 4482073
|
||||
silesia, level 16, old streaming advanced, 4377391
|
||||
silesia, level 19, old streaming advanced, 4293262
|
||||
silesia, no source size, old streaming advanced, 4862341
|
||||
silesia, long distance mode, old streaming advanced, 12000408
|
||||
silesia, multithreaded, old streaming advanced, 12000408
|
||||
silesia, multithreaded long distance mode, old streaming advanced, 12000408
|
||||
silesia, small window log, old streaming advanced, 12000408
|
||||
silesia, small hash log, old streaming advanced, 12000408
|
||||
silesia, small chain log, old streaming advanced, 12000408
|
||||
silesia, explicit params, old streaming advanced, 12000408
|
||||
silesia, uncompressed literals, old streaming advanced, 4862377
|
||||
silesia, uncompressed literals optimal, old streaming advanced, 4293262
|
||||
silesia, huffman literals, old streaming advanced, 6191549
|
||||
silesia, multithreaded with advanced params, old streaming advanced, 12000408
|
||||
silesia.tar, level -5, old streaming advanced, 7160440
|
||||
silesia.tar, level -3, old streaming advanced, 6789026
|
||||
silesia.tar, level -1, old streaming advanced, 6195465
|
||||
silesia.tar, level 0, old streaming advanced, 4875010
|
||||
silesia.tar, level 1, old streaming advanced, 5339701
|
||||
silesia.tar, level 3, old streaming advanced, 4875010
|
||||
silesia.tar, level 4, old streaming advanced, 4813507
|
||||
silesia.tar, level 5, old streaming advanced, 4722240
|
||||
silesia.tar, level 6, old streaming advanced, 4672203
|
||||
silesia.tar, level 7, old streaming advanced, 4606658
|
||||
silesia.tar, level 9, old streaming advanced, 4554105
|
||||
silesia.tar, level 13, old streaming advanced, 4491703
|
||||
silesia.tar, level 16, old streaming advanced, 4381277
|
||||
silesia.tar, level 19, old streaming advanced, 4281514
|
||||
silesia.tar, no source size, old streaming advanced, 4875006
|
||||
silesia.tar, long distance mode, old streaming advanced, 12022046
|
||||
silesia.tar, multithreaded, old streaming advanced, 12022046
|
||||
silesia.tar, multithreaded long distance mode, old streaming advanced, 12022046
|
||||
silesia.tar, small window log, old streaming advanced, 12022046
|
||||
silesia.tar, small hash log, old streaming advanced, 12022046
|
||||
silesia.tar, small chain log, old streaming advanced, 12022046
|
||||
silesia.tar, explicit params, old streaming advanced, 12022046
|
||||
silesia.tar, uncompressed literals, old streaming advanced, 4875010
|
||||
silesia.tar, uncompressed literals optimal, old streaming advanced, 4281514
|
||||
silesia.tar, huffman literals, old streaming advanced, 6195465
|
||||
silesia.tar, multithreaded with advanced params, old streaming advanced, 12022046
|
||||
github, level -5, old streaming advanced, 232744
|
||||
github, level -5 with dict, old streaming advanced, 46718
|
||||
github, level -3, old streaming advanced, 220611
|
||||
github, level -3 with dict, old streaming advanced, 45395
|
||||
github, level -1, old streaming advanced, 176575
|
||||
github, level -1 with dict, old streaming advanced, 43170
|
||||
github, level 0, old streaming advanced, 136397
|
||||
github, level 0 with dict, old streaming advanced, 41170
|
||||
github, level 1, old streaming advanced, 143457
|
||||
github, level 1 with dict, old streaming advanced, 41682
|
||||
github, level 3, old streaming advanced, 136397
|
||||
github, level 3 with dict, old streaming advanced, 41170
|
||||
github, level 4, old streaming advanced, 136144
|
||||
github, level 4 with dict, old streaming advanced, 41306
|
||||
github, level 5, old streaming advanced, 135106
|
||||
github, level 5 with dict, old streaming advanced, 38938
|
||||
github, level 6, old streaming advanced, 135108
|
||||
github, level 6 with dict, old streaming advanced, 38632
|
||||
github, level 7, old streaming advanced, 135108
|
||||
github, level 7 with dict, old streaming advanced, 38766
|
||||
github, level 9, old streaming advanced, 135108
|
||||
github, level 9 with dict, old streaming advanced, 39326
|
||||
github, level 13, old streaming advanced, 133717
|
||||
github, level 13 with dict, old streaming advanced, 39716
|
||||
github, level 16, old streaming advanced, 133717
|
||||
github, level 16 with dict, old streaming advanced, 37577
|
||||
github, level 19, old streaming advanced, 133717
|
||||
github, level 19 with dict, old streaming advanced, 37576
|
||||
github, no source size, old streaming advanced, 141003
|
||||
github, long distance mode, old streaming advanced, 412933
|
||||
github, multithreaded, old streaming advanced, 412933
|
||||
github, multithreaded long distance mode, old streaming advanced, 412933
|
||||
github, small window log, old streaming advanced, 412933
|
||||
github, small hash log, old streaming advanced, 412933
|
||||
github, small chain log, old streaming advanced, 412933
|
||||
github, explicit params, old streaming advanced, 412933
|
||||
github, uncompressed literals, old streaming advanced, 136397
|
||||
github, uncompressed literals optimal, old streaming advanced, 133717
|
||||
github, huffman literals, old streaming advanced, 176575
|
||||
github, multithreaded with advanced params, old streaming advanced, 412933
|
||||
silesia, level -5, old streaming cdcit, 7152294
|
||||
silesia, level -3, old streaming cdcit, 6789973
|
||||
silesia, level -1, old streaming cdcit, 6191549
|
||||
silesia, level 0, old streaming cdcit, 4862377
|
||||
silesia, level 1, old streaming cdcit, 5318036
|
||||
silesia, level 3, old streaming cdcit, 4862377
|
||||
silesia, level 4, old streaming cdcit, 4800629
|
||||
silesia, level 5, old streaming cdcit, 4710178
|
||||
silesia, level 6, old streaming cdcit, 4659996
|
||||
silesia, level 7, old streaming cdcit, 4596234
|
||||
silesia, level 9, old streaming cdcit, 4543862
|
||||
silesia, level 13, old streaming cdcit, 4482073
|
||||
silesia, level 16, old streaming cdcit, 4377391
|
||||
silesia, level 19, old streaming cdcit, 4293262
|
||||
silesia, no source size, old streaming cdcit, 4862341
|
||||
silesia, long distance mode, old streaming cdcit, 12000408
|
||||
silesia, multithreaded, old streaming cdcit, 12000408
|
||||
silesia, multithreaded long distance mode, old streaming cdcit, 12000408
|
||||
silesia, small window log, old streaming cdcit, 12000408
|
||||
silesia, small hash log, old streaming cdcit, 12000408
|
||||
silesia, small chain log, old streaming cdcit, 12000408
|
||||
silesia, explicit params, old streaming cdcit, 12000408
|
||||
silesia, uncompressed literals, old streaming cdcit, 4862377
|
||||
silesia, uncompressed literals optimal, old streaming cdcit, 4293262
|
||||
silesia, huffman literals, old streaming cdcit, 6191549
|
||||
silesia, multithreaded with advanced params, old streaming cdcit, 12000408
|
||||
silesia.tar, level -5, old streaming cdcit, 7160440
|
||||
silesia.tar, level -3, old streaming cdcit, 6789026
|
||||
silesia.tar, level -1, old streaming cdcit, 6195465
|
||||
silesia.tar, level 0, old streaming cdcit, 4875010
|
||||
silesia.tar, level 1, old streaming cdcit, 5339701
|
||||
silesia.tar, level 3, old streaming cdcit, 4875010
|
||||
silesia.tar, level 4, old streaming cdcit, 4813507
|
||||
silesia.tar, level 5, old streaming cdcit, 4722240
|
||||
silesia.tar, level 6, old streaming cdcit, 4672203
|
||||
silesia.tar, level 7, old streaming cdcit, 4606658
|
||||
silesia.tar, level 9, old streaming cdcit, 4554105
|
||||
silesia.tar, level 13, old streaming cdcit, 4491703
|
||||
silesia.tar, level 16, old streaming cdcit, 4381277
|
||||
silesia.tar, level 19, old streaming cdcit, 4281514
|
||||
silesia.tar, no source size, old streaming cdcit, 4875006
|
||||
silesia.tar, long distance mode, old streaming cdcit, 12022046
|
||||
silesia.tar, multithreaded, old streaming cdcit, 12022046
|
||||
silesia.tar, multithreaded long distance mode, old streaming cdcit, 12022046
|
||||
silesia.tar, small window log, old streaming cdcit, 12022046
|
||||
silesia.tar, small hash log, old streaming cdcit, 12022046
|
||||
silesia.tar, small chain log, old streaming cdcit, 12022046
|
||||
silesia.tar, explicit params, old streaming cdcit, 12022046
|
||||
silesia.tar, uncompressed literals, old streaming cdcit, 4875010
|
||||
silesia.tar, uncompressed literals optimal, old streaming cdcit, 4281514
|
||||
silesia.tar, huffman literals, old streaming cdcit, 6195465
|
||||
silesia.tar, multithreaded with advanced params, old streaming cdcit, 12022046
|
||||
github, level -5, old streaming cdcit, 232744
|
||||
github, level -5 with dict, old streaming cdcit, 46718
|
||||
github, level -3, old streaming cdcit, 220611
|
||||
github, level -3 with dict, old streaming cdcit, 45395
|
||||
github, level -1, old streaming cdcit, 176575
|
||||
github, level -1 with dict, old streaming cdcit, 43170
|
||||
github, level 0, old streaming cdcit, 136397
|
||||
github, level 0 with dict, old streaming cdcit, 41170
|
||||
github, level 1, old streaming cdcit, 143457
|
||||
github, level 1 with dict, old streaming cdcit, 41682
|
||||
github, level 3, old streaming cdcit, 136397
|
||||
github, level 3 with dict, old streaming cdcit, 41170
|
||||
github, level 4, old streaming cdcit, 136144
|
||||
github, level 4 with dict, old streaming cdcit, 41306
|
||||
github, level 5, old streaming cdcit, 135106
|
||||
github, level 5 with dict, old streaming cdcit, 38938
|
||||
github, level 6, old streaming cdcit, 135108
|
||||
github, level 6 with dict, old streaming cdcit, 38632
|
||||
github, level 7, old streaming cdcit, 135108
|
||||
github, level 7 with dict, old streaming cdcit, 38766
|
||||
github, level 9, old streaming cdcit, 135108
|
||||
github, level 9 with dict, old streaming cdcit, 39326
|
||||
github, level 13, old streaming cdcit, 133717
|
||||
github, level 13 with dict, old streaming cdcit, 39716
|
||||
github, level 16, old streaming cdcit, 133717
|
||||
github, level 16 with dict, old streaming cdcit, 37577
|
||||
github, level 19, old streaming cdcit, 133717
|
||||
github, level 19 with dict, old streaming cdcit, 37576
|
||||
github, no source size, old streaming cdcit, 141003
|
||||
github, long distance mode, old streaming cdcit, 412933
|
||||
github, multithreaded, old streaming cdcit, 412933
|
||||
github, multithreaded long distance mode, old streaming cdcit, 412933
|
||||
github, small window log, old streaming cdcit, 412933
|
||||
github, small hash log, old streaming cdcit, 412933
|
||||
github, small chain log, old streaming cdcit, 412933
|
||||
github, explicit params, old streaming cdcit, 412933
|
||||
github, uncompressed literals, old streaming cdcit, 136397
|
||||
github, uncompressed literals optimal, old streaming cdcit, 133717
|
||||
github, huffman literals, old streaming cdcit, 176575
|
||||
github, multithreaded with advanced params, old streaming cdcit, 412933
|
||||
silesia, level -5, old streaming advanced cdict, 7152294
|
||||
silesia, level -3, old streaming advanced cdict, 6789973
|
||||
silesia, level -1, old streaming advanced cdict, 6191549
|
||||
silesia, level 0, old streaming advanced cdict, 4862377
|
||||
silesia, level 1, old streaming advanced cdict, 5318036
|
||||
silesia, level 3, old streaming advanced cdict, 4862377
|
||||
silesia, level 4, old streaming advanced cdict, 4800629
|
||||
silesia, level 5, old streaming advanced cdict, 4710178
|
||||
silesia, level 6, old streaming advanced cdict, 4659996
|
||||
silesia, level 7, old streaming advanced cdict, 4596234
|
||||
silesia, level 9, old streaming advanced cdict, 4543862
|
||||
silesia, level 13, old streaming advanced cdict, 4482073
|
||||
silesia, level 16, old streaming advanced cdict, 4377391
|
||||
silesia, level 19, old streaming advanced cdict, 4293262
|
||||
silesia, no source size, old streaming advanced cdict, 4862341
|
||||
silesia, long distance mode, old streaming advanced cdict, 12000408
|
||||
silesia, multithreaded, old streaming advanced cdict, 12000408
|
||||
silesia, multithreaded long distance mode, old streaming advanced cdict, 12000408
|
||||
silesia, small window log, old streaming advanced cdict, 12000408
|
||||
silesia, small hash log, old streaming advanced cdict, 12000408
|
||||
silesia, small chain log, old streaming advanced cdict, 12000408
|
||||
silesia, explicit params, old streaming advanced cdict, 12000408
|
||||
silesia, uncompressed literals, old streaming advanced cdict, 4862377
|
||||
silesia, uncompressed literals optimal, old streaming advanced cdict, 4293262
|
||||
silesia, huffman literals, old streaming advanced cdict, 6191549
|
||||
silesia, multithreaded with advanced params, old streaming advanced cdict, 12000408
|
||||
silesia.tar, level -5, old streaming advanced cdict, 7160440
|
||||
silesia.tar, level -3, old streaming advanced cdict, 6789026
|
||||
silesia.tar, level -1, old streaming advanced cdict, 6195465
|
||||
silesia.tar, level 0, old streaming advanced cdict, 4875010
|
||||
silesia.tar, level 1, old streaming advanced cdict, 5339701
|
||||
silesia.tar, level 3, old streaming advanced cdict, 4875010
|
||||
silesia.tar, level 4, old streaming advanced cdict, 4813507
|
||||
silesia.tar, level 5, old streaming advanced cdict, 4722240
|
||||
silesia.tar, level 6, old streaming advanced cdict, 4672203
|
||||
silesia.tar, level 7, old streaming advanced cdict, 4606658
|
||||
silesia.tar, level 9, old streaming advanced cdict, 4554105
|
||||
silesia.tar, level 13, old streaming advanced cdict, 4491703
|
||||
silesia.tar, level 16, old streaming advanced cdict, 4381277
|
||||
silesia.tar, level 19, old streaming advanced cdict, 4281514
|
||||
silesia.tar, no source size, old streaming advanced cdict, 4875006
|
||||
silesia.tar, long distance mode, old streaming advanced cdict, 12022046
|
||||
silesia.tar, multithreaded, old streaming advanced cdict, 12022046
|
||||
silesia.tar, multithreaded long distance mode, old streaming advanced cdict, 12022046
|
||||
silesia.tar, small window log, old streaming advanced cdict, 12022046
|
||||
silesia.tar, small hash log, old streaming advanced cdict, 12022046
|
||||
silesia.tar, small chain log, old streaming advanced cdict, 12022046
|
||||
silesia.tar, explicit params, old streaming advanced cdict, 12022046
|
||||
silesia.tar, uncompressed literals, old streaming advanced cdict, 4875010
|
||||
silesia.tar, uncompressed literals optimal, old streaming advanced cdict, 4281514
|
||||
silesia.tar, huffman literals, old streaming advanced cdict, 6195465
|
||||
silesia.tar, multithreaded with advanced params, old streaming advanced cdict, 12022046
|
||||
github, level -5, old streaming advanced cdict, 232744
|
||||
github, level -5 with dict, old streaming advanced cdict, 46718
|
||||
github, level -3, old streaming advanced cdict, 220611
|
||||
github, level -3 with dict, old streaming advanced cdict, 45395
|
||||
github, level -1, old streaming advanced cdict, 176575
|
||||
github, level -1 with dict, old streaming advanced cdict, 43170
|
||||
github, level 0, old streaming advanced cdict, 136397
|
||||
github, level 0 with dict, old streaming advanced cdict, 41170
|
||||
github, level 1, old streaming advanced cdict, 143457
|
||||
github, level 1 with dict, old streaming advanced cdict, 41682
|
||||
github, level 3, old streaming advanced cdict, 136397
|
||||
github, level 3 with dict, old streaming advanced cdict, 41170
|
||||
github, level 4, old streaming advanced cdict, 136144
|
||||
github, level 4 with dict, old streaming advanced cdict, 41306
|
||||
github, level 5, old streaming advanced cdict, 135106
|
||||
github, level 5 with dict, old streaming advanced cdict, 38938
|
||||
github, level 6, old streaming advanced cdict, 135108
|
||||
github, level 6 with dict, old streaming advanced cdict, 38632
|
||||
github, level 7, old streaming advanced cdict, 135108
|
||||
github, level 7 with dict, old streaming advanced cdict, 38766
|
||||
github, level 9, old streaming advanced cdict, 135108
|
||||
github, level 9 with dict, old streaming advanced cdict, 39326
|
||||
github, level 13, old streaming advanced cdict, 133717
|
||||
github, level 13 with dict, old streaming advanced cdict, 39716
|
||||
github, level 16, old streaming advanced cdict, 133717
|
||||
github, level 16 with dict, old streaming advanced cdict, 37577
|
||||
github, level 19, old streaming advanced cdict, 133717
|
||||
github, level 19 with dict, old streaming advanced cdict, 37576
|
||||
github, no source size, old streaming advanced cdict, 141003
|
||||
github, long distance mode, old streaming advanced cdict, 412933
|
||||
github, multithreaded, old streaming advanced cdict, 412933
|
||||
github, multithreaded long distance mode, old streaming advanced cdict, 412933
|
||||
github, small window log, old streaming advanced cdict, 412933
|
||||
github, small hash log, old streaming advanced cdict, 412933
|
||||
github, small chain log, old streaming advanced cdict, 412933
|
||||
github, explicit params, old streaming advanced cdict, 412933
|
||||
github, uncompressed literals, old streaming advanced cdict, 136397
|
||||
github, uncompressed literals optimal, old streaming advanced cdict, 133717
|
||||
github, huffman literals, old streaming advanced cdict, 176575
|
||||
github, multithreaded with advanced params, old streaming advanced cdict, 412933
|
||||
|
||||
|
+9
-6
@@ -495,7 +495,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
|
||||
/* _srcSize compression test */
|
||||
DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
ZSTD_initCStream_srcSize(zc, 1, CNBufferSize);
|
||||
CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) );
|
||||
outBuff.dst = (char*)(compressedBuffer);
|
||||
outBuff.size = compressedBufferSize;
|
||||
outBuff.pos = 0;
|
||||
@@ -503,11 +503,14 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
inBuff.size = CNBufferSize;
|
||||
inBuff.pos = 0;
|
||||
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
|
||||
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
||||
{ size_t const r = ZSTD_endStream(zc, &outBuff);
|
||||
if (r != 0) goto _output_error; } /* error, or some data not flushed */
|
||||
{ unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
|
||||
if ((size_t)origSize != CNBufferSize) goto _output_error; } /* exact original size must be present */
|
||||
CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
|
||||
{ size_t const r = ZSTD_endStream(zc, &outBuff);
|
||||
CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
|
||||
}
|
||||
{ unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
|
||||
CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!");
|
||||
CHECK((size_t)origSize != CNBufferSize, "Exact original size must be present (got %llu)", origSize);
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||
|
||||
/* wrong _srcSize compression test */
|
||||
|
||||
Reference in New Issue
Block a user