ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4329 from Cyan4973/cmd_split

Browse Source 
New commands --split and --jobsize
This commit is contained in:
Yann Collet
2025-03-05 07:46:04 -08:00
committed by GitHub
parent 190a620974 5ae1cb9fa1
commit 99cf130cfc
12 changed files with 130 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/benchzstd.c
+51 -62
View File
@@ -208,7 +208,7 @@ BMK_advancedParams_t BMK_initAdvancedParams(void)
BMK_advancedParams_t const res = { BMK_advancedParams_t const res = {
BMK_both, /* mode */ BMK_both, /* mode */
BMK_TIMETEST_DEFAULT_S, /* nbSeconds */ BMK_TIMETEST_DEFAULT_S, /* nbSeconds */
0, /* blockSize */ 0, /* chunkSizeMax */
0, /* targetCBlockSize */ 0, /* targetCBlockSize */
0, /* nbWorkers */ 0, /* nbWorkers */
0, /* realTime */ 0, /* realTime */
@@ -227,16 +227,6 @@ BMK_advancedParams_t BMK_initAdvancedParams(void)
/* ******************************************************** /* ********************************************************
* Bench functions * Bench functions
**********************************************************/ **********************************************************/
typedef struct {
const void* srcPtr;
size_t srcSize;
void* cPtr;
size_t cRoom;
size_t cSize;
void* resPtr;
size_t resSize;
} blockParam_t;
#undef MIN #undef MIN
#undef MAX #undef MAX
#define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MIN(a, b) ((a) < (b) ? (a) : (b))
@@ -435,16 +425,16 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
const char* displayName, const char* displayName,
const BMK_advancedParams_t* adv) const BMK_advancedParams_t* adv)
{ {
size_t const blockSize = size_t const chunkSizeMax =
((adv->blockSize >= 32 && (adv->mode != BMK_decodeOnly)) ((adv->chunkSizeMax >= 32 && (adv->mode != BMK_decodeOnly))
? adv->blockSize ? adv->chunkSizeMax
: srcSize) : srcSize)
+ (!srcSize); /* avoid div by 0 */ + (!srcSize); /* avoid div by 0 */
BMK_benchResult_t benchResult; BMK_benchResult_t benchResult;
size_t const loadedCompressedSize = srcSize; size_t const loadedCompressedSize = srcSize;
size_t cSize = 0; size_t cSize = 0;
double ratio = 0.; double ratio = 0.;
U32 nbBlocks; U32 nbChunks = 0;
assert(cctx != NULL); assert(cctx != NULL);
assert(dctx != NULL); assert(dctx != NULL);
@@ -500,41 +490,42 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
} }
} }
/* Init data blocks */ /* Init data chunks */
{ {
const char* srcPtr = (const char*)srcBuffer; const char* srcPtr = (const char*)srcBuffer;
char* cPtr = (char*)compressedBuffer; char* cPtr = (char*)compressedBuffer;
char* resPtr = (char*)(*resultBufferPtr); char* resPtr = (char*)(*resultBufferPtr);
U32 fileNb; U32 fileNb, chunkID;
for (nbBlocks = 0, fileNb = 0; fileNb < nbFiles; fileNb++) { for (chunkID = 0, fileNb = 0; fileNb < nbFiles; fileNb++) {
size_t remaining = fileSizes[fileNb]; size_t remaining = fileSizes[fileNb];
U32 const nbBlocksforThisFile = (adv->mode == BMK_decodeOnly) U32 const nbChunksforThisFile = (adv->mode == BMK_decodeOnly)
? 1 ? 1
: (U32)((remaining + (blockSize - 1)) / blockSize); : (U32)((remaining + (chunkSizeMax - 1)) / chunkSizeMax);
U32 const blockEnd = nbBlocks + nbBlocksforThisFile; U32 const chunkIdEnd = chunkID + nbChunksforThisFile;
for (; nbBlocks < blockEnd; nbBlocks++) { for (; chunkID < chunkIdEnd; chunkID++) {
size_t const thisBlockSize = MIN(remaining, blockSize); size_t const chunkSize = MIN(remaining, chunkSizeMax);
srcPtrs[nbBlocks] = srcPtr; srcPtrs[chunkID] = srcPtr;
srcSizes[nbBlocks] = thisBlockSize; srcSizes[chunkID] = chunkSize;
cPtrs[nbBlocks] = cPtr; cPtrs[chunkID] = cPtr;
cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly) cCapacities[chunkID] = (adv->mode == BMK_decodeOnly)
? thisBlockSize ? chunkSize
: ZSTD_compressBound(thisBlockSize); : ZSTD_compressBound(chunkSize);
resPtrs[nbBlocks] = resPtr; resPtrs[chunkID] = resPtr;
resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) resSizes[chunkID] = (adv->mode == BMK_decodeOnly)
? (size_t)ZSTD_findDecompressedSize( ? (size_t)ZSTD_findDecompressedSize(
srcPtr, thisBlockSize) srcPtr, chunkSize)
: thisBlockSize; : chunkSize;
srcPtr += thisBlockSize; srcPtr += chunkSize;
cPtr += cCapacities[nbBlocks]; cPtr += cCapacities[chunkID];
resPtr += thisBlockSize; resPtr += chunkSize;
remaining -= thisBlockSize; remaining -= chunkSize;
if (adv->mode == BMK_decodeOnly) { if (adv->mode == BMK_decodeOnly) {
cSizes[nbBlocks] = thisBlockSize; cSizes[chunkID] = chunkSize;
benchResult.cSize = thisBlockSize; benchResult.cSize = chunkSize;
} }
} }
} }
nbChunks = chunkID;
} }
/* warming up `compressedBuffer` */ /* warming up `compressedBuffer` */
@@ -569,7 +560,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
cbp.initFn = local_initCCtx; /* BMK_initCCtx */ cbp.initFn = local_initCCtx; /* BMK_initCCtx */
cbp.initPayload = &cctxprep; cbp.initPayload = &cctxprep;
cbp.errorFn = ZSTD_isError; cbp.errorFn = ZSTD_isError;
cbp.blockCount = nbBlocks; cbp.blockCount = nbChunks;
cbp.srcBuffers = srcPtrs; cbp.srcBuffers = srcPtrs;
cbp.srcSizes = srcSizes; cbp.srcSizes = srcSizes;
cbp.dstBuffers = cPtrs; cbp.dstBuffers = cPtrs;
@@ -588,7 +579,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
dbp.initFn = local_initDCtx; dbp.initFn = local_initDCtx;
dbp.initPayload = &dctxprep; dbp.initPayload = &dctxprep;
dbp.errorFn = ZSTD_isError; dbp.errorFn = ZSTD_isError;
dbp.blockCount = nbBlocks; dbp.blockCount = nbChunks;
dbp.srcBuffers = (const void* const*)cPtrs; dbp.srcBuffers = (const void* const*)cPtrs;
dbp.srcSizes = cSizes; dbp.srcSizes = cSizes;
dbp.dstBuffers = resPtrs; dbp.dstBuffers = resPtrs;
@@ -690,8 +681,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
} /* while (!(compressionCompleted && decompressionCompleted)) */ } /* while (!(compressionCompleted && decompressionCompleted)) */
/* CRC Checking */ /* CRC Checking */
{ { const BYTE* resultBuffer = (const BYTE*)(*resultBufferPtr);
const BYTE* resultBuffer = (const BYTE*)(*resultBufferPtr);
U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if ((adv->mode == BMK_both) && (crcOrig != crcCheck)) { if ((adv->mode == BMK_both) && (crcOrig != crcCheck)) {
size_t u; size_t u;
@@ -704,14 +694,14 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
unsigned segNb, bNb, pos; unsigned segNb, bNb, pos;
size_t bacc = 0; size_t bacc = 0;
DISPLAY("Decoding error at pos %u ", (unsigned)u); DISPLAY("Decoding error at pos %u ", (unsigned)u);
for (segNb = 0; segNb < nbBlocks; segNb++) { for (segNb = 0; segNb < nbChunks; segNb++) {
if (bacc + srcSizes[segNb] > u) if (bacc + srcSizes[segNb] > u)
break; break;
bacc += srcSizes[segNb]; bacc += srcSizes[segNb];
} }
pos = (U32)(u - bacc); pos = (U32)(u - bacc);
bNb = pos / (128 KB); bNb = pos / (128 KB);
DISPLAY("(sample %u, block %u, pos %u) \n", DISPLAY("(sample %u, chunk %u, pos %u) \n",
segNb, segNb,
bNb, bNb,
pos); pos);
@@ -795,25 +785,24 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(
int const dstParamsError = int const dstParamsError =
!dstBuffer ^ !dstCapacity; /* must be both NULL or none */ !dstBuffer ^ !dstCapacity; /* must be both NULL or none */
size_t const blockSize = size_t const chunkSize =
((adv->blockSize >= 32 && (adv->mode != BMK_decodeOnly)) ((adv->chunkSizeMax >= 32 && (adv->mode != BMK_decodeOnly))
? adv->blockSize ? adv->chunkSizeMax
: srcSize) : srcSize)
+ (!srcSize) /* avoid div by 0 */; + (!srcSize) /* avoid div by 0 */;
U32 const maxNbBlocks = U32 const nbChunksMax =
(U32)((srcSize + (blockSize - 1)) / blockSize) + nbFiles; (U32)((srcSize + (chunkSize - 1)) / chunkSize) + nbFiles;
/* these are the blockTable parameters, just split up */
const void** const srcPtrs = const void** const srcPtrs =
(const void**)malloc(maxNbBlocks * sizeof(void*)); (const void**)malloc(nbChunksMax * sizeof(void*));
size_t* const srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); size_t* const srcSizes = (size_t*)malloc(nbChunksMax * sizeof(size_t));
void** const cPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); void** const cPtrs = (void**)malloc(nbChunksMax * sizeof(void*));
size_t* const cSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); size_t* const cSizes = (size_t*)malloc(nbChunksMax * sizeof(size_t));
size_t* const cCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); size_t* const cCapacities = (size_t*)malloc(nbChunksMax * sizeof(size_t));
void** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); void** const resPtrs = (void**)malloc(nbChunksMax * sizeof(void*));
size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); size_t* const resSizes = (size_t*)malloc(nbChunksMax * sizeof(size_t));
BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState( BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState(
adv->nbSeconds * 1000, BMK_RUNTEST_DEFAULT_MS); adv->nbSeconds * 1000, BMK_RUNTEST_DEFAULT_MS);
@@ -825,7 +814,7 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(
const size_t maxCompressedSize = dstCapacity const size_t maxCompressedSize = dstCapacity
? dstCapacity ? dstCapacity
: ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); : ZSTD_compressBound(srcSize) + (nbChunksMax * 1024);
void* const internalDstBuffer = void* const internalDstBuffer =
dstBuffer ? NULL : malloc(maxCompressedSize); dstBuffer ? NULL : malloc(maxCompressedSize);
@@ -964,12 +953,12 @@ static int BMK_benchCLevels(
} }
if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */ if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */
OUTPUT("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", OUTPUT("bench %s %s: input %u bytes, %u seconds, %u KB chunks\n",
ZSTD_VERSION_STRING, ZSTD_VERSION_STRING,
ZSTD_GIT_COMMIT_STRING, ZSTD_GIT_COMMIT_STRING,
(unsigned)benchedSize, (unsigned)benchedSize,
adv->nbSeconds, adv->nbSeconds,
(unsigned)(adv->blockSize >> 10)); (unsigned)(adv->chunkSizeMax >> 10));
for (level = startCLevel; level <= endCLevel; level++) { for (level = startCLevel; level <= endCLevel; level++) {
BMK_benchOutcome_t res = BMK_benchMemAdvanced( BMK_benchOutcome_t res = BMK_benchMemAdvanced(
@@ -1000,7 +989,7 @@ int BMK_syntheticTest(
{ {
char nameBuff[20] = { 0 }; char nameBuff[20] = { 0 };
const char* name = nameBuff; const char* name = nameBuff;
size_t const benchedSize = adv->blockSize ? adv->blockSize : 10000000; size_t const benchedSize = adv->chunkSizeMax ? adv->chunkSizeMax : 10000000;
/* Memory allocation */ /* Memory allocation */
void* const srcBuffer = malloc(benchedSize); void* const srcBuffer = malloc(benchedSize);
programs/benchzstd.h
+2 -2
View File
@@ -92,9 +92,9 @@ typedef enum {
} BMK_mode_t; } BMK_mode_t;
typedef struct { typedef struct {
BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */ BMK_mode_t mode; /* 0: both, 1: compress only 2: decode only */
unsigned nbSeconds; /* default timing is in nbSeconds */ unsigned nbSeconds; /* default timing is in nbSeconds */
size_t blockSize; /* Maximum size of each block*/ size_t chunkSizeMax; /* Maximum size of each independent chunk */
size_t targetCBlockSize;/* Approximative size of compressed blocks */ size_t targetCBlockSize;/* Approximative size of compressed blocks */
int nbWorkers; /* multithreading */ int nbWorkers; /* multithreading */
unsigned realTime; /* real time priority */ unsigned realTime; /* real time priority */
programs/dibio.c
+3 -3
View File
@@ -362,9 +362,9 @@ int DiB_trainFromFiles(const char* dictFileName, size_t maxDictSize,
DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each sample are loaded \n", SAMPLESIZE_MAX); DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each sample are loaded \n", SAMPLESIZE_MAX);
} }
if (fs.nbSamples < 5) { if (fs.nbSamples < 5) {
DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n"); DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing !\n");
DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n"); DISPLAYLEVEL(2, "! Please provide _one file per sample_.\n");
DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n"); DISPLAYLEVEL(2, "! Alternatively, split file(s) into fixed-size samples, with --split=#\n");
EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */ EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */
} }
if (fs.totalSizeToLoad < (S64)maxDictSize * 8) { if (fs.totalSizeToLoad < (S64)maxDictSize * 8) {
programs/fileio.c
+6 -6
View File
@@ -288,7 +288,7 @@ FIO_prefs_t* FIO_createPreferences(void)
ret->removeSrcFile = 0; ret->removeSrcFile = 0;
ret->memLimit = 0; ret->memLimit = 0;
ret->nbWorkers = 1; ret->nbWorkers = 1;
ret->blockSize = 0; ret->jobSize = 0;
ret->overlapLog = FIO_OVERLAP_LOG_NOTSET; ret->overlapLog = FIO_OVERLAP_LOG_NOTSET;
ret->adaptiveMode = 0; ret->adaptiveMode = 0;
ret->rsyncable = 0; ret->rsyncable = 0;
@@ -377,10 +377,10 @@ void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompresse
void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockDevices = allowBlockDevices; } void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockDevices = allowBlockDevices; }
void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) { void FIO_setJobSize(FIO_prefs_t* const prefs, int jobSize) {
if (blockSize && prefs->nbWorkers==0) if (jobSize && prefs->nbWorkers==0)
DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
prefs->blockSize = blockSize; prefs->jobSize = jobSize;
} }
void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){ void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){
@@ -1183,7 +1183,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
#ifdef ZSTD_MULTITHREAD #ifdef ZSTD_MULTITHREAD
DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers); DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->jobSize) );
if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) { if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) {
DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog); DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) );
@@ -2118,7 +2118,7 @@ void FIO_displayCompressionParameters(const FIO_prefs_t* prefs)
DISPLAY("%s", INDEX(sparseOptions, prefs->sparseFileSupport)); DISPLAY("%s", INDEX(sparseOptions, prefs->sparseFileSupport));
DISPLAY("%s", prefs->dictIDFlag ? "" : " --no-dictID"); DISPLAY("%s", prefs->dictIDFlag ? "" : " --no-dictID");
DISPLAY("%s", INDEX(checkSumOptions, prefs->checksumFlag)); DISPLAY("%s", INDEX(checkSumOptions, prefs->checksumFlag));
DISPLAY(" --block-size=%d", prefs->blockSize); DISPLAY(" --jobsize=%d", prefs->jobSize);
if (prefs->adaptiveMode) if (prefs->adaptiveMode)
DISPLAY(" --adapt=min=%d,max=%d", prefs->minAdaptLevel, prefs->maxAdaptLevel); DISPLAY(" --adapt=min=%d,max=%d", prefs->minAdaptLevel, prefs->maxAdaptLevel);
DISPLAY("%s", INDEX(rowMatchFinderOptions, prefs->useRowMatchFinder)); DISPLAY("%s", INDEX(rowMatchFinderOptions, prefs->useRowMatchFinder));
programs/fileio.h
+1 -1
View File
@@ -70,7 +70,7 @@ void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt);
void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel); void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel);
void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel); void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel);
void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder); void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder);
void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize); void FIO_setJobSize(FIO_prefs_t* const prefs, int jobSize);
void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag); void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag);
void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag); void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag);
void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog); void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog);
programs/fileio_types.h
+1 -1
View File
@@ -37,7 +37,7 @@ typedef struct FIO_prefs_s {
int sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ int sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
int dictIDFlag; int dictIDFlag;
int checksumFlag; int checksumFlag;
int blockSize; int jobSize;
int overlapLog; int overlapLog;
int adaptiveMode; int adaptiveMode;
int useRowMatchFinder; int useRowMatchFinder;
programs/zstd.1.md
+13 -13
View File
@@ -113,7 +113,11 @@ the last one takes effect.
Because the compressor's behavior highly depends on the content to compress, there's no guarantee of a smooth progression from one level to another. Because the compressor's behavior highly depends on the content to compress, there's no guarantee of a smooth progression from one level to another.
* `--ultra`: * `--ultra`:
unlocks high compression levels 20+ (maximum 22), using a lot more memory. unlocks high compression levels 20+ (maximum 22), using a lot more memory.
Note that decompression will also require more memory when using these levels. Decompression will also need more memory when using these levels.
* `--max`:
set advanced parameters to reach maximum compression.
warning: this setting is very slow and uses a lot of resources.
It's inappropriate for 32-bit mode and therefore disabled in this mode.
* `--fast[=#]`: * `--fast[=#]`:
switch to ultra-fast compression levels. switch to ultra-fast compression levels.
If `=#` is not present, it defaults to `1`. If `=#` is not present, it defaults to `1`.
@@ -161,10 +165,6 @@ the last one takes effect.
Note: If `windowLog` is set to larger than 27, `--long=windowLog` or Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
`--memory=windowSize` needs to be passed to the decompressor. `--memory=windowSize` needs to be passed to the decompressor.
* `--max`:
set advanced parameters to maximum compression.
warning: this setting is very slow and uses a lot of resources.
It's inappropriate for 32-bit mode and therefore disabled in this mode.
* `-D DICT`: * `-D DICT`:
use `DICT` as Dictionary to compress or decompress FILE(s) use `DICT` as Dictionary to compress or decompress FILE(s)
* `--patch-from FILE`: * `--patch-from FILE`:
@@ -503,12 +503,12 @@ similar to predefined level 19 for files bigger than 256 KB:
`--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6 `--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
### -B#: ### --jobsize=#:
Specify the size of each compression job. Specify the size of each compression job.
This parameter is only available when multi-threading is enabled. This parameter is only meaningful when multi-threading is enabled.
Each compression job is run in parallel, so this value indirectly impacts the nb of active threads. Each compression job is run in parallel, so this value can indirectly impact the nb of active threads.
Default job size varies depending on compression level (generally `4 * windowSize`). Default job size varies depending on compression level (generally `4 * windowSize`).
`-B#` makes it possible to manually select a custom size. `--jobsize=#` makes it possible to manually select a custom size.
Note that job size must respect a minimum value which is enforced transparently. Note that job size must respect a minimum value which is enforced transparently.
This minimum is either 512 KB, or `overlapSize`, whichever is largest. This minimum is either 512 KB, or `overlapSize`, whichever is largest.
Different job sizes will lead to non-identical compressed frames. Different job sizes will lead to non-identical compressed frames.
@@ -554,8 +554,8 @@ Compression of small files similar to the sample set will be greatly improved.
Use `#` compression level during training (optional). Use `#` compression level during training (optional).
Will generate statistics more tuned for selected compression level, Will generate statistics more tuned for selected compression level,
resulting in a _small_ compression ratio improvement for this level. resulting in a _small_ compression ratio improvement for this level.
* `-B#`: * `--split=#`:
Split input files into blocks of size # (default: no split) Split input files into independent chunks of size # (default: no split)
* `-M#`, `--memory=#`: * `-M#`, `--memory=#`:
Limit the amount of sample data loaded for training (default: 2 GB). Limit the amount of sample data loaded for training (default: 2 GB).
Note that the default (2 GB) is also the maximum. Note that the default (2 GB) is also the maximum.
@@ -683,8 +683,8 @@ Benchmarking will employ `max(1, min(4, nbCores/4))` worker threads by default i
benchmark decompression speed only (requires providing a zstd-compressed content) benchmark decompression speed only (requires providing a zstd-compressed content)
* `-i#`: * `-i#`:
minimum evaluation time, in seconds (default: 3s), benchmark mode only minimum evaluation time, in seconds (default: 3s), benchmark mode only
* `-B#`, `--block-size=#`: * `--split=#`:
cut file(s) into independent chunks of size # (default: no chunking) split input file(s) into independent chunks of size # (default: no chunking)
* `-S`: * `-S`:
output one benchmark result per input file (default: consolidated result) output one benchmark result per input file (default: consolidated result)
* `-D dictionary` * `-D dictionary`
programs/zstdcli.c
+42 -31
View File
@@ -235,8 +235,8 @@ static void usageAdvanced(const char* programName)
DISPLAYOUT(" --single-thread Share a single thread for I/O and compression (slightly different than `-T1`).\n"); DISPLAYOUT(" --single-thread Share a single thread for I/O and compression (slightly different than `-T1`).\n");
DISPLAYOUT(" --auto-threads={physical|logical}\n"); DISPLAYOUT(" --auto-threads={physical|logical}\n");
DISPLAYOUT(" Use physical/logical cores when using `-T0`. [Default: Physical]\n\n"); DISPLAYOUT(" Use physical/logical cores when using `-T0`. [Default: Physical]\n\n");
DISPLAYOUT(" -B# Set job size to #. [Default: 0 (automatic)]\n"); DISPLAYOUT(" --jobsize=# Set job size to #. [Default: 0 (automatic)]\n");
DISPLAYOUT(" --rsyncable Compress using a rsync-friendly method (`-B` sets block size). \n"); DISPLAYOUT(" --rsyncable Compress using a rsync-friendly method (`--jobsize=#` sets unit size). \n");
DISPLAYOUT("\n"); DISPLAYOUT("\n");
# endif # endif
DISPLAYOUT(" --exclude-compressed Only compress files that are not already compressed.\n\n"); DISPLAYOUT(" --exclude-compressed Only compress files that are not already compressed.\n\n");
@@ -307,7 +307,7 @@ static void usageAdvanced(const char* programName)
DISPLAYOUT(" -b# Perform benchmarking with compression level #. [Default: %d]\n", ZSTDCLI_CLEVEL_DEFAULT); DISPLAYOUT(" -b# Perform benchmarking with compression level #. [Default: %d]\n", ZSTDCLI_CLEVEL_DEFAULT);
DISPLAYOUT(" -e# Test all compression levels up to #; starting level is `-b#`. [Default: 1]\n"); DISPLAYOUT(" -e# Test all compression levels up to #; starting level is `-b#`. [Default: 1]\n");
DISPLAYOUT(" -i# Set the minimum evaluation to time # seconds. [Default: 3]\n"); DISPLAYOUT(" -i# Set the minimum evaluation to time # seconds. [Default: 3]\n");
DISPLAYOUT(" -B# Cut file into independent chunks of size #. [Default: No chunking]\n"); DISPLAYOUT(" --split=# Split input into independent chunks of size #. [Default: No chunking]\n");
DISPLAYOUT(" -S Output one benchmark result per input file. [Default: Consolidated result]\n"); DISPLAYOUT(" -S Output one benchmark result per input file. [Default: Consolidated result]\n");
DISPLAYOUT(" -D dictionary Benchmark using dictionary \n"); DISPLAYOUT(" -D dictionary Benchmark using dictionary \n");
DISPLAYOUT(" --priority=rt Set process priority to real-time.\n"); DISPLAYOUT(" --priority=rt Set process priority to real-time.\n");
@@ -773,7 +773,7 @@ static int init_cLevel(void) {
} }
#ifdef ZSTD_MULTITHREAD #ifdef ZSTD_MULTITHREAD
static unsigned default_nbThreads(void) { static int default_nbThreads(void) {
const char* const env = getenv(ENV_NBTHREADS); const char* const env = getenv(ENV_NBTHREADS);
if (env != NULL) { if (env != NULL) {
const char* ptr = env; const char* ptr = env;
@@ -783,7 +783,7 @@ static unsigned default_nbThreads(void) {
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NBTHREADS, env); DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NBTHREADS, env);
return ZSTDCLI_NBTHREADS_DEFAULT; return ZSTDCLI_NBTHREADS_DEFAULT;
} else if (*ptr == 0) { } else if (*ptr == 0) {
return nbThreads; return (int)nbThreads;
} }
} }
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBTHREADS, env); DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBTHREADS, env);
@@ -810,22 +810,31 @@ static unsigned default_nbThreads(void) {
CLEAN_RETURN(1); \ CLEAN_RETURN(1); \
} } } } } }
#define NEXT_UINT32(val32) { \ #define NEXT_INT32(_vari32) { \
const char* __nb; \ const char* __nb; \
NEXT_FIELD(__nb); \ NEXT_FIELD(__nb); \
val32 = readU32FromChar(&__nb); \ _vari32 = (int)readU32FromChar(&__nb); \
if(*__nb != 0) { \ if(*__nb != 0) { \
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \ errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
} \ } \
} }
#define NEXT_TSIZE(valTsize) { \ #define NEXT_UINT32(_varu32) { \
const char* __nb; \ const char* __nb; \
NEXT_FIELD(__nb); \ NEXT_FIELD(__nb); \
valTsize = readSizeTFromChar(&__nb); \ _varu32 = readU32FromChar(&__nb); \
if(*__nb != 0) { \ if(*__nb != 0) { \
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \ errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
} \ } \
}
#define NEXT_TSIZE(_varTsize) { \
const char* __nb; \
NEXT_FIELD(__nb); \
_varTsize = readSizeTFromChar(&__nb); \
if(*__nb != 0) { \
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
} \
} }
typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode; typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
@@ -871,7 +880,7 @@ int main(int argCount, const char* argv[])
int nbWorkers = -1; /* -1 means unset */ int nbWorkers = -1; /* -1 means unset */
double compressibility = -1.0; /* lorem ipsum generator */ double compressibility = -1.0; /* lorem ipsum generator */
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0; size_t chunkSize = 0;
FIO_prefs_t* const prefs = FIO_createPreferences(); FIO_prefs_t* const prefs = FIO_createPreferences();
FIO_ctx_t* const fCtx = FIO_createContext(); FIO_ctx_t* const fCtx = FIO_createContext();
@@ -1069,11 +1078,13 @@ int main(int argCount, const char* argv[])
continue; continue;
} }
#endif #endif
if (longCommandWArg(&argument, "--threads")) { NEXT_UINT32(nbWorkers); continue; } if (longCommandWArg(&argument, "--threads")) { NEXT_INT32(nbWorkers); continue; }
if (longCommandWArg(&argument, "--memlimit")) { NEXT_UINT32(memLimit); continue; } if (longCommandWArg(&argument, "--memlimit")) { NEXT_UINT32(memLimit); continue; }
if (longCommandWArg(&argument, "--memory")) { NEXT_UINT32(memLimit); continue; } if (longCommandWArg(&argument, "--memory")) { NEXT_UINT32(memLimit); continue; }
if (longCommandWArg(&argument, "--memlimit-decompress")) { NEXT_UINT32(memLimit); continue; } if (longCommandWArg(&argument, "--memlimit-decompress")) { NEXT_UINT32(memLimit); continue; }
if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(blockSize); continue; } if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(chunkSize); continue; } /* hidden command, prefer --split below */
if (longCommandWArg(&argument, "--split")) { NEXT_TSIZE(chunkSize); continue; }
if (longCommandWArg(&argument, "--jobsize")) { NEXT_TSIZE(chunkSize); continue; } /* note: overloaded variable */
if (longCommandWArg(&argument, "--maxdict")) { NEXT_UINT32(maxDictSize); continue; } if (longCommandWArg(&argument, "--maxdict")) { NEXT_UINT32(maxDictSize); continue; }
if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; } if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; }
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badUsage(programName, originalArgument); CLEAN_RETURN(1); } ; cType = FIO_zstdCompression; continue; } if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badUsage(programName, originalArgument); CLEAN_RETURN(1); } ; cType = FIO_zstdCompression; continue; }
@@ -1256,10 +1267,10 @@ int main(int argCount, const char* argv[])
bench_nbSeconds = readU32FromChar(&argument); bench_nbSeconds = readU32FromChar(&argument);
break; break;
/* cut input into blocks (benchmark only) */ /* hidden shortcut for --split=# and --jobsize=# */
case 'B': case 'B':
argument++; argument++;
blockSize = readU32FromChar(&argument); chunkSize = readU32FromChar(&argument);
break; break;
/* benchmark files separately (hidden option) */ /* benchmark files separately (hidden option) */
@@ -1273,7 +1284,7 @@ int main(int argCount, const char* argv[])
/* nb of threads (hidden option) */ /* nb of threads (hidden option) */
case 'T': case 'T':
argument++; argument++;
nbWorkers = readU32FromChar(&argument); nbWorkers = (int)readU32FromChar(&argument);
break; break;
/* Dictionary Selection level */ /* Dictionary Selection level */
@@ -1324,10 +1335,10 @@ int main(int argCount, const char* argv[])
if ((nbWorkers==0) && (!singleThread)) { if ((nbWorkers==0) && (!singleThread)) {
/* automatically set # workers based on # of reported cpus */ /* automatically set # workers based on # of reported cpus */
if (defaultLogicalCores) { if (defaultLogicalCores) {
nbWorkers = (unsigned)UTIL_countLogicalCores(); nbWorkers = UTIL_countLogicalCores();
DISPLAYLEVEL(3, "Note: %d logical core(s) detected \n", nbWorkers); DISPLAYLEVEL(3, "Note: %d logical core(s) detected \n", nbWorkers);
} else { } else {
nbWorkers = (unsigned)UTIL_countPhysicalCores(); nbWorkers = UTIL_countPhysicalCores();
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers); DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
} }
} }
@@ -1404,7 +1415,7 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(1, "benchmark mode is only compatible with zstd format \n"); DISPLAYLEVEL(1, "benchmark mode is only compatible with zstd format \n");
CLEAN_RETURN(1); CLEAN_RETURN(1);
} }
benchParams.blockSize = blockSize; benchParams.chunkSizeMax = chunkSize;
benchParams.targetCBlockSize = targetCBlockSize; benchParams.targetCBlockSize = targetCBlockSize;
benchParams.nbWorkers = (int)nbWorkers; benchParams.nbWorkers = (int)nbWorkers;
benchParams.realTime = (unsigned)setRealTimePrio; benchParams.realTime = (unsigned)setRealTimePrio;
@@ -1448,7 +1459,7 @@ int main(int argCount, const char* argv[])
} }
#else #else
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility; (void)bench_nbSeconds; (void)chunkSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;
#endif #endif
goto _end; goto _end;
} }
@@ -1464,18 +1475,18 @@ int main(int argCount, const char* argv[])
int const optimize = !coverParams.k || !coverParams.d; int const optimize = !coverParams.k || !coverParams.d;
coverParams.nbThreads = (unsigned)nbWorkers; coverParams.nbThreads = (unsigned)nbWorkers;
coverParams.zParams = zParams; coverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, NULL, &coverParams, NULL, optimize, memLimit); operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, chunkSize, NULL, &coverParams, NULL, optimize, memLimit);
} else if (dict == fastCover) { } else if (dict == fastCover) {
int const optimize = !fastCoverParams.k || !fastCoverParams.d; int const optimize = !fastCoverParams.k || !fastCoverParams.d;
fastCoverParams.nbThreads = (unsigned)nbWorkers; fastCoverParams.nbThreads = (unsigned)nbWorkers;
fastCoverParams.zParams = zParams; fastCoverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, NULL, NULL, &fastCoverParams, optimize, memLimit); operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, chunkSize, NULL, NULL, &fastCoverParams, optimize, memLimit);
} else { } else {
ZDICT_legacy_params_t dictParams; ZDICT_legacy_params_t dictParams;
memset(&dictParams, 0, sizeof(dictParams)); memset(&dictParams, 0, sizeof(dictParams));
dictParams.selectivityLevel = dictSelect; dictParams.selectivityLevel = dictSelect;
dictParams.zParams = zParams; dictParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, &dictParams, NULL, NULL, 0, memLimit); operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, chunkSize, &dictParams, NULL, NULL, 0, memLimit);
} }
#else #else
(void)dictCLevel; (void)dictSelect; (void)dictID; (void)maxDictSize; /* not used when ZSTD_NODICT set */ (void)dictCLevel; (void)dictSelect; (void)dictID; (void)maxDictSize; /* not used when ZSTD_NODICT set */
@@ -1583,7 +1594,7 @@ int main(int argCount, const char* argv[])
FIO_setCompressionType(prefs, cType); FIO_setCompressionType(prefs, cType);
FIO_setContentSize(prefs, contentSize); FIO_setContentSize(prefs, contentSize);
FIO_setNbWorkers(prefs, (int)nbWorkers); FIO_setNbWorkers(prefs, (int)nbWorkers);
FIO_setBlockSize(prefs, (int)blockSize); FIO_setJobSize(prefs, (int)chunkSize);
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog); if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
FIO_setLdmFlag(prefs, (unsigned)ldmFlag); FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
FIO_setLdmHashLog(prefs, (int)g_ldmHashLog); FIO_setLdmHashLog(prefs, (int)g_ldmHashLog);
tests/cli-tests/compression/multi-threaded.sh
+1
View File
@@ -9,6 +9,7 @@ zstd --rsyncable -f file -q ; zstd -t file.zst
zstd -T0 -f file -q ; zstd -t file.zst zstd -T0 -f file -q ; zstd -t file.zst
zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst
zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst
zstd -T0 --jobsize=1M -f file -q ; zstd -t file.zst
# multi-thread decompression warning test # multi-thread decompression warning test
zstd -T0 -f file -q ; zstd -t file.zst; zstd -T0 -d file.zst -o file3 zstd -T0 -f file -q ; zstd -t file.zst; zstd -T0 -d file.zst -o file3
tests/cli-tests/compression/multi-threaded.sh.stderr.exact
+1
View File
@@ -7,5 +7,6 @@ file.zst : 65537 bytes
file.zst : 65537 bytes file.zst : 65537 bytes
file.zst : 65537 bytes file.zst : 65537 bytes
file.zst : 65537 bytes file.zst : 65537 bytes
file.zst : 65537 bytes
Warning : decompression does not support multi-threading Warning : decompression does not support multi-threading
file.zst : 65537 bytes file.zst : 65537 bytes
tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact
+1 -1
View File
@@ -1,5 +1,5 @@
zstd --train zstd --train
! Warning : nb of samples too low for proper processing ! ! Warning : nb of samples too low for proper processing !
! Please provide _one file per sample_. ! Please provide _one file per sample_.
! Alternatively, split files into fixed-size blocks representative of samples, with -B# ! Alternatively, split file(s) into fixed-size samples, with --split=#
Error 14 : nb of samples too low Error 14 : nb of samples too low
tests/playTests.sh
+6 -6
View File
@@ -1093,8 +1093,8 @@ println "\n===> dictionary tests "
println "- Test high/low compressibility corpus training" println "- Test high/low compressibility corpus training"
datagen -g12M -P90 > tmpCorpusHighCompress datagen -g12M -P90 > tmpCorpusHighCompress
datagen -g12M -P5 > tmpCorpusLowCompress datagen -g12M -P5 > tmpCorpusLowCompress
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress zstd --train --split=2K tmpCorpusHighCompress -o tmpDictHighCompress
zstd --train -B2K tmpCorpusLowCompress -o tmpDictLowCompress zstd --train --split=2K tmpCorpusLowCompress -o tmpDictLowCompress
rm -f tmpCorpusHighCompress tmpCorpusLowCompress tmpDictHighCompress tmpDictLowCompress rm -f tmpCorpusHighCompress tmpCorpusLowCompress tmpDictHighCompress tmpDictLowCompress
println "- Test with raw dict (content only) " println "- Test with raw dict (content only) "
datagen > tmpDict datagen > tmpDict
@@ -1179,8 +1179,8 @@ rm -f tmp* dictionary
println "- Test --memory for dictionary compression" println "- Test --memory for dictionary compression"
datagen -g12M -P90 > tmpCorpusHighCompress datagen -g12M -P90 > tmpCorpusHighCompress
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)" zstd --train --split=2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)"
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr zstd --train --split=2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr
cat zstTrainWithMemLimitStdErr | $GREP "setting manual memory limit for dictionary training data at 5 MB" cat zstTrainWithMemLimitStdErr | $GREP "setting manual memory limit for dictionary training data at 5 MB"
cat zstTrainWithMemLimitStdErr | $GREP "Training samples set too large (12 MB); training on 5 MB only..." cat zstTrainWithMemLimitStdErr | $GREP "Training samples set too large (12 MB); training on 5 MB only..."
rm zstTrainWithMemLimitStdErr rm zstTrainWithMemLimitStdErr
@@ -1555,7 +1555,7 @@ then
roundTripTest -g4M "1 -T0 --auto-threads=logical" roundTripTest -g4M "1 -T0 --auto-threads=logical"
roundTripTest -g8M "3 -T2" roundTripTest -g8M "3 -T2"
roundTripTest -g8000K "2 --threads=2" roundTripTest -g8000K "2 --threads=2"
fileRoundTripTest -g4M "19 -T2 -B1M" fileRoundTripTest -g4M "19 -T2 --split=1M"
println "\n===> zstdmt long distance matching round-trip tests " println "\n===> zstdmt long distance matching round-trip tests "
roundTripTest -g8M "3 --long=24 -T2" roundTripTest -g8M "3 --long=24 -T2"
@@ -1770,7 +1770,7 @@ then
println "\n===> rsyncable mode " println "\n===> rsyncable mode "
roundTripTest -g10M " --rsyncable" roundTripTest -g10M " --rsyncable"
roundTripTest -g10M " --rsyncable -B100K" roundTripTest -g10M " --rsyncable --split=100K"
println "===> test: --rsyncable must fail with --single-thread" println "===> test: --rsyncable must fail with --single-thread"
zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread" zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread"
fi fi