ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4329 from Cyan4973/cmd_split

Browse Source 
New commands --split and --jobsize
This commit is contained in:
Yann Collet
2025-03-05 07:46:04 -08:00
committed by GitHub
parent 190a620974 5ae1cb9fa1
commit 99cf130cfc
12 changed files with 130 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/benchzstd.c
+51 -62
View File
@@ -208,7 +208,7 @@ BMK_advancedParams_t BMK_initAdvancedParams(void)
BMK_advancedParams_t const res = {
BMK_both, /* mode */
BMK_TIMETEST_DEFAULT_S, /* nbSeconds */
0, /* blockSize */
0, /* chunkSizeMax */
0, /* targetCBlockSize */
0, /* nbWorkers */
0, /* realTime */
@@ -227,16 +227,6 @@ BMK_advancedParams_t BMK_initAdvancedParams(void)
/* ********************************************************
* Bench functions
**********************************************************/
typedef struct {
const void* srcPtr;
size_t srcSize;
void* cPtr;
size_t cRoom;
size_t cSize;
void* resPtr;
size_t resSize;
} blockParam_t;
#undef MIN
#undef MAX
#define MIN(a, b) ((a) < (b) ? (a) : (b))
@@ -435,16 +425,16 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
const char* displayName,
const BMK_advancedParams_t* adv)
{
size_t const blockSize =
((adv->blockSize >= 32 && (adv->mode != BMK_decodeOnly))
? adv->blockSize
size_t const chunkSizeMax =
((adv->chunkSizeMax >= 32 && (adv->mode != BMK_decodeOnly))
? adv->chunkSizeMax
: srcSize)
+ (!srcSize); /* avoid div by 0 */
BMK_benchResult_t benchResult;
size_t const loadedCompressedSize = srcSize;
size_t cSize = 0;
double ratio = 0.;
U32 nbBlocks;
U32 nbChunks = 0;
assert(cctx != NULL);
assert(dctx != NULL);
@@ -500,41 +490,42 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
}
}
/* Init data blocks */
/* Init data chunks */
{
const char* srcPtr = (const char*)srcBuffer;
char* cPtr = (char*)compressedBuffer;
char* resPtr = (char*)(*resultBufferPtr);
U32 fileNb;
for (nbBlocks = 0, fileNb = 0; fileNb < nbFiles; fileNb++) {
U32 fileNb, chunkID;
for (chunkID = 0, fileNb = 0; fileNb < nbFiles; fileNb++) {
size_t remaining = fileSizes[fileNb];
U32 const nbBlocksforThisFile = (adv->mode == BMK_decodeOnly)
U32 const nbChunksforThisFile = (adv->mode == BMK_decodeOnly)
? 1
: (U32)((remaining + (blockSize - 1)) / blockSize);
U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
for (; nbBlocks < blockEnd; nbBlocks++) {
size_t const thisBlockSize = MIN(remaining, blockSize);
srcPtrs[nbBlocks] = srcPtr;
srcSizes[nbBlocks] = thisBlockSize;
cPtrs[nbBlocks] = cPtr;
cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly)
? thisBlockSize
: ZSTD_compressBound(thisBlockSize);
resPtrs[nbBlocks] = resPtr;
resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly)
: (U32)((remaining + (chunkSizeMax - 1)) / chunkSizeMax);
U32 const chunkIdEnd = chunkID + nbChunksforThisFile;
for (; chunkID < chunkIdEnd; chunkID++) {
size_t const chunkSize = MIN(remaining, chunkSizeMax);
srcPtrs[chunkID] = srcPtr;
srcSizes[chunkID] = chunkSize;
cPtrs[chunkID] = cPtr;
cCapacities[chunkID] = (adv->mode == BMK_decodeOnly)
? chunkSize
: ZSTD_compressBound(chunkSize);
resPtrs[chunkID] = resPtr;
resSizes[chunkID] = (adv->mode == BMK_decodeOnly)
? (size_t)ZSTD_findDecompressedSize(
srcPtr, thisBlockSize)
: thisBlockSize;
srcPtr += thisBlockSize;
cPtr += cCapacities[nbBlocks];
resPtr += thisBlockSize;
remaining -= thisBlockSize;
srcPtr, chunkSize)
: chunkSize;
srcPtr += chunkSize;
cPtr += cCapacities[chunkID];
resPtr += chunkSize;
remaining -= chunkSize;
if (adv->mode == BMK_decodeOnly) {
cSizes[nbBlocks] = thisBlockSize;
benchResult.cSize = thisBlockSize;
cSizes[chunkID] = chunkSize;
benchResult.cSize = chunkSize;
}
}
}
nbChunks = chunkID;
}
/* warming up `compressedBuffer` */
@@ -569,7 +560,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
cbp.initFn = local_initCCtx; /* BMK_initCCtx */
cbp.initPayload = &cctxprep;
cbp.errorFn = ZSTD_isError;
cbp.blockCount = nbBlocks;
cbp.blockCount = nbChunks;
cbp.srcBuffers = srcPtrs;
cbp.srcSizes = srcSizes;
cbp.dstBuffers = cPtrs;
@@ -588,7 +579,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
dbp.initFn = local_initDCtx;
dbp.initPayload = &dctxprep;
dbp.errorFn = ZSTD_isError;
dbp.blockCount = nbBlocks;
dbp.blockCount = nbChunks;
dbp.srcBuffers = (const void* const*)cPtrs;
dbp.srcSizes = cSizes;
dbp.dstBuffers = resPtrs;
@@ -690,8 +681,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
} /* while (!(compressionCompleted && decompressionCompleted)) */
/* CRC Checking */
{
const BYTE* resultBuffer = (const BYTE*)(*resultBufferPtr);
{ const BYTE* resultBuffer = (const BYTE*)(*resultBufferPtr);
U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if ((adv->mode == BMK_both) && (crcOrig != crcCheck)) {
size_t u;
@@ -704,14 +694,14 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
unsigned segNb, bNb, pos;
size_t bacc = 0;
DISPLAY("Decoding error at pos %u ", (unsigned)u);
for (segNb = 0; segNb < nbBlocks; segNb++) {
for (segNb = 0; segNb < nbChunks; segNb++) {
if (bacc + srcSizes[segNb] > u)
break;
bacc += srcSizes[segNb];
}
pos = (U32)(u - bacc);
bNb = pos / (128 KB);
DISPLAY("(sample %u, block %u, pos %u) \n",
DISPLAY("(sample %u, chunk %u, pos %u) \n",
segNb,
bNb,
pos);
@@ -795,25 +785,24 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(
int const dstParamsError =
!dstBuffer ^ !dstCapacity; /* must be both NULL or none */
size_t const blockSize =
((adv->blockSize >= 32 && (adv->mode != BMK_decodeOnly))
? adv->blockSize
size_t const chunkSize =
((adv->chunkSizeMax >= 32 && (adv->mode != BMK_decodeOnly))
? adv->chunkSizeMax
: srcSize)
+ (!srcSize) /* avoid div by 0 */;
U32 const maxNbBlocks =
(U32)((srcSize + (blockSize - 1)) / blockSize) + nbFiles;
U32 const nbChunksMax =
(U32)((srcSize + (chunkSize - 1)) / chunkSize) + nbFiles;
/* these are the blockTable parameters, just split up */
const void** const srcPtrs =
(const void**)malloc(maxNbBlocks * sizeof(void*));
size_t* const srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
(const void**)malloc(nbChunksMax * sizeof(void*));
size_t* const srcSizes = (size_t*)malloc(nbChunksMax * sizeof(size_t));
void** const cPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
size_t* const cSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
size_t* const cCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
void** const cPtrs = (void**)malloc(nbChunksMax * sizeof(void*));
size_t* const cSizes = (size_t*)malloc(nbChunksMax * sizeof(size_t));
size_t* const cCapacities = (size_t*)malloc(nbChunksMax * sizeof(size_t));
void** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
void** const resPtrs = (void**)malloc(nbChunksMax * sizeof(void*));
size_t* const resSizes = (size_t*)malloc(nbChunksMax * sizeof(size_t));
BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState(
adv->nbSeconds * 1000, BMK_RUNTEST_DEFAULT_MS);
@@ -825,7 +814,7 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(
const size_t maxCompressedSize = dstCapacity
? dstCapacity
: ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);
: ZSTD_compressBound(srcSize) + (nbChunksMax * 1024);
void* const internalDstBuffer =
dstBuffer ? NULL : malloc(maxCompressedSize);
@@ -964,12 +953,12 @@ static int BMK_benchCLevels(
}
if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */
OUTPUT("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n",
OUTPUT("bench %s %s: input %u bytes, %u seconds, %u KB chunks\n",
ZSTD_VERSION_STRING,
ZSTD_GIT_COMMIT_STRING,
(unsigned)benchedSize,
adv->nbSeconds,
(unsigned)(adv->blockSize >> 10));
(unsigned)(adv->chunkSizeMax >> 10));
for (level = startCLevel; level <= endCLevel; level++) {
BMK_benchOutcome_t res = BMK_benchMemAdvanced(
@@ -1000,7 +989,7 @@ int BMK_syntheticTest(
{
char nameBuff[20] = { 0 };
const char* name = nameBuff;
size_t const benchedSize = adv->blockSize ? adv->blockSize : 10000000;
size_t const benchedSize = adv->chunkSizeMax ? adv->chunkSizeMax : 10000000;
/* Memory allocation */
void* const srcBuffer = malloc(benchedSize);
programs/benchzstd.h
+2 -2
View File
@@ -92,9 +92,9 @@ typedef enum {
} BMK_mode_t;
typedef struct {
BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */
BMK_mode_t mode; /* 0: both, 1: compress only 2: decode only */
unsigned nbSeconds; /* default timing is in nbSeconds */
size_t blockSize; /* Maximum size of each block*/
size_t chunkSizeMax; /* Maximum size of each independent chunk */
size_t targetCBlockSize;/* Approximative size of compressed blocks */
int nbWorkers; /* multithreading */
unsigned realTime; /* real time priority */
programs/dibio.c
+1 -1
View File
@@ -364,7 +364,7 @@ int DiB_trainFromFiles(const char* dictFileName, size_t maxDictSize,
if (fs.nbSamples < 5) {
DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing !\n");
DISPLAYLEVEL(2, "! Please provide _one file per sample_.\n");
DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n");
DISPLAYLEVEL(2, "! Alternatively, split file(s) into fixed-size samples, with --split=#\n");
EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */
}
if (fs.totalSizeToLoad < (S64)maxDictSize * 8) {
programs/fileio.c
+6 -6
View File
@@ -288,7 +288,7 @@ FIO_prefs_t* FIO_createPreferences(void)
ret->removeSrcFile = 0;
ret->memLimit = 0;
ret->nbWorkers = 1;
ret->blockSize = 0;
ret->jobSize = 0;
ret->overlapLog = FIO_OVERLAP_LOG_NOTSET;
ret->adaptiveMode = 0;
ret->rsyncable = 0;
@@ -377,10 +377,10 @@ void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompresse
void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockDevices = allowBlockDevices; }
void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) {
if (blockSize && prefs->nbWorkers==0)
void FIO_setJobSize(FIO_prefs_t* const prefs, int jobSize) {
if (jobSize && prefs->nbWorkers==0)
DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
prefs->blockSize = blockSize;
prefs->jobSize = jobSize;
}
void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){
@@ -1183,7 +1183,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
#ifdef ZSTD_MULTITHREAD
DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->jobSize) );
if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) {
DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) );
@@ -2118,7 +2118,7 @@ void FIO_displayCompressionParameters(const FIO_prefs_t* prefs)
DISPLAY("%s", INDEX(sparseOptions, prefs->sparseFileSupport));
DISPLAY("%s", prefs->dictIDFlag ? "" : " --no-dictID");
DISPLAY("%s", INDEX(checkSumOptions, prefs->checksumFlag));
DISPLAY(" --block-size=%d", prefs->blockSize);
DISPLAY(" --jobsize=%d", prefs->jobSize);
if (prefs->adaptiveMode)
DISPLAY(" --adapt=min=%d,max=%d", prefs->minAdaptLevel, prefs->maxAdaptLevel);
DISPLAY("%s", INDEX(rowMatchFinderOptions, prefs->useRowMatchFinder));
programs/fileio.h
+1 -1
View File
@@ -70,7 +70,7 @@ void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt);
void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel);
void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel);
void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder);
void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize);
void FIO_setJobSize(FIO_prefs_t* const prefs, int jobSize);
void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag);
void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag);
void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog);
programs/fileio_types.h
+1 -1
View File
@@ -37,7 +37,7 @@ typedef struct FIO_prefs_s {
int sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
int dictIDFlag;
int checksumFlag;
int blockSize;
int jobSize;
int overlapLog;
int adaptiveMode;
int useRowMatchFinder;
programs/zstd.1.md
+13 -13
View File
@@ -113,7 +113,11 @@ the last one takes effect.
Because the compressor's behavior highly depends on the content to compress, there's no guarantee of a smooth progression from one level to another.
* `--ultra`:
unlocks high compression levels 20+ (maximum 22), using a lot more memory.
Note that decompression will also require more memory when using these levels.
Decompression will also need more memory when using these levels.
* `--max`:
set advanced parameters to reach maximum compression.
warning: this setting is very slow and uses a lot of resources.
It's inappropriate for 32-bit mode and therefore disabled in this mode.
* `--fast[=#]`:
switch to ultra-fast compression levels.
If `=#` is not present, it defaults to `1`.
@@ -161,10 +165,6 @@ the last one takes effect.
Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
`--memory=windowSize` needs to be passed to the decompressor.
* `--max`:
set advanced parameters to maximum compression.
warning: this setting is very slow and uses a lot of resources.
It's inappropriate for 32-bit mode and therefore disabled in this mode.
* `-D DICT`:
use `DICT` as Dictionary to compress or decompress FILE(s)
* `--patch-from FILE`:
@@ -503,12 +503,12 @@ similar to predefined level 19 for files bigger than 256 KB:
`--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
### -B#:
### --jobsize=#:
Specify the size of each compression job.
This parameter is only available when multi-threading is enabled.
Each compression job is run in parallel, so this value indirectly impacts the nb of active threads.
This parameter is only meaningful when multi-threading is enabled.
Each compression job is run in parallel, so this value can indirectly impact the nb of active threads.
Default job size varies depending on compression level (generally `4 * windowSize`).
`-B#` makes it possible to manually select a custom size.
`--jobsize=#` makes it possible to manually select a custom size.
Note that job size must respect a minimum value which is enforced transparently.
This minimum is either 512 KB, or `overlapSize`, whichever is largest.
Different job sizes will lead to non-identical compressed frames.
@@ -554,8 +554,8 @@ Compression of small files similar to the sample set will be greatly improved.
Use `#` compression level during training (optional).
Will generate statistics more tuned for selected compression level,
resulting in a _small_ compression ratio improvement for this level.
* `-B#`:
Split input files into blocks of size # (default: no split)
* `--split=#`:
Split input files into independent chunks of size # (default: no split)
* `-M#`, `--memory=#`:
Limit the amount of sample data loaded for training (default: 2 GB).
Note that the default (2 GB) is also the maximum.
@@ -683,8 +683,8 @@ Benchmarking will employ `max(1, min(4, nbCores/4))` worker threads by default i
benchmark decompression speed only (requires providing a zstd-compressed content)
* `-i#`:
minimum evaluation time, in seconds (default: 3s), benchmark mode only
* `-B#`, `--block-size=#`:
cut file(s) into independent chunks of size # (default: no chunking)
* `--split=#`:
split input file(s) into independent chunks of size # (default: no chunking)
* `-S`:
output one benchmark result per input file (default: consolidated result)
* `-D dictionary`
programs/zstdcli.c
+34 -23
View File
@@ -235,8 +235,8 @@ static void usageAdvanced(const char* programName)
DISPLAYOUT(" --single-thread Share a single thread for I/O and compression (slightly different than `-T1`).\n");
DISPLAYOUT(" --auto-threads={physical|logical}\n");
DISPLAYOUT(" Use physical/logical cores when using `-T0`. [Default: Physical]\n\n");
DISPLAYOUT(" -B# Set job size to #. [Default: 0 (automatic)]\n");
DISPLAYOUT(" --rsyncable Compress using a rsync-friendly method (`-B` sets block size). \n");
DISPLAYOUT(" --jobsize=# Set job size to #. [Default: 0 (automatic)]\n");
DISPLAYOUT(" --rsyncable Compress using a rsync-friendly method (`--jobsize=#` sets unit size). \n");
DISPLAYOUT("\n");
# endif
DISPLAYOUT(" --exclude-compressed Only compress files that are not already compressed.\n\n");
@@ -307,7 +307,7 @@ static void usageAdvanced(const char* programName)
DISPLAYOUT(" -b# Perform benchmarking with compression level #. [Default: %d]\n", ZSTDCLI_CLEVEL_DEFAULT);
DISPLAYOUT(" -e# Test all compression levels up to #; starting level is `-b#`. [Default: 1]\n");
DISPLAYOUT(" -i# Set the minimum evaluation to time # seconds. [Default: 3]\n");
DISPLAYOUT(" -B# Cut file into independent chunks of size #. [Default: No chunking]\n");
DISPLAYOUT(" --split=# Split input into independent chunks of size #. [Default: No chunking]\n");
DISPLAYOUT(" -S Output one benchmark result per input file. [Default: Consolidated result]\n");
DISPLAYOUT(" -D dictionary Benchmark using dictionary \n");
DISPLAYOUT(" --priority=rt Set process priority to real-time.\n");
@@ -773,7 +773,7 @@ static int init_cLevel(void) {
}
#ifdef ZSTD_MULTITHREAD
static unsigned default_nbThreads(void) {
static int default_nbThreads(void) {
const char* const env = getenv(ENV_NBTHREADS);
if (env != NULL) {
const char* ptr = env;
@@ -783,7 +783,7 @@ static unsigned default_nbThreads(void) {
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NBTHREADS, env);
return ZSTDCLI_NBTHREADS_DEFAULT;
} else if (*ptr == 0) {
return nbThreads;
return (int)nbThreads;
}
}
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBTHREADS, env);
@@ -810,19 +810,28 @@ static unsigned default_nbThreads(void) {
CLEAN_RETURN(1); \
} } }
#define NEXT_UINT32(val32) { \
#define NEXT_INT32(_vari32) { \
const char* __nb; \
NEXT_FIELD(__nb); \
val32 = readU32FromChar(&__nb); \
_vari32 = (int)readU32FromChar(&__nb); \
if(*__nb != 0) { \
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
} \
}
#define NEXT_TSIZE(valTsize) { \
#define NEXT_UINT32(_varu32) { \
const char* __nb; \
NEXT_FIELD(__nb); \
valTsize = readSizeTFromChar(&__nb); \
_varu32 = readU32FromChar(&__nb); \
if(*__nb != 0) { \
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
} \
}
#define NEXT_TSIZE(_varTsize) { \
const char* __nb; \
NEXT_FIELD(__nb); \
_varTsize = readSizeTFromChar(&__nb); \
if(*__nb != 0) { \
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
} \
@@ -871,7 +880,7 @@ int main(int argCount, const char* argv[])
int nbWorkers = -1; /* -1 means unset */
double compressibility = -1.0; /* lorem ipsum generator */
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0;
size_t chunkSize = 0;
FIO_prefs_t* const prefs = FIO_createPreferences();
FIO_ctx_t* const fCtx = FIO_createContext();
@@ -1069,11 +1078,13 @@ int main(int argCount, const char* argv[])
continue;
}
#endif
if (longCommandWArg(&argument, "--threads")) { NEXT_UINT32(nbWorkers); continue; }
if (longCommandWArg(&argument, "--threads")) { NEXT_INT32(nbWorkers); continue; }
if (longCommandWArg(&argument, "--memlimit")) { NEXT_UINT32(memLimit); continue; }
if (longCommandWArg(&argument, "--memory")) { NEXT_UINT32(memLimit); continue; }
if (longCommandWArg(&argument, "--memlimit-decompress")) { NEXT_UINT32(memLimit); continue; }
if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(blockSize); continue; }
if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(chunkSize); continue; } /* hidden command, prefer --split below */
if (longCommandWArg(&argument, "--split")) { NEXT_TSIZE(chunkSize); continue; }
if (longCommandWArg(&argument, "--jobsize")) { NEXT_TSIZE(chunkSize); continue; } /* note: overloaded variable */
if (longCommandWArg(&argument, "--maxdict")) { NEXT_UINT32(maxDictSize); continue; }
if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; }
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badUsage(programName, originalArgument); CLEAN_RETURN(1); } ; cType = FIO_zstdCompression; continue; }
@@ -1256,10 +1267,10 @@ int main(int argCount, const char* argv[])
bench_nbSeconds = readU32FromChar(&argument);
break;
/* cut input into blocks (benchmark only) */
/* hidden shortcut for --split=# and --jobsize=# */
case 'B':
argument++;
blockSize = readU32FromChar(&argument);
chunkSize = readU32FromChar(&argument);
break;
/* benchmark files separately (hidden option) */
@@ -1273,7 +1284,7 @@ int main(int argCount, const char* argv[])
/* nb of threads (hidden option) */
case 'T':
argument++;
nbWorkers = readU32FromChar(&argument);
nbWorkers = (int)readU32FromChar(&argument);
break;
/* Dictionary Selection level */
@@ -1324,10 +1335,10 @@ int main(int argCount, const char* argv[])
if ((nbWorkers==0) && (!singleThread)) {
/* automatically set # workers based on # of reported cpus */
if (defaultLogicalCores) {
nbWorkers = (unsigned)UTIL_countLogicalCores();
nbWorkers = UTIL_countLogicalCores();
DISPLAYLEVEL(3, "Note: %d logical core(s) detected \n", nbWorkers);
} else {
nbWorkers = (unsigned)UTIL_countPhysicalCores();
nbWorkers = UTIL_countPhysicalCores();
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
}
}
@@ -1404,7 +1415,7 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(1, "benchmark mode is only compatible with zstd format \n");
CLEAN_RETURN(1);
}
benchParams.blockSize = blockSize;
benchParams.chunkSizeMax = chunkSize;
benchParams.targetCBlockSize = targetCBlockSize;
benchParams.nbWorkers = (int)nbWorkers;
benchParams.realTime = (unsigned)setRealTimePrio;
@@ -1448,7 +1459,7 @@ int main(int argCount, const char* argv[])
}
#else
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;
(void)bench_nbSeconds; (void)chunkSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;
#endif
goto _end;
}
@@ -1464,18 +1475,18 @@ int main(int argCount, const char* argv[])
int const optimize = !coverParams.k || !coverParams.d;
coverParams.nbThreads = (unsigned)nbWorkers;
coverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, NULL, &coverParams, NULL, optimize, memLimit);
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, chunkSize, NULL, &coverParams, NULL, optimize, memLimit);
} else if (dict == fastCover) {
int const optimize = !fastCoverParams.k || !fastCoverParams.d;
fastCoverParams.nbThreads = (unsigned)nbWorkers;
fastCoverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, NULL, NULL, &fastCoverParams, optimize, memLimit);
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, chunkSize, NULL, NULL, &fastCoverParams, optimize, memLimit);
} else {
ZDICT_legacy_params_t dictParams;
memset(&dictParams, 0, sizeof(dictParams));
dictParams.selectivityLevel = dictSelect;
dictParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, &dictParams, NULL, NULL, 0, memLimit);
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, chunkSize, &dictParams, NULL, NULL, 0, memLimit);
}
#else
(void)dictCLevel; (void)dictSelect; (void)dictID; (void)maxDictSize; /* not used when ZSTD_NODICT set */
@@ -1583,7 +1594,7 @@ int main(int argCount, const char* argv[])
FIO_setCompressionType(prefs, cType);
FIO_setContentSize(prefs, contentSize);
FIO_setNbWorkers(prefs, (int)nbWorkers);
FIO_setBlockSize(prefs, (int)blockSize);
FIO_setJobSize(prefs, (int)chunkSize);
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
FIO_setLdmHashLog(prefs, (int)g_ldmHashLog);
tests/cli-tests/compression/multi-threaded.sh
+1
View File
@@ -9,6 +9,7 @@ zstd --rsyncable -f file -q ; zstd -t file.zst
zstd -T0 -f file -q ; zstd -t file.zst
zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst
zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst
zstd -T0 --jobsize=1M -f file -q ; zstd -t file.zst
# multi-thread decompression warning test
zstd -T0 -f file -q ; zstd -t file.zst; zstd -T0 -d file.zst -o file3
tests/cli-tests/compression/multi-threaded.sh.stderr.exact
+1
View File
@@ -7,5 +7,6 @@ file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
Warning : decompression does not support multi-threading
file.zst : 65537 bytes
tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact
+1 -1
View File
@@ -1,5 +1,5 @@
zstd --train
! Warning : nb of samples too low for proper processing !
! Please provide _one file per sample_.
! Alternatively, split files into fixed-size blocks representative of samples, with -B#
! Alternatively, split file(s) into fixed-size samples, with --split=#
Error 14 : nb of samples too low
tests/playTests.sh
+6 -6
View File
@@ -1093,8 +1093,8 @@ println "\n===> dictionary tests "
println "- Test high/low compressibility corpus training"
datagen -g12M -P90 > tmpCorpusHighCompress
datagen -g12M -P5 > tmpCorpusLowCompress
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress
zstd --train -B2K tmpCorpusLowCompress -o tmpDictLowCompress
zstd --train --split=2K tmpCorpusHighCompress -o tmpDictHighCompress
zstd --train --split=2K tmpCorpusLowCompress -o tmpDictLowCompress
rm -f tmpCorpusHighCompress tmpCorpusLowCompress tmpDictHighCompress tmpDictLowCompress
println "- Test with raw dict (content only) "
datagen > tmpDict
@@ -1179,8 +1179,8 @@ rm -f tmp* dictionary
println "- Test --memory for dictionary compression"
datagen -g12M -P90 > tmpCorpusHighCompress
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)"
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr
zstd --train --split=2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)"
zstd --train --split=2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr
cat zstTrainWithMemLimitStdErr | $GREP "setting manual memory limit for dictionary training data at 5 MB"
cat zstTrainWithMemLimitStdErr | $GREP "Training samples set too large (12 MB); training on 5 MB only..."
rm zstTrainWithMemLimitStdErr
@@ -1555,7 +1555,7 @@ then
roundTripTest -g4M "1 -T0 --auto-threads=logical"
roundTripTest -g8M "3 -T2"
roundTripTest -g8000K "2 --threads=2"
fileRoundTripTest -g4M "19 -T2 -B1M"
fileRoundTripTest -g4M "19 -T2 --split=1M"
println "\n===> zstdmt long distance matching round-trip tests "
roundTripTest -g8M "3 --long=24 -T2"
@@ -1770,7 +1770,7 @@ then
println "\n===> rsyncable mode "
roundTripTest -g10M " --rsyncable"
roundTripTest -g10M " --rsyncable -B100K"
roundTripTest -g10M " --rsyncable --split=100K"
println "===> test: --rsyncable must fail with --single-thread"
zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread"
fi