diff --git a/programs/benchzstd.c b/programs/benchzstd.c index f9274a517..f55c86975 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -208,7 +208,7 @@ BMK_advancedParams_t BMK_initAdvancedParams(void) BMK_advancedParams_t const res = { BMK_both, /* mode */ BMK_TIMETEST_DEFAULT_S, /* nbSeconds */ - 0, /* blockSize */ + 0, /* chunkSizeMax */ 0, /* targetCBlockSize */ 0, /* nbWorkers */ 0, /* realTime */ @@ -227,16 +227,6 @@ BMK_advancedParams_t BMK_initAdvancedParams(void) /* ******************************************************** * Bench functions **********************************************************/ -typedef struct { - const void* srcPtr; - size_t srcSize; - void* cPtr; - size_t cRoom; - size_t cSize; - void* resPtr; - size_t resSize; -} blockParam_t; - #undef MIN #undef MAX #define MIN(a, b) ((a) < (b) ? (a) : (b)) @@ -435,16 +425,16 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc( const char* displayName, const BMK_advancedParams_t* adv) { - size_t const blockSize = - ((adv->blockSize >= 32 && (adv->mode != BMK_decodeOnly)) - ? adv->blockSize + size_t const chunkSizeMax = + ((adv->chunkSizeMax >= 32 && (adv->mode != BMK_decodeOnly)) + ? adv->chunkSizeMax : srcSize) + (!srcSize); /* avoid div by 0 */ BMK_benchResult_t benchResult; size_t const loadedCompressedSize = srcSize; size_t cSize = 0; double ratio = 0.; - U32 nbBlocks; + U32 nbChunks = 0; assert(cctx != NULL); assert(dctx != NULL); @@ -500,41 +490,42 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc( } } - /* Init data blocks */ + /* Init data chunks */ { const char* srcPtr = (const char*)srcBuffer; char* cPtr = (char*)compressedBuffer; char* resPtr = (char*)(*resultBufferPtr); - U32 fileNb; - for (nbBlocks = 0, fileNb = 0; fileNb < nbFiles; fileNb++) { + U32 fileNb, chunkID; + for (chunkID = 0, fileNb = 0; fileNb < nbFiles; fileNb++) { size_t remaining = fileSizes[fileNb]; - U32 const nbBlocksforThisFile = (adv->mode == BMK_decodeOnly) + U32 const nbChunksforThisFile = (adv->mode == BMK_decodeOnly) ? 1 - : (U32)((remaining + (blockSize - 1)) / blockSize); - U32 const blockEnd = nbBlocks + nbBlocksforThisFile; - for (; nbBlocks < blockEnd; nbBlocks++) { - size_t const thisBlockSize = MIN(remaining, blockSize); - srcPtrs[nbBlocks] = srcPtr; - srcSizes[nbBlocks] = thisBlockSize; - cPtrs[nbBlocks] = cPtr; - cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly) - ? thisBlockSize - : ZSTD_compressBound(thisBlockSize); - resPtrs[nbBlocks] = resPtr; - resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) + : (U32)((remaining + (chunkSizeMax - 1)) / chunkSizeMax); + U32 const chunkIdEnd = chunkID + nbChunksforThisFile; + for (; chunkID < chunkIdEnd; chunkID++) { + size_t const chunkSize = MIN(remaining, chunkSizeMax); + srcPtrs[chunkID] = srcPtr; + srcSizes[chunkID] = chunkSize; + cPtrs[chunkID] = cPtr; + cCapacities[chunkID] = (adv->mode == BMK_decodeOnly) + ? chunkSize + : ZSTD_compressBound(chunkSize); + resPtrs[chunkID] = resPtr; + resSizes[chunkID] = (adv->mode == BMK_decodeOnly) ? (size_t)ZSTD_findDecompressedSize( - srcPtr, thisBlockSize) - : thisBlockSize; - srcPtr += thisBlockSize; - cPtr += cCapacities[nbBlocks]; - resPtr += thisBlockSize; - remaining -= thisBlockSize; + srcPtr, chunkSize) + : chunkSize; + srcPtr += chunkSize; + cPtr += cCapacities[chunkID]; + resPtr += chunkSize; + remaining -= chunkSize; if (adv->mode == BMK_decodeOnly) { - cSizes[nbBlocks] = thisBlockSize; - benchResult.cSize = thisBlockSize; + cSizes[chunkID] = chunkSize; + benchResult.cSize = chunkSize; } } } + nbChunks = chunkID; } /* warming up `compressedBuffer` */ @@ -569,7 +560,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc( cbp.initFn = local_initCCtx; /* BMK_initCCtx */ cbp.initPayload = &cctxprep; cbp.errorFn = ZSTD_isError; - cbp.blockCount = nbBlocks; + cbp.blockCount = nbChunks; cbp.srcBuffers = srcPtrs; cbp.srcSizes = srcSizes; cbp.dstBuffers = cPtrs; @@ -588,7 +579,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc( dbp.initFn = local_initDCtx; dbp.initPayload = &dctxprep; dbp.errorFn = ZSTD_isError; - dbp.blockCount = nbBlocks; + dbp.blockCount = nbChunks; dbp.srcBuffers = (const void* const*)cPtrs; dbp.srcSizes = cSizes; dbp.dstBuffers = resPtrs; @@ -690,8 +681,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc( } /* while (!(compressionCompleted && decompressionCompleted)) */ /* CRC Checking */ - { - const BYTE* resultBuffer = (const BYTE*)(*resultBufferPtr); + { const BYTE* resultBuffer = (const BYTE*)(*resultBufferPtr); U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); if ((adv->mode == BMK_both) && (crcOrig != crcCheck)) { size_t u; @@ -704,14 +694,14 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc( unsigned segNb, bNb, pos; size_t bacc = 0; DISPLAY("Decoding error at pos %u ", (unsigned)u); - for (segNb = 0; segNb < nbBlocks; segNb++) { + for (segNb = 0; segNb < nbChunks; segNb++) { if (bacc + srcSizes[segNb] > u) break; bacc += srcSizes[segNb]; } pos = (U32)(u - bacc); bNb = pos / (128 KB); - DISPLAY("(sample %u, block %u, pos %u) \n", + DISPLAY("(sample %u, chunk %u, pos %u) \n", segNb, bNb, pos); @@ -795,25 +785,24 @@ BMK_benchOutcome_t BMK_benchMemAdvanced( int const dstParamsError = !dstBuffer ^ !dstCapacity; /* must be both NULL or none */ - size_t const blockSize = - ((adv->blockSize >= 32 && (adv->mode != BMK_decodeOnly)) - ? adv->blockSize + size_t const chunkSize = + ((adv->chunkSizeMax >= 32 && (adv->mode != BMK_decodeOnly)) + ? adv->chunkSizeMax : srcSize) + (!srcSize) /* avoid div by 0 */; - U32 const maxNbBlocks = - (U32)((srcSize + (blockSize - 1)) / blockSize) + nbFiles; + U32 const nbChunksMax = + (U32)((srcSize + (chunkSize - 1)) / chunkSize) + nbFiles; - /* these are the blockTable parameters, just split up */ const void** const srcPtrs = - (const void**)malloc(maxNbBlocks * sizeof(void*)); - size_t* const srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + (const void**)malloc(nbChunksMax * sizeof(void*)); + size_t* const srcSizes = (size_t*)malloc(nbChunksMax * sizeof(size_t)); - void** const cPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); - size_t* const cSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - size_t* const cCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + void** const cPtrs = (void**)malloc(nbChunksMax * sizeof(void*)); + size_t* const cSizes = (size_t*)malloc(nbChunksMax * sizeof(size_t)); + size_t* const cCapacities = (size_t*)malloc(nbChunksMax * sizeof(size_t)); - void** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); - size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + void** const resPtrs = (void**)malloc(nbChunksMax * sizeof(void*)); + size_t* const resSizes = (size_t*)malloc(nbChunksMax * sizeof(size_t)); BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState( adv->nbSeconds * 1000, BMK_RUNTEST_DEFAULT_MS); @@ -825,7 +814,7 @@ BMK_benchOutcome_t BMK_benchMemAdvanced( const size_t maxCompressedSize = dstCapacity ? dstCapacity - : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); + : ZSTD_compressBound(srcSize) + (nbChunksMax * 1024); void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize); @@ -964,12 +953,12 @@ static int BMK_benchCLevels( } if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */ - OUTPUT("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", + OUTPUT("bench %s %s: input %u bytes, %u seconds, %u KB chunks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (unsigned)benchedSize, adv->nbSeconds, - (unsigned)(adv->blockSize >> 10)); + (unsigned)(adv->chunkSizeMax >> 10)); for (level = startCLevel; level <= endCLevel; level++) { BMK_benchOutcome_t res = BMK_benchMemAdvanced( @@ -1000,7 +989,7 @@ int BMK_syntheticTest( { char nameBuff[20] = { 0 }; const char* name = nameBuff; - size_t const benchedSize = adv->blockSize ? adv->blockSize : 10000000; + size_t const benchedSize = adv->chunkSizeMax ? adv->chunkSizeMax : 10000000; /* Memory allocation */ void* const srcBuffer = malloc(benchedSize); diff --git a/programs/benchzstd.h b/programs/benchzstd.h index 4fd0e5a8a..d62a33c0a 100644 --- a/programs/benchzstd.h +++ b/programs/benchzstd.h @@ -92,9 +92,9 @@ typedef enum { } BMK_mode_t; typedef struct { - BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */ + BMK_mode_t mode; /* 0: both, 1: compress only 2: decode only */ unsigned nbSeconds; /* default timing is in nbSeconds */ - size_t blockSize; /* Maximum size of each block*/ + size_t chunkSizeMax; /* Maximum size of each independent chunk */ size_t targetCBlockSize;/* Approximative size of compressed blocks */ int nbWorkers; /* multithreading */ unsigned realTime; /* real time priority */ diff --git a/programs/dibio.c b/programs/dibio.c index 7ba22d15b..63c455a22 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -362,9 +362,9 @@ int DiB_trainFromFiles(const char* dictFileName, size_t maxDictSize, DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each sample are loaded \n", SAMPLESIZE_MAX); } if (fs.nbSamples < 5) { - DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n"); - DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n"); - DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n"); + DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing !\n"); + DISPLAYLEVEL(2, "! Please provide _one file per sample_.\n"); + DISPLAYLEVEL(2, "! Alternatively, split file(s) into fixed-size samples, with --split=#\n"); EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */ } if (fs.totalSizeToLoad < (S64)maxDictSize * 8) { diff --git a/programs/fileio.c b/programs/fileio.c index 0ecca40d2..3f4460594 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -288,7 +288,7 @@ FIO_prefs_t* FIO_createPreferences(void) ret->removeSrcFile = 0; ret->memLimit = 0; ret->nbWorkers = 1; - ret->blockSize = 0; + ret->jobSize = 0; ret->overlapLog = FIO_OVERLAP_LOG_NOTSET; ret->adaptiveMode = 0; ret->rsyncable = 0; @@ -377,10 +377,10 @@ void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompresse void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockDevices = allowBlockDevices; } -void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) { - if (blockSize && prefs->nbWorkers==0) +void FIO_setJobSize(FIO_prefs_t* const prefs, int jobSize) { + if (jobSize && prefs->nbWorkers==0) DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); - prefs->blockSize = blockSize; + prefs->jobSize = jobSize; } void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){ @@ -1183,7 +1183,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, #ifdef ZSTD_MULTITHREAD DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->jobSize) ); if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) { DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) ); @@ -2118,7 +2118,7 @@ void FIO_displayCompressionParameters(const FIO_prefs_t* prefs) DISPLAY("%s", INDEX(sparseOptions, prefs->sparseFileSupport)); DISPLAY("%s", prefs->dictIDFlag ? "" : " --no-dictID"); DISPLAY("%s", INDEX(checkSumOptions, prefs->checksumFlag)); - DISPLAY(" --block-size=%d", prefs->blockSize); + DISPLAY(" --jobsize=%d", prefs->jobSize); if (prefs->adaptiveMode) DISPLAY(" --adapt=min=%d,max=%d", prefs->minAdaptLevel, prefs->maxAdaptLevel); DISPLAY("%s", INDEX(rowMatchFinderOptions, prefs->useRowMatchFinder)); diff --git a/programs/fileio.h b/programs/fileio.h index cb53ef537..5d7334ef5 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -70,7 +70,7 @@ void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt); void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel); void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel); void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder); -void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize); +void FIO_setJobSize(FIO_prefs_t* const prefs, int jobSize); void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag); void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag); void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog); diff --git a/programs/fileio_types.h b/programs/fileio_types.h index 23bda4168..9bbb51549 100644 --- a/programs/fileio_types.h +++ b/programs/fileio_types.h @@ -37,7 +37,7 @@ typedef struct FIO_prefs_s { int sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ int dictIDFlag; int checksumFlag; - int blockSize; + int jobSize; int overlapLog; int adaptiveMode; int useRowMatchFinder; diff --git a/programs/zstd.1.md b/programs/zstd.1.md index e5c1b7fd2..cf1810d59 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -113,7 +113,11 @@ the last one takes effect. Because the compressor's behavior highly depends on the content to compress, there's no guarantee of a smooth progression from one level to another. * `--ultra`: unlocks high compression levels 20+ (maximum 22), using a lot more memory. - Note that decompression will also require more memory when using these levels. + Decompression will also need more memory when using these levels. +* `--max`: + set advanced parameters to reach maximum compression. + warning: this setting is very slow and uses a lot of resources. + It's inappropriate for 32-bit mode and therefore disabled in this mode. * `--fast[=#]`: switch to ultra-fast compression levels. If `=#` is not present, it defaults to `1`. @@ -161,10 +165,6 @@ the last one takes effect. Note: If `windowLog` is set to larger than 27, `--long=windowLog` or `--memory=windowSize` needs to be passed to the decompressor. -* `--max`: - set advanced parameters to maximum compression. - warning: this setting is very slow and uses a lot of resources. - It's inappropriate for 32-bit mode and therefore disabled in this mode. * `-D DICT`: use `DICT` as Dictionary to compress or decompress FILE(s) * `--patch-from FILE`: @@ -503,12 +503,12 @@ similar to predefined level 19 for files bigger than 256 KB: `--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6 -### -B#: +### --jobsize=#: Specify the size of each compression job. -This parameter is only available when multi-threading is enabled. -Each compression job is run in parallel, so this value indirectly impacts the nb of active threads. +This parameter is only meaningful when multi-threading is enabled. +Each compression job is run in parallel, so this value can indirectly impact the nb of active threads. Default job size varies depending on compression level (generally `4 * windowSize`). -`-B#` makes it possible to manually select a custom size. +`--jobsize=#` makes it possible to manually select a custom size. Note that job size must respect a minimum value which is enforced transparently. This minimum is either 512 KB, or `overlapSize`, whichever is largest. Different job sizes will lead to non-identical compressed frames. @@ -554,8 +554,8 @@ Compression of small files similar to the sample set will be greatly improved. Use `#` compression level during training (optional). Will generate statistics more tuned for selected compression level, resulting in a _small_ compression ratio improvement for this level. -* `-B#`: - Split input files into blocks of size # (default: no split) +* `--split=#`: + Split input files into independent chunks of size # (default: no split) * `-M#`, `--memory=#`: Limit the amount of sample data loaded for training (default: 2 GB). Note that the default (2 GB) is also the maximum. @@ -683,8 +683,8 @@ Benchmarking will employ `max(1, min(4, nbCores/4))` worker threads by default i benchmark decompression speed only (requires providing a zstd-compressed content) * `-i#`: minimum evaluation time, in seconds (default: 3s), benchmark mode only -* `-B#`, `--block-size=#`: - cut file(s) into independent chunks of size # (default: no chunking) +* `--split=#`: + split input file(s) into independent chunks of size # (default: no chunking) * `-S`: output one benchmark result per input file (default: consolidated result) * `-D dictionary` diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 83d9b881e..e2771f534 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -235,8 +235,8 @@ static void usageAdvanced(const char* programName) DISPLAYOUT(" --single-thread Share a single thread for I/O and compression (slightly different than `-T1`).\n"); DISPLAYOUT(" --auto-threads={physical|logical}\n"); DISPLAYOUT(" Use physical/logical cores when using `-T0`. [Default: Physical]\n\n"); - DISPLAYOUT(" -B# Set job size to #. [Default: 0 (automatic)]\n"); - DISPLAYOUT(" --rsyncable Compress using a rsync-friendly method (`-B` sets block size). \n"); + DISPLAYOUT(" --jobsize=# Set job size to #. [Default: 0 (automatic)]\n"); + DISPLAYOUT(" --rsyncable Compress using a rsync-friendly method (`--jobsize=#` sets unit size). \n"); DISPLAYOUT("\n"); # endif DISPLAYOUT(" --exclude-compressed Only compress files that are not already compressed.\n\n"); @@ -307,7 +307,7 @@ static void usageAdvanced(const char* programName) DISPLAYOUT(" -b# Perform benchmarking with compression level #. [Default: %d]\n", ZSTDCLI_CLEVEL_DEFAULT); DISPLAYOUT(" -e# Test all compression levels up to #; starting level is `-b#`. [Default: 1]\n"); DISPLAYOUT(" -i# Set the minimum evaluation to time # seconds. [Default: 3]\n"); - DISPLAYOUT(" -B# Cut file into independent chunks of size #. [Default: No chunking]\n"); + DISPLAYOUT(" --split=# Split input into independent chunks of size #. [Default: No chunking]\n"); DISPLAYOUT(" -S Output one benchmark result per input file. [Default: Consolidated result]\n"); DISPLAYOUT(" -D dictionary Benchmark using dictionary \n"); DISPLAYOUT(" --priority=rt Set process priority to real-time.\n"); @@ -773,7 +773,7 @@ static int init_cLevel(void) { } #ifdef ZSTD_MULTITHREAD -static unsigned default_nbThreads(void) { +static int default_nbThreads(void) { const char* const env = getenv(ENV_NBTHREADS); if (env != NULL) { const char* ptr = env; @@ -783,7 +783,7 @@ static unsigned default_nbThreads(void) { DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NBTHREADS, env); return ZSTDCLI_NBTHREADS_DEFAULT; } else if (*ptr == 0) { - return nbThreads; + return (int)nbThreads; } } DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBTHREADS, env); @@ -810,22 +810,31 @@ static unsigned default_nbThreads(void) { CLEAN_RETURN(1); \ } } } -#define NEXT_UINT32(val32) { \ - const char* __nb; \ - NEXT_FIELD(__nb); \ - val32 = readU32FromChar(&__nb); \ - if(*__nb != 0) { \ +#define NEXT_INT32(_vari32) { \ + const char* __nb; \ + NEXT_FIELD(__nb); \ + _vari32 = (int)readU32FromChar(&__nb); \ + if(*__nb != 0) { \ errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \ - } \ + } \ } -#define NEXT_TSIZE(valTsize) { \ - const char* __nb; \ - NEXT_FIELD(__nb); \ - valTsize = readSizeTFromChar(&__nb); \ - if(*__nb != 0) { \ +#define NEXT_UINT32(_varu32) { \ + const char* __nb; \ + NEXT_FIELD(__nb); \ + _varu32 = readU32FromChar(&__nb); \ + if(*__nb != 0) { \ errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \ - } \ + } \ +} + +#define NEXT_TSIZE(_varTsize) { \ + const char* __nb; \ + NEXT_FIELD(__nb); \ + _varTsize = readSizeTFromChar(&__nb); \ + if(*__nb != 0) { \ + errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \ + } \ } typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode; @@ -871,7 +880,7 @@ int main(int argCount, const char* argv[]) int nbWorkers = -1; /* -1 means unset */ double compressibility = -1.0; /* lorem ipsum generator */ unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ - size_t blockSize = 0; + size_t chunkSize = 0; FIO_prefs_t* const prefs = FIO_createPreferences(); FIO_ctx_t* const fCtx = FIO_createContext(); @@ -1069,11 +1078,13 @@ int main(int argCount, const char* argv[]) continue; } #endif - if (longCommandWArg(&argument, "--threads")) { NEXT_UINT32(nbWorkers); continue; } + if (longCommandWArg(&argument, "--threads")) { NEXT_INT32(nbWorkers); continue; } if (longCommandWArg(&argument, "--memlimit")) { NEXT_UINT32(memLimit); continue; } if (longCommandWArg(&argument, "--memory")) { NEXT_UINT32(memLimit); continue; } if (longCommandWArg(&argument, "--memlimit-decompress")) { NEXT_UINT32(memLimit); continue; } - if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(blockSize); continue; } + if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(chunkSize); continue; } /* hidden command, prefer --split below */ + if (longCommandWArg(&argument, "--split")) { NEXT_TSIZE(chunkSize); continue; } + if (longCommandWArg(&argument, "--jobsize")) { NEXT_TSIZE(chunkSize); continue; } /* note: overloaded variable */ if (longCommandWArg(&argument, "--maxdict")) { NEXT_UINT32(maxDictSize); continue; } if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; } if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badUsage(programName, originalArgument); CLEAN_RETURN(1); } ; cType = FIO_zstdCompression; continue; } @@ -1256,10 +1267,10 @@ int main(int argCount, const char* argv[]) bench_nbSeconds = readU32FromChar(&argument); break; - /* cut input into blocks (benchmark only) */ + /* hidden shortcut for --split=# and --jobsize=# */ case 'B': argument++; - blockSize = readU32FromChar(&argument); + chunkSize = readU32FromChar(&argument); break; /* benchmark files separately (hidden option) */ @@ -1273,7 +1284,7 @@ int main(int argCount, const char* argv[]) /* nb of threads (hidden option) */ case 'T': argument++; - nbWorkers = readU32FromChar(&argument); + nbWorkers = (int)readU32FromChar(&argument); break; /* Dictionary Selection level */ @@ -1324,10 +1335,10 @@ int main(int argCount, const char* argv[]) if ((nbWorkers==0) && (!singleThread)) { /* automatically set # workers based on # of reported cpus */ if (defaultLogicalCores) { - nbWorkers = (unsigned)UTIL_countLogicalCores(); + nbWorkers = UTIL_countLogicalCores(); DISPLAYLEVEL(3, "Note: %d logical core(s) detected \n", nbWorkers); } else { - nbWorkers = (unsigned)UTIL_countPhysicalCores(); + nbWorkers = UTIL_countPhysicalCores(); DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers); } } @@ -1404,7 +1415,7 @@ int main(int argCount, const char* argv[]) DISPLAYLEVEL(1, "benchmark mode is only compatible with zstd format \n"); CLEAN_RETURN(1); } - benchParams.blockSize = blockSize; + benchParams.chunkSizeMax = chunkSize; benchParams.targetCBlockSize = targetCBlockSize; benchParams.nbWorkers = (int)nbWorkers; benchParams.realTime = (unsigned)setRealTimePrio; @@ -1448,7 +1459,7 @@ int main(int argCount, const char* argv[]) } #else - (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility; + (void)bench_nbSeconds; (void)chunkSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility; #endif goto _end; } @@ -1464,18 +1475,18 @@ int main(int argCount, const char* argv[]) int const optimize = !coverParams.k || !coverParams.d; coverParams.nbThreads = (unsigned)nbWorkers; coverParams.zParams = zParams; - operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, NULL, &coverParams, NULL, optimize, memLimit); + operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, chunkSize, NULL, &coverParams, NULL, optimize, memLimit); } else if (dict == fastCover) { int const optimize = !fastCoverParams.k || !fastCoverParams.d; fastCoverParams.nbThreads = (unsigned)nbWorkers; fastCoverParams.zParams = zParams; - operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, NULL, NULL, &fastCoverParams, optimize, memLimit); + operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, chunkSize, NULL, NULL, &fastCoverParams, optimize, memLimit); } else { ZDICT_legacy_params_t dictParams; memset(&dictParams, 0, sizeof(dictParams)); dictParams.selectivityLevel = dictSelect; dictParams.zParams = zParams; - operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, &dictParams, NULL, NULL, 0, memLimit); + operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, chunkSize, &dictParams, NULL, NULL, 0, memLimit); } #else (void)dictCLevel; (void)dictSelect; (void)dictID; (void)maxDictSize; /* not used when ZSTD_NODICT set */ @@ -1583,7 +1594,7 @@ int main(int argCount, const char* argv[]) FIO_setCompressionType(prefs, cType); FIO_setContentSize(prefs, contentSize); FIO_setNbWorkers(prefs, (int)nbWorkers); - FIO_setBlockSize(prefs, (int)blockSize); + FIO_setJobSize(prefs, (int)chunkSize); if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog); FIO_setLdmFlag(prefs, (unsigned)ldmFlag); FIO_setLdmHashLog(prefs, (int)g_ldmHashLog); diff --git a/tests/cli-tests/compression/multi-threaded.sh b/tests/cli-tests/compression/multi-threaded.sh index 17a5eb518..ac094129e 100755 --- a/tests/cli-tests/compression/multi-threaded.sh +++ b/tests/cli-tests/compression/multi-threaded.sh @@ -9,6 +9,7 @@ zstd --rsyncable -f file -q ; zstd -t file.zst zstd -T0 -f file -q ; zstd -t file.zst zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst +zstd -T0 --jobsize=1M -f file -q ; zstd -t file.zst # multi-thread decompression warning test zstd -T0 -f file -q ; zstd -t file.zst; zstd -T0 -d file.zst -o file3 diff --git a/tests/cli-tests/compression/multi-threaded.sh.stderr.exact b/tests/cli-tests/compression/multi-threaded.sh.stderr.exact index 11daff6ba..0dcf52ac4 100644 --- a/tests/cli-tests/compression/multi-threaded.sh.stderr.exact +++ b/tests/cli-tests/compression/multi-threaded.sh.stderr.exact @@ -7,5 +7,6 @@ file.zst : 65537 bytes file.zst : 65537 bytes file.zst : 65537 bytes file.zst : 65537 bytes +file.zst : 65537 bytes Warning : decompression does not support multi-threading file.zst : 65537 bytes diff --git a/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact b/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact index d7b3ea020..61f217c34 100644 --- a/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact +++ b/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact @@ -1,5 +1,5 @@ zstd --train -! Warning : nb of samples too low for proper processing ! -! Please provide _one file per sample_. -! Alternatively, split files into fixed-size blocks representative of samples, with -B# +! Warning : nb of samples too low for proper processing ! +! Please provide _one file per sample_. +! Alternatively, split file(s) into fixed-size samples, with --split=# Error 14 : nb of samples too low diff --git a/tests/playTests.sh b/tests/playTests.sh index 5435ff5b3..65aa5c0b1 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -1093,8 +1093,8 @@ println "\n===> dictionary tests " println "- Test high/low compressibility corpus training" datagen -g12M -P90 > tmpCorpusHighCompress datagen -g12M -P5 > tmpCorpusLowCompress -zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress -zstd --train -B2K tmpCorpusLowCompress -o tmpDictLowCompress +zstd --train --split=2K tmpCorpusHighCompress -o tmpDictHighCompress +zstd --train --split=2K tmpCorpusLowCompress -o tmpDictLowCompress rm -f tmpCorpusHighCompress tmpCorpusLowCompress tmpDictHighCompress tmpDictLowCompress println "- Test with raw dict (content only) " datagen > tmpDict @@ -1179,8 +1179,8 @@ rm -f tmp* dictionary println "- Test --memory for dictionary compression" datagen -g12M -P90 > tmpCorpusHighCompress -zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)" -zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr +zstd --train --split=2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)" +zstd --train --split=2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr cat zstTrainWithMemLimitStdErr | $GREP "setting manual memory limit for dictionary training data at 5 MB" cat zstTrainWithMemLimitStdErr | $GREP "Training samples set too large (12 MB); training on 5 MB only..." rm zstTrainWithMemLimitStdErr @@ -1555,7 +1555,7 @@ then roundTripTest -g4M "1 -T0 --auto-threads=logical" roundTripTest -g8M "3 -T2" roundTripTest -g8000K "2 --threads=2" - fileRoundTripTest -g4M "19 -T2 -B1M" + fileRoundTripTest -g4M "19 -T2 --split=1M" println "\n===> zstdmt long distance matching round-trip tests " roundTripTest -g8M "3 --long=24 -T2" @@ -1770,7 +1770,7 @@ then println "\n===> rsyncable mode " roundTripTest -g10M " --rsyncable" - roundTripTest -g10M " --rsyncable -B100K" + roundTripTest -g10M " --rsyncable --split=100K" println "===> test: --rsyncable must fail with --single-thread" zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread" fi