From 1c5d45fc11529e72c78a8108bddeb794cca905b7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 2 Sep 2025 15:46:51 -0700 Subject: [PATCH 1/5] fixed -T# documentation in zstd -H provide the local value for default nbThreads which is dynamic and depends on local nb of cores. --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 124ea47aa..142fc4c53 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -233,7 +233,7 @@ static void usageAdvanced(const char* programName) DISPLAYOUT(" --patch-from=REF Use REF as the reference point for Zstandard's diff engine. \n"); DISPLAYOUT(" --patch-apply Equivalent for `-d --patch-from` \n\n"); # ifdef ZSTD_MULTITHREAD - DISPLAYOUT(" -T# Spawn # compression threads. [Default: 1; pass 0 for core count.]\n"); + DISPLAYOUT(" -T# Spawn # compression threads. [Default: %i; pass 0 for core count.]\n", ZSTDCLI_NBTHREADS_DEFAULT); DISPLAYOUT(" --single-thread Share a single thread for I/O and compression (slightly different than `-T1`).\n"); DISPLAYOUT(" --auto-threads={physical|logical}\n"); DISPLAYOUT(" Use physical/logical cores when using `-T0`. [Default: Physical]\n\n"); From 6551db345922c9b40aa48a8a57c9bfb48fd1855b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 2 Sep 2025 15:53:45 -0700 Subject: [PATCH 2/5] specify nb of threads used during benchmarking used to require `-v` (verbose) modifier --- programs/zstdcli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 142fc4c53..e44384ea1 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1435,7 +1435,7 @@ int main(int argCount, const char* argv[]) if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); if (cLevelLast < cLevel) cLevelLast = cLevel; - DISPLAYLEVEL(3, "Benchmarking "); + DISPLAYLEVEL(2, "Benchmarking "); if (filenames->tableSize > 1) DISPLAYLEVEL(3, "%u files ", (unsigned)filenames->tableSize); if (cLevelLast > cLevel) { @@ -1443,7 +1443,7 @@ int main(int argCount, const char* argv[]) } else { DISPLAYLEVEL(3, "at level %d ", cLevel); } - DISPLAYLEVEL(3, "using %i threads \n", nbWorkers); + DISPLAYLEVEL(2, "using %i threads \n", nbWorkers); if (filenames->tableSize > 0) { if(separateFiles) { unsigned i; From c41fc1aa01a4bbc4ccd6877f4d198e0c76eb122d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 2 Sep 2025 16:05:35 -0700 Subject: [PATCH 3/5] only display nbThread Msg in nbThreads > 1 --- programs/zstdcli.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index e44384ea1..cb87604b1 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1410,6 +1410,7 @@ int main(int argCount, const char* argv[]) /* Check if benchmark is selected */ if (operation==zom_bench) { #ifndef ZSTD_NOBENCH + int const benchThreadsMsgLevel = 2 + (nbWorkers <= 1); if (cType != FIO_zstdCompression) { DISPLAYLEVEL(1, "benchmark mode is only compatible with zstd format \n"); CLEAN_RETURN(1); @@ -1435,7 +1436,7 @@ int main(int argCount, const char* argv[]) if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); if (cLevelLast < cLevel) cLevelLast = cLevel; - DISPLAYLEVEL(2, "Benchmarking "); + DISPLAYLEVEL(benchThreadsMsgLevel, "Benchmarking "); if (filenames->tableSize > 1) DISPLAYLEVEL(3, "%u files ", (unsigned)filenames->tableSize); if (cLevelLast > cLevel) { @@ -1443,7 +1444,7 @@ int main(int argCount, const char* argv[]) } else { DISPLAYLEVEL(3, "at level %d ", cLevel); } - DISPLAYLEVEL(2, "using %i threads \n", nbWorkers); + DISPLAYLEVEL(benchThreadsMsgLevel, "using %i threads \n", nbWorkers); if (filenames->tableSize > 0) { if(separateFiles) { unsigned i; From 725a152c7bf91244b0e16eabc312762a39dad583 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 2 Sep 2025 16:21:48 -0700 Subject: [PATCH 4/5] benchmark uses 1 thread by default --- programs/zstdcli.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index cb87604b1..696e4a1ba 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -47,7 +47,7 @@ #define ZSTDCLI_NBTHREADS_DEFAULT (unsigned)(MAX(1, MIN(4, UTIL_countLogicalCores() / 4))) #endif - +static unsigned init_nbWorkers(unsigned defaultNbWorkers); /*-************************************ * Constants @@ -95,6 +95,7 @@ static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT; #define DEFAULT_ACCEL 1 #define NBWORKERS_AUTOCPU 0 +#define NBWORKERS_UNSET UINT_MAX typedef enum { cover, fastCover, legacy } dictType; @@ -233,7 +234,7 @@ static void usageAdvanced(const char* programName) DISPLAYOUT(" --patch-from=REF Use REF as the reference point for Zstandard's diff engine. \n"); DISPLAYOUT(" --patch-apply Equivalent for `-d --patch-from` \n\n"); # ifdef ZSTD_MULTITHREAD - DISPLAYOUT(" -T# Spawn # compression threads. [Default: %i; pass 0 for core count.]\n", ZSTDCLI_NBTHREADS_DEFAULT); + DISPLAYOUT(" -T# Spawn # compression threads. [Default: %u; pass 0 for core count.]\n", init_nbWorkers(ZSTDCLI_NBTHREADS_DEFAULT)); DISPLAYOUT(" --single-thread Share a single thread for I/O and compression (slightly different than `-T1`).\n"); DISPLAYOUT(" --auto-threads={physical|logical}\n"); DISPLAYOUT(" Use physical/logical cores when using `-T0`. [Default: Physical]\n\n"); @@ -783,7 +784,7 @@ static int init_cLevel(void) { return ZSTDCLI_CLEVEL_DEFAULT; } -static unsigned init_nbWorkers(void) { +static unsigned init_nbWorkers(unsigned defaultNbWorkers) { #ifdef ZSTD_MULTITHREAD const char* const env = getenv(ENV_NBWORKERS); if (env != NULL) { @@ -792,7 +793,7 @@ static unsigned init_nbWorkers(void) { unsigned nbThreads; if (readU32FromCharChecked(&ptr, &nbThreads)) { DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NBWORKERS, env); - return ZSTDCLI_NBTHREADS_DEFAULT; + return defaultNbWorkers; } else if (*ptr == 0) { return nbThreads; } @@ -800,7 +801,7 @@ static unsigned init_nbWorkers(void) { DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBWORKERS, env); } - return ZSTDCLI_NBTHREADS_DEFAULT; + return defaultNbWorkers; #else return 1; #endif @@ -881,7 +882,7 @@ int main(int argCount, const char* argv[]) ultra = 0, cLevelLast = MINCLEVEL - 1, /* for benchmark range */ setThreads_non1 = 0; - unsigned nbWorkers = init_nbWorkers(); + unsigned nbWorkers = init_nbWorkers(NBWORKERS_UNSET); ZSTD_ParamSwitch_e mmapDict = ZSTD_ps_auto; ZSTD_ParamSwitch_e useRowMatchFinder = ZSTD_ps_auto; FIO_compressionType_t cType = FIO_zstdCompression; @@ -1410,7 +1411,9 @@ int main(int argCount, const char* argv[]) /* Check if benchmark is selected */ if (operation==zom_bench) { #ifndef ZSTD_NOBENCH - int const benchThreadsMsgLevel = 2 + (nbWorkers <= 1); + int benchThreadsMsgLevel = 2; + if (nbWorkers == NBWORKERS_UNSET) nbWorkers = 1; + if (nbWorkers <= 1) benchThreadsMsgLevel = 3; if (cType != FIO_zstdCompression) { DISPLAYLEVEL(1, "benchmark mode is only compatible with zstd format \n"); CLEAN_RETURN(1); @@ -1471,6 +1474,7 @@ int main(int argCount, const char* argv[]) zParams.compressionLevel = dictCLevel; zParams.notificationLevel = (unsigned)g_displayLevel; zParams.dictID = dictID; + if (nbWorkers == NBWORKERS_UNSET) nbWorkers = init_nbWorkers(ZSTDCLI_NBTHREADS_DEFAULT); if (dict == cover) { int const optimize = !coverParams.k || !coverParams.d; coverParams.nbThreads = (unsigned)nbWorkers; @@ -1593,6 +1597,7 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NOCOMPRESS FIO_setCompressionType(prefs, cType); FIO_setContentSize(prefs, contentSize); + if (nbWorkers == NBWORKERS_UNSET) nbWorkers = init_nbWorkers(ZSTDCLI_NBTHREADS_DEFAULT); FIO_setNbWorkers(prefs, (int)nbWorkers); FIO_setJobSize(prefs, (int)chunkSize); if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog); From 249e4a07d53415a8161ce5ad65a725ffd99060ac Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 2 Sep 2025 16:36:44 -0700 Subject: [PATCH 5/5] fixed minor unused variable warning in certain compilation modes --- programs/zstdcli.c | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 696e4a1ba..01760ff8c 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -803,6 +803,7 @@ static unsigned init_nbWorkers(unsigned defaultNbWorkers) { return defaultNbWorkers; #else + (void)defaultNbWorkers; return 1; #endif }