ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4334 from Cyan4973/fix4332

Browse Source 
fix #4332: setting ZSTD_NBTHREADS=0 via environment variable
This commit is contained in:
Yann Collet
2025-03-10 21:12:24 -08:00
committed by GitHub
parent b16d193512 3c4096c83e
commit f9a6031963
5 changed files with 43 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/zstd.h
+1 -1
View File
@@ -111,7 +111,7 @@ extern "C" {
/*------ Version ------*/ /*------ Version ------*/
#define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 5 #define ZSTD_VERSION_MINOR 5
#define ZSTD_VERSION_RELEASE 7 #define ZSTD_VERSION_RELEASE 8
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
/*! ZSTD_versionNumber() : /*! ZSTD_versionNumber() :
programs/zstdcli.c
+33 -35
View File
@@ -44,7 +44,7 @@
#endif #endif
#ifndef ZSTDCLI_NBTHREADS_DEFAULT #ifndef ZSTDCLI_NBTHREADS_DEFAULT
#define ZSTDCLI_NBTHREADS_DEFAULT MAX(1, MIN(4, UTIL_countLogicalCores() / 4)) #define ZSTDCLI_NBTHREADS_DEFAULT (unsigned)(MAX(1, MIN(4, UTIL_countLogicalCores() / 4)))
#endif #endif
@@ -94,6 +94,7 @@ static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
#define DEFAULT_ACCEL 1 #define DEFAULT_ACCEL 1
#define NBWORKERS_AUTOCPU 0
typedef enum { cover, fastCover, legacy } dictType; typedef enum { cover, fastCover, legacy } dictType;
@@ -685,6 +686,12 @@ static void printVersion(void)
DISPLAYOUT("lz4 version %s\n", FIO_lz4Version()); DISPLAYOUT("lz4 version %s\n", FIO_lz4Version());
DISPLAYOUT("lzma version %s\n", FIO_lzmaVersion()); DISPLAYOUT("lzma version %s\n", FIO_lzmaVersion());
#ifdef ZSTD_MULTITHREAD
DISPLAYOUT("supports Multithreading \n");
#else
DISPLAYOUT("single-thread operations only \n");
#endif
/* posix support */ /* posix support */
#ifdef _POSIX_C_SOURCE #ifdef _POSIX_C_SOURCE
DISPLAYOUT("_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE); DISPLAYOUT("_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
@@ -745,7 +752,7 @@ static void printActualCParams(const char* filename, const char* dictFileName, i
/* Environment variables for parameter setting */ /* Environment variables for parameter setting */
#define ENV_CLEVEL "ZSTD_CLEVEL" #define ENV_CLEVEL "ZSTD_CLEVEL"
#define ENV_NBTHREADS "ZSTD_NBTHREADS" /* takes lower precedence than directly specifying -T# in the CLI */ #define ENV_NBWORKERS "ZSTD_NBTHREADS" /* takes lower precedence than directly specifying -T# in the CLI */
/* pick up environment variable */ /* pick up environment variable */
static int init_cLevel(void) { static int init_cLevel(void) {
@@ -775,26 +782,28 @@ static int init_cLevel(void) {
return ZSTDCLI_CLEVEL_DEFAULT; return ZSTDCLI_CLEVEL_DEFAULT;
} }
static unsigned init_nbWorkers(void) {
#ifdef ZSTD_MULTITHREAD #ifdef ZSTD_MULTITHREAD
static int default_nbThreads(void) { const char* const env = getenv(ENV_NBWORKERS);
const char* const env = getenv(ENV_NBTHREADS);
if (env != NULL) { if (env != NULL) {
const char* ptr = env; const char* ptr = env;
if ((*ptr>='0') && (*ptr<='9')) { if ((*ptr>='0') && (*ptr<='9')) {
unsigned nbThreads; unsigned nbThreads;
if (readU32FromCharChecked(&ptr, &nbThreads)) { if (readU32FromCharChecked(&ptr, &nbThreads)) {
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NBTHREADS, env); DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NBWORKERS, env);
return ZSTDCLI_NBTHREADS_DEFAULT; return ZSTDCLI_NBTHREADS_DEFAULT;
} else if (*ptr == 0) { } else if (*ptr == 0) {
return (int)nbThreads; return nbThreads;
} }
} }
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBTHREADS, env); DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBWORKERS, env);
} }
return ZSTDCLI_NBTHREADS_DEFAULT; return ZSTDCLI_NBTHREADS_DEFAULT;
} #else
return 1;
#endif #endif
}
#define NEXT_FIELD(ptr) { \ #define NEXT_FIELD(ptr) { \
if (*argument == '=') { \ if (*argument == '=') { \
@@ -874,13 +883,15 @@ int main(int argCount, const char* argv[])
singleThread = 0, singleThread = 0,
defaultLogicalCores = 0, defaultLogicalCores = 0,
showDefaultCParams = 0, showDefaultCParams = 0,
ultra=0, contentSize = 1,
contentSize=1, removeSrcFile = 0,
removeSrcFile=0; cLevel = init_cLevel(),
ZSTD_ParamSwitch_e mmapDict=ZSTD_ps_auto; ultra = 0,
cLevelLast = MINCLEVEL - 1; /* for benchmark range */
unsigned nbWorkers = init_nbWorkers();
ZSTD_ParamSwitch_e mmapDict = ZSTD_ps_auto;
ZSTD_ParamSwitch_e useRowMatchFinder = ZSTD_ps_auto; ZSTD_ParamSwitch_e useRowMatchFinder = ZSTD_ps_auto;
FIO_compressionType_t cType = FIO_zstdCompression; FIO_compressionType_t cType = FIO_zstdCompression;
int nbWorkers = -1; /* -1 means unset */
double compressibility = -1.0; /* lorem ipsum generator */ double compressibility = -1.0; /* lorem ipsum generator */
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t chunkSize = 0; size_t chunkSize = 0;
@@ -890,8 +901,6 @@ int main(int argCount, const char* argv[])
FIO_progressSetting_e progress = FIO_ps_auto; FIO_progressSetting_e progress = FIO_ps_auto;
zstd_operation_mode operation = zom_compress; zstd_operation_mode operation = zom_compress;
ZSTD_compressionParameters compressionParams; ZSTD_compressionParameters compressionParams;
int cLevel = init_cLevel();
int cLevelLast = MINCLEVEL - 1; /* lower than minimum */
unsigned recursive = 0; unsigned recursive = 0;
unsigned memLimit = 0; unsigned memLimit = 0;
FileNamesTable* filenames = UTIL_allocateFileNamesTable((size_t)argCount); /* argCount >= 1 */ FileNamesTable* filenames = UTIL_allocateFileNamesTable((size_t)argCount); /* argCount >= 1 */
@@ -930,7 +939,7 @@ int main(int argCount, const char* argv[])
programName = lastNameFromPath(programName); programName = lastNameFromPath(programName);
/* preset behaviors */ /* preset behaviors */
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0; if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=NBWORKERS_AUTOCPU, singleThread=0;
if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress; if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */ if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */
if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */ if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */
@@ -938,7 +947,7 @@ int main(int argCount, const char* argv[])
suffix = GZ_EXTENSION; cType = FIO_gzipCompression; removeSrcFile=1; suffix = GZ_EXTENSION; cType = FIO_gzipCompression; removeSrcFile=1;
dictCLevel = cLevel = 6; /* gzip default is -6 */ dictCLevel = cLevel = 6; /* gzip default is -6 */
} }
if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; removeSrcFile=1; } /* behave like gunzip, also supports multiple formats */ if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; removeSrcFile=1; } /* behave like gunzip, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */ if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; cType = FIO_lzmaCompression; removeSrcFile=1; } /* behave like lzma */ if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; cType = FIO_lzmaCompression; removeSrcFile=1; } /* behave like lzma */
if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; cType = FIO_lzmaCompression; removeSrcFile=1; } /* behave like unlzma, also supports multiple formats */ if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; cType = FIO_lzmaCompression; removeSrcFile=1; } /* behave like unlzma, also supports multiple formats */
@@ -1081,7 +1090,7 @@ int main(int argCount, const char* argv[])
continue; continue;
} }
#endif #endif
if (longCommandWArg(&argument, "--threads")) { NEXT_INT32(nbWorkers); continue; } if (longCommandWArg(&argument, "--threads")) { NEXT_UINT32(nbWorkers); continue; }
if (longCommandWArg(&argument, "--memlimit")) { NEXT_UINT32(memLimit); continue; } if (longCommandWArg(&argument, "--memlimit")) { NEXT_UINT32(memLimit); continue; }
if (longCommandWArg(&argument, "--memory")) { NEXT_UINT32(memLimit); continue; } if (longCommandWArg(&argument, "--memory")) { NEXT_UINT32(memLimit); continue; }
if (longCommandWArg(&argument, "--memlimit-decompress")) { NEXT_UINT32(memLimit); continue; } if (longCommandWArg(&argument, "--memlimit-decompress")) { NEXT_UINT32(memLimit); continue; }
@@ -1287,7 +1296,7 @@ int main(int argCount, const char* argv[])
/* nb of threads (hidden option) */ /* nb of threads (hidden option) */
case 'T': case 'T':
argument++; argument++;
nbWorkers = (int)readU32FromChar(&argument); nbWorkers = readU32FromChar(&argument);
break; break;
/* Dictionary Selection level */ /* Dictionary Selection level */
@@ -1332,28 +1341,17 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(3, WELCOME_MESSAGE); DISPLAYLEVEL(3, WELCOME_MESSAGE);
#ifdef ZSTD_MULTITHREAD #ifdef ZSTD_MULTITHREAD
if ((operation==zom_decompress) && (nbWorkers > 1)) { if ((nbWorkers==NBWORKERS_AUTOCPU) && (!singleThread)) {
DISPLAYLEVEL(2, "Warning : decompression does not support multi-threading\n"); /* automatically set # workers based on # of reported cpu cores */
}
if ((nbWorkers==0) && (!singleThread)) {
/* automatically set # workers based on # of reported cpus */
if (defaultLogicalCores) { if (defaultLogicalCores) {
nbWorkers = UTIL_countLogicalCores(); nbWorkers = (unsigned)UTIL_countLogicalCores();
DISPLAYLEVEL(3, "Note: %d logical core(s) detected \n", nbWorkers); DISPLAYLEVEL(3, "Note: %d logical core(s) detected \n", nbWorkers);
} else { } else {
nbWorkers = UTIL_countPhysicalCores(); nbWorkers = (unsigned)UTIL_countPhysicalCores();
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers); DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
} }
} }
/* Resolve to default if nbWorkers is still unset */ if (operation == zom_compress)
if (nbWorkers == -1) {
if (operation == zom_decompress) {
nbWorkers = 1;
} else {
nbWorkers = default_nbThreads();
}
}
if (operation != zom_bench)
DISPLAYLEVEL(4, "Compressing with %u worker threads \n", nbWorkers); DISPLAYLEVEL(4, "Compressing with %u worker threads \n", nbWorkers);
#else #else
(void)singleThread; (void)nbWorkers; (void)defaultLogicalCores; (void)singleThread; (void)nbWorkers; (void)defaultLogicalCores;
tests/cli-tests/compression/multi-threaded.sh
-4
View File
@@ -10,7 +10,3 @@ zstd -T0 -f file -q ; zstd -t file.zst
zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst
zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst
zstd -T0 --jobsize=1M -f file -q ; zstd -t file.zst zstd -T0 --jobsize=1M -f file -q ; zstd -t file.zst
# multi-thread decompression warning test
zstd -T0 -f file -q ; zstd -t file.zst; zstd -T0 -d file.zst -o file3
zstd -T0 -f file -q ; zstd -t file.zst; zstd -T2 -d file.zst -o file4
tests/cli-tests/compression/multi-threaded.sh.stderr.exact
-5
View File
@@ -5,8 +5,3 @@ file.zst : 65537 bytes
file.zst : 65537 bytes file.zst : 65537 bytes
file.zst : 65537 bytes file.zst : 65537 bytes
file.zst : 65537 bytes file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
Warning : decompression does not support multi-threading
file.zst : 65537 bytes
tests/playTests.sh
+9 -13
View File
@@ -1,7 +1,7 @@
#!/bin/sh #!/bin/sh
set -e # exit immediately on error set -e # exit immediately on error
# set -x # print commands before execution (debug) set -x # print commands before execution (debug)
unset ZSTD_CLEVEL unset ZSTD_CLEVEL
unset ZSTD_NBTHREADS unset ZSTD_NBTHREADS
@@ -16,13 +16,7 @@ datagen() {
"$DATAGEN_BIN" "$@" "$DATAGEN_BIN" "$@"
} }
zstd() { alias zstd='$EXE_PREFIX $ZSTD_BIN'
if [ -z "$EXE_PREFIX" ]; then
"$ZSTD_BIN" "$@"
else
"$EXE_PREFIX" "$ZSTD_BIN" "$@"
fi
}
sudoZstd() { sudoZstd() {
if [ -z "$EXE_PREFIX" ]; then if [ -z "$EXE_PREFIX" ]; then
@@ -1563,14 +1557,16 @@ then
println "\n===> zstdmt environment variable tests " println "\n===> zstdmt environment variable tests "
echo "multifoo" >> mt_tmp echo "multifoo" >> mt_tmp
ZSTD_NBTHREADS=-3 zstd -f mt_tmp # negative value, warn and revert to default setting ZSTD_NBTHREADS=-3 zstd -f mt_tmp # negative value, warn and revert to default setting
ZSTD_NBTHREADS='' zstd -f mt_tmp # empty env var, warn and revert to default setting ZSTD_NBTHREADS='' zstd -f mt_tmp # empty env var, warn and revert to default setting
ZSTD_NBTHREADS=- zstd -f mt_tmp # malformed env var, warn and revert to default setting ZSTD_NBTHREADS=- zstd -f mt_tmp # malformed env var, warn and revert to default setting
ZSTD_NBTHREADS=a zstd -f mt_tmp # malformed env var, warn and revert to default setting ZSTD_NBTHREADS=a zstd -f mt_tmp # malformed env var, warn and revert to default setting
ZSTD_NBTHREADS=+a zstd -f mt_tmp # malformed env var, warn and revert to default setting ZSTD_NBTHREADS=+a zstd -f mt_tmp # malformed env var, warn and revert to default setting
ZSTD_NBTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting ZSTD_NBTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting
ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting= ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting=
ZSTD_NBTHREADS=2 zstd -f mt_tmp # correct usage ZSTD_NBTHREADS=2 zstd -f mt_tmp # correct usage
ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single thread ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single worker
ZSTD_NBTHREADS=4 zstd -f mt_tmp -vv 2>&1 | $GREP "4 worker threads" # check message
ZSTD_NBTHREADS=0 zstd -f mt_tmp -vv 2>&1 | $GREP "core(s) detected" # check core count autodetection is triggered
# temporary envvar changes in the above tests would actually persist in macos /bin/sh # temporary envvar changes in the above tests would actually persist in macos /bin/sh
unset ZSTD_NBTHREADS unset ZSTD_NBTHREADS
rm -f mt_tmp* rm -f mt_tmp*