From 042ba122ae753dd770db44d8730a6a2dc3bf98df Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 23 Mar 2017 11:13:52 -0700 Subject: [PATCH 1/4] Change g_displayLevel to int and fix DISPLAYUPDATE flush --- lib/dictBuilder/cover.c | 2 -- lib/dictBuilder/zdict.c | 2 +- programs/bench.c | 4 +-- programs/dibio.c | 4 +-- programs/fileio.c | 4 +-- programs/zstdcli.c | 48 +++++++++++++++---------------- tests/fuzzer.c | 2 +- tests/zbufftest.c | 2 +- tests/zstreamtest.c | 2 +- zlibWrapper/examples/zwrapbench.c | 4 +-- 10 files changed, 36 insertions(+), 38 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 3a7b9f39f..1db42f95b 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -59,8 +59,6 @@ static int g_displayLevel = 2; if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \ g_time = clock(); \ DISPLAY(__VA_ARGS__); \ - if (displayLevel >= 4) \ - fflush(stdout); \ } \ } #define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 0757dbbbb..c84195791 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -480,7 +480,7 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, # define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \ if (ZDICT_clockSpan(displayClock) > refreshRate) \ { displayClock = clock(); DISPLAY(__VA_ARGS__); \ - if (notificationLevel>=4) fflush(stdout); } } + if (notificationLevel>=4) fflush(stderr); } } /* init */ DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ diff --git a/programs/bench.c b/programs/bench.c index 2dd1cfb0f..2643dd45a 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -70,12 +70,12 @@ static U32 g_compressibilityDefault = 50; ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ +static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(stderr); } } static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; static clock_t g_time = 0; diff --git a/programs/dibio.c b/programs/dibio.c index 5ef202c8a..b7ed280ea 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -53,12 +53,12 @@ static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_ ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */ +static int g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */ #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ if ((DIB_clockSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(stderr); } } static const clock_t refreshRate = CLOCKS_PER_SEC * 2 / 10; static clock_t g_time = 0; diff --git a/programs/fileio.c b/programs/fileio.c index e6481f1fa..572e9c9ac 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -81,13 +81,13 @@ ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } -static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ +static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } #define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \ if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } } + if (g_displayLevel>=4) fflush(stderr); } } } static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; static clock_t g_time = 0; diff --git a/programs/zstdcli.c b/programs/zstdcli.c index cd53d4c36..29bc76377 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -74,10 +74,10 @@ static U32 g_overlapLog = OVERLAP_LOG_DEFAULT; /*-************************************ * Display Macros **************************************/ -#define DISPLAY(...) fprintf(displayOut, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static FILE* displayOut; -static unsigned displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */ +#define DISPLAY(...) fprintf(g_displayOut, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } +static int g_displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */ +static FILE* g_displayOut; /*-************************************ @@ -167,7 +167,7 @@ static int usage_advanced(const char* programName) static int badusage(const char* programName) { DISPLAYLEVEL(1, "Incorrect parameters\n"); - if (displayLevel >= 2) usage(programName); + if (g_displayLevel >= 2) usage(programName); return 1; } @@ -316,7 +316,7 @@ int main(int argCount, const char* argv[]) (void)memLimit; /* not used when ZSTD_NODECOMPRESS set */ if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); } filenameTable[0] = stdinmark; - displayOut = stderr; + g_displayOut = stderr; /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ { size_t pos; for (pos = (int)strlen(programName); pos > 0; pos--) { if (programName[pos] == '/') { pos++; break; } } @@ -325,10 +325,10 @@ int main(int argCount, const char* argv[]) /* preset behaviors */ if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress; - if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; } + if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } if (!strcmp(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); } /* behave like gzip */ if (!strcmp(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip */ - if (!strcmp(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; } /* behave like gzcat */ + if (!strcmp(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat */ if (!strcmp(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like lzma */ if (!strcmp(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */ memset(&compressionParams, 0, sizeof(compressionParams)); @@ -344,7 +344,7 @@ int main(int argCount, const char* argv[]) if (!filenameIdx) { filenameIdx=1, filenameTable[0]=stdinmark; outFileName=stdoutmark; - displayLevel-=(displayLevel==2); + g_displayLevel-=(g_displayLevel==2); continue; } } @@ -358,11 +358,11 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; } if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; } if (!strcmp(argument, "--force")) { FIO_overwriteMode(); continue; } - if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); } - if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); } - if (!strcmp(argument, "--verbose")) { displayLevel++; continue; } - if (!strcmp(argument, "--quiet")) { displayLevel--; continue; } - if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; } + if (!strcmp(argument, "--version")) { g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); } + if (!strcmp(argument, "--help")) { g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); } + if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; } + if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; } + if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; } if (!strcmp(argument, "--ultra")) { ultra=1; continue; } if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; } if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; } @@ -424,9 +424,9 @@ int main(int argCount, const char* argv[]) switch(argument[0]) { /* Display help */ - case 'V': displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); /* Version Only */ + case 'V': g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); /* Version Only */ case 'H': - case 'h': displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); + case 'h': g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); /* Compress */ case 'z': operation=zom_compress; argument++; break; @@ -448,10 +448,10 @@ int main(int argCount, const char* argv[]) case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break; /* Verbose mode */ - case 'v': displayLevel++; argument++; break; + case 'v': g_displayLevel++; argument++; break; /* Quiet mode */ - case 'q': displayLevel--; argument++; break; + case 'q': g_displayLevel--; argument++; break; /* keep source file (default); for gzip/xz compatibility */ case 'k': FIO_setRemoveSrcFile(0); argument++; break; @@ -597,7 +597,7 @@ int main(int argCount, const char* argv[]) /* Check if benchmark is selected */ if (operation==zom_bench) { #ifndef ZSTD_NOBENCH - BMK_setNotificationLevel(displayLevel); + BMK_setNotificationLevel(g_displayLevel); BMK_setBlockSize(blockSize); BMK_setNbThreads(nbThreads); BMK_setNbSeconds(bench_nbSeconds); @@ -613,7 +613,7 @@ int main(int argCount, const char* argv[]) if (cover) { coverParams.nbThreads = nbThreads; coverParams.compressionLevel = dictCLevel; - coverParams.notificationLevel = displayLevel; + coverParams.notificationLevel = g_displayLevel; coverParams.dictID = dictID; DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, NULL, &coverParams, cover - 1); } else { @@ -621,7 +621,7 @@ int main(int argCount, const char* argv[]) memset(&dictParams, 0, sizeof(dictParams)); dictParams.compressionLevel = dictCLevel; dictParams.selectivityLevel = dictSelect; - dictParams.notificationLevel = displayLevel; + dictParams.notificationLevel = g_displayLevel; dictParams.dictID = dictID; DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, &dictParams, NULL, 0); } @@ -654,11 +654,11 @@ int main(int argCount, const char* argv[]) #endif /* No status message in pipe mode (stdin - stdout) or multi-files mode */ - if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1; - if ((filenameIdx>1) & (displayLevel==2)) displayLevel=1; + if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1; + if ((filenameIdx>1) & (g_displayLevel==2)) g_displayLevel=1; /* IO Stream/File */ - FIO_setNotificationLevel(displayLevel); + FIO_setNotificationLevel(g_displayLevel); if (operation==zom_compress) { #ifndef ZSTD_NOCOMPRESS FIO_setNbThreads(nbThreads); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 1d8be17af..111ca8243 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -57,7 +57,7 @@ static U32 g_displayLevel = 2; #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ if ((FUZ_clockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(stderr); } } static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6; static clock_t g_displayClock = 0; diff --git a/tests/zbufftest.c b/tests/zbufftest.c index 14b739233..601aa808d 100644 --- a/tests/zbufftest.c +++ b/tests/zbufftest.c @@ -60,7 +60,7 @@ static U32 g_displayLevel = 2; #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(stderr); } } static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100; static clock_t g_displayClock = 0; diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index a03ee9b1f..370859ca9 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -59,7 +59,7 @@ static U32 g_displayLevel = 2; #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(stderr); } } static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6; static clock_t g_displayClock = 0; diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index 23c3ca4da..951c315b8 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -73,13 +73,13 @@ static U32 g_compressibilityDefault = 50; #define DEFAULT_DISPLAY_LEVEL 2 #define DISPLAY(...) fprintf(displayOut, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static U32 g_displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ +static int g_displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ static FILE* displayOut; #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(displayOut); } } static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; static clock_t g_time = 0; From 680e4e0953bf7c895165e41098a37c3279870c1f Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 23 Mar 2017 11:52:09 -0700 Subject: [PATCH 2/4] Ignore symbolic links unless --force specified --- programs/util.h | 41 ++++++++++++++++++++++++------- programs/zstd.1 | 4 +-- programs/zstd.1.md | 2 +- programs/zstdcli.c | 21 +++++++++++++--- tests/playTests.sh | 13 ++++++++++ zlibWrapper/examples/zwrapbench.c | 2 +- 6 files changed, 66 insertions(+), 17 deletions(-) diff --git a/programs/util.h b/programs/util.h index 59e19d027..e0b705117 100644 --- a/programs/util.h +++ b/programs/util.h @@ -1,6 +1,6 @@ /** * util.h - utility functions - * + * * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. * All rights reserved. * @@ -228,6 +228,20 @@ UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) return 0; } +UTIL_STATIC U32 UTIL_isLink(const char* infilename) +{ +#if defined(_WIN32) + /* no symlinks on windows */ + (void)infilename; +#else + int r; + stat_t statbuf; + r = lstat(infilename, &statbuf); + if (!r && S_ISLNK(statbuf.st_mode)) return 1; +#endif + return 0; +} + UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { @@ -271,11 +285,10 @@ UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size) return NULL; } - #ifdef _WIN32 # define UTIL_HAS_CREATEFILELIST -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) +UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks) { char* path; int dirLength, fnameLength, pathLength, nbFiles = 0; @@ -311,7 +324,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ if (strcmp (cFile.cFileName, "..") == 0 || strcmp (cFile.cFileName, ".") == 0) continue; - nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd); /* Recursively call "UTIL_prepareFileList" with the new path. */ + nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */ if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } } else if ((cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED)) { @@ -339,7 +352,11 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ # include /* opendir, readdir */ # include /* strerror, memcpy */ -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) +static int g_displayLevel; +#define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define UTIL_DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } } + +UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks) { DIR *dir; struct dirent *entry; @@ -360,13 +377,19 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ path = (char*) malloc(dirLength + fnameLength + 2); if (!path) { closedir(dir); return 0; } memcpy(path, dirName, dirLength); + path[dirLength] = '/'; memcpy(path+dirLength+1, entry->d_name, fnameLength); pathLength = dirLength+1+fnameLength; path[pathLength] = 0; + if (!followLinks && UTIL_isLink(path)) { + UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path); + continue; + } + if (UTIL_isDirectory(path)) { - nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd); /* Recursively call "UTIL_prepareFileList" with the new path. */ + nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */ if (*bufStart == NULL) { free(path); closedir(dir); return 0; } } else { if (*bufStart + *pos + pathLength >= *bufEnd) { @@ -396,7 +419,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ #else -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) +UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks) { (void)bufStart; (void)bufEnd; (void)pos; fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); @@ -411,7 +434,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer) * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called. */ -UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb) +UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb, int followLinks) { size_t pos; unsigned i, nbFiles; @@ -436,7 +459,7 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i nbFiles++; } } else { - nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend); + nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks); if (buf == NULL) return NULL; } } diff --git a/programs/zstd.1 b/programs/zstd.1 index f79b3ce16..02423358d 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -1,5 +1,5 @@ . -.TH "ZSTD" "1" "March 2017" "zstd 1.1.4" "User Commands" +.TH "ZSTD" "1" "March 2017" "zstd 1.1.5" "User Commands" . .SH "NAME" \fBzstd\fR \- zstd, unzstd, zstdcat \- Compress or decompress \.zst files @@ -110,7 +110,7 @@ save result into \fBfile\fR (only possible with a single INPUT\-FILE) . .TP \fB\-f\fR, \fB\-\-force\fR -overwrite output without prompting +overwrite output without prompting, and (de)compress symbolic links . .TP \fB\-c\fR, \fB\-\-stdout\fR diff --git a/programs/zstd.1.md b/programs/zstd.1.md index bb5b22664..c9ff33276 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -108,7 +108,7 @@ the last one takes effect. * `-o file`: save result into `file` (only possible with a single INPUT-FILE) * `-f`, `--force`: - overwrite output without prompting + overwrite output without prompting, and (de)compress symbolic links * `-c`, `--stdout`: force write to standard output, even if it is the console * `--[no-]sparse`: diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 29bc76377..38b8394a0 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -99,7 +99,7 @@ static int usage(const char* programName) #endif DISPLAY( " -D file: use `file` as Dictionary \n"); DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n"); - DISPLAY( " -f : overwrite output without prompting \n"); + DISPLAY( " -f : overwrite output without prompting and (de)compress links \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); DISPLAY( " -k : preserve source file(s) (default) \n"); DISPLAY( " -h/-H : display help/long help and exit\n"); @@ -270,6 +270,7 @@ int main(int argCount, const char* argv[]) { int argNb, forceStdout=0, + followLinks=0, main_pause=0, nextEntryIsDictionary=0, operationResult=0, @@ -357,7 +358,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; } if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; } if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; } - if (!strcmp(argument, "--force")) { FIO_overwriteMode(); continue; } + if (!strcmp(argument, "--force")) { FIO_overwriteMode(); forceStdout=1; followLinks=1; continue; } if (!strcmp(argument, "--version")) { g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); } if (!strcmp(argument, "--help")) { g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); } if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; } @@ -445,7 +446,7 @@ int main(int argCount, const char* argv[]) case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break; /* Overwrite */ - case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break; + case 'f': FIO_overwriteMode(); forceStdout=1; followLinks=1; argument++; break; /* Verbose mode */ case 'v': g_displayLevel++; argument++; break; @@ -581,9 +582,21 @@ int main(int argCount, const char* argv[]) DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION); #endif + + if (!followLinks) { + unsigned u; + for (u=0, fileNamesNb=0; u /dev/full" $ECHO foo | $ZSTD > /dev/full && die "write error not detected!" $ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full" $ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" + +$ECHO "\n**** symbolic link test **** " + +rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst +$ECHO "hello world" > hello.tmp +ln -s hello.tmp world.tmp +$ZSTD world.tmp hello.tmp +ls hello.tmp.zst || die "regular file should have been compressed!" +ls world.tmp.zst && die "symbolic link should not have been compressed!" +$ZSTD world.tmp hello.tmp -f +ls world.tmp.zst || die "symbol link should have been compressed with --force" +rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst + fi diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index 951c315b8..1c3391e90 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -982,7 +982,7 @@ int main(int argCount, char** argv) #ifdef UTIL_HAS_CREATEFILELIST if (recursive) { - fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb); + fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, 1); if (fileNamesTable) { unsigned u; for (u=0; u Date: Thu, 23 Mar 2017 12:09:35 -0700 Subject: [PATCH 3/4] Fix displayLevel redefinition issues --- programs/util.h | 8 ++++---- programs/zstdcli.c | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/programs/util.h b/programs/util.h index e0b705117..9d23cbb8e 100644 --- a/programs/util.h +++ b/programs/util.h @@ -285,6 +285,10 @@ UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size) return NULL; } +static int g_utilDisplayLevel; +#define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } } + #ifdef _WIN32 # define UTIL_HAS_CREATEFILELIST @@ -352,10 +356,6 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ # include /* opendir, readdir */ # include /* strerror, memcpy */ -static int g_displayLevel; -#define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define UTIL_DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } } - UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks) { DIR *dir; diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 38b8394a0..4d7fbb357 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -583,6 +583,7 @@ int main(int argCount, const char* argv[]) #endif + g_utilDisplayLevel = g_displayLevel; if (!followLinks) { unsigned u; for (u=0, fileNamesNb=0; u Date: Thu, 23 Mar 2017 12:41:51 -0700 Subject: [PATCH 4/4] Add symbolic link checking to pzstd --- contrib/pzstd/Options.cpp | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp index a0d969393..1f53f2bff 100644 --- a/contrib/pzstd/Options.cpp +++ b/contrib/pzstd/Options.cpp @@ -91,7 +91,7 @@ void usage() { std::fprintf(stderr, " -# : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel); std::fprintf(stderr, " -d, --decompress : decompression\n"); std::fprintf(stderr, " -o file : result stored into `file` (only if 1 input file)\n"); - std::fprintf(stderr, " -f, --force : overwrite output without prompting\n"); + std::fprintf(stderr, " -f, --force : overwrite output without prompting, (de)compress links\n"); std::fprintf(stderr, " --rm : remove source file(s) after successful (de)compression\n"); std::fprintf(stderr, " -k, --keep : preserve source file(s) (default)\n"); std::fprintf(stderr, " -h, --help : display help and exit\n"); @@ -121,6 +121,7 @@ Options::Status Options::parse(int argc, const char **argv) { bool recursive = false; bool ultra = false; bool forceStdout = false; + bool followLinks = false; // Local copy of input files, which are pointers into argv. std::vector localInputFiles; for (int i = 1; i < argc; ++i) { @@ -255,6 +256,7 @@ Options::Status Options::parse(int argc, const char **argv) { case 'f': overwrite = true; forceStdout = true; + followLinks = true; break; case 't': test = true; @@ -328,13 +330,29 @@ Options::Status Options::parse(int argc, const char **argv) { } } + g_utilDisplayLevel = verbosity; + // Remove local input files that are symbolic links + if (!followLinks) { + std::remove_if(localInputFiles.begin(), localInputFiles.end(), + [&](const char *path) { + bool isLink = UTIL_isLink(path); + if (isLink && verbosity >= 2) { + std::fprintf( + stderr, + "Warning : %s is symbolic link, ignoring\n", + path); + } + return isLink; + }); + } + // Translate input files/directories into files to (de)compress if (recursive) { char *scratchBuffer = nullptr; unsigned numFiles = 0; const char **files = UTIL_createFileList(localInputFiles.data(), localInputFiles.size(), - &scratchBuffer, &numFiles); + &scratchBuffer, &numFiles, followLinks); if (files == nullptr) { std::fprintf(stderr, "Error traversing directories\n"); return Status::Failure;