From e5811e5520cc3f03f279a26a19e9be3618172f2e Mon Sep 17 00:00:00 2001 From: shakeelrao Date: Sat, 23 Mar 2019 19:04:56 -0700 Subject: [PATCH 1/7] Extract file comparison into utility func --- programs/fileio.c | 26 ++++---------------------- programs/util.c | 17 +++++++++++++++++ programs/util.h | 1 + 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 8a5715113..ffa43ab22 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -515,28 +515,10 @@ static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, return stdout; } - /* ensure dst is not the same file as src */ - if (srcFileName != NULL) { -#ifdef _MSC_VER - /* note : Visual does not support file identification by inode. - * The following work-around is limited to detecting exact name repetition only, - * aka `filename` is considered different from `subdir/../filename` */ - if (!strcmp(srcFileName, dstFileName)) { - DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n"); - return NULL; - } -#else - stat_t srcStat; - stat_t dstStat; - if (UTIL_getFileStat(srcFileName, &srcStat) - && UTIL_getFileStat(dstFileName, &dstStat)) { - if (srcStat.st_dev == dstStat.st_dev - && srcStat.st_ino == dstStat.st_ino) { - DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n"); - return NULL; - } - } -#endif + /* check if src file is the same as dst */ + if (srcFileName != NULL && UTIL_isSameFile(srcFileName, dstFileName)) { + DISPLAYLEVEL(1, "zstd: Refusing to open an output file which will overwrite the input file \n"); + return NULL; } if (prefs->sparseFileSupport == 1) { diff --git a/programs/util.c b/programs/util.c index d64660635..622e5025f 100644 --- a/programs/util.c +++ b/programs/util.c @@ -87,6 +87,23 @@ U32 UTIL_isDirectory(const char* infilename) return 0; } +int UTIL_isSameFile(const char* file1, const char* file2) +{ +#if defined(_MSC_VER) + /* note : Visual does not support file identification by inode. + * The following work-around is limited to detecting exact name repetition only, + * aka `filename` is considered different from `subdir/../filename` */ + return !strcmp(file1, file2); +#else + stat_t file1Stat; + stat_t file2Stat; + return UTIL_getFileStat(file1, &file1Stat) + && UTIL_getFileStat(file2, &file2Stat) + && (file1Stat.st_dev == file2Stat.st_dev) + && (file1Stat.st_ino == file2Stat.st_ino); +#endif +} + U32 UTIL_isLink(const char* infilename) { /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ diff --git a/programs/util.h b/programs/util.h index f78bcbe1b..eee7ebfc3 100644 --- a/programs/util.h +++ b/programs/util.h @@ -174,6 +174,7 @@ int UTIL_isRegularFile(const char* infilename); int UTIL_setFileStat(const char* filename, stat_t* statbuf); U32 UTIL_isDirectory(const char* infilename); int UTIL_getFileStat(const char* infilename, stat_t* statbuf); +int UTIL_isSameFile(const char* file1, const char* file2); U32 UTIL_isLink(const char* infilename); #define UTIL_FILESIZE_UNKNOWN ((U64)(-1)) From 1290933d192c4a42db5ad5b90978f53a8a8a8d26 Mon Sep 17 00:00:00 2001 From: shakeelrao Date: Sat, 23 Mar 2019 21:53:13 -0700 Subject: [PATCH 2/7] Implement file check --- programs/fileio.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index ffa43ab22..5672de02d 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -515,7 +515,7 @@ static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, return stdout; } - /* check if src file is the same as dst */ + /* ensure dst is not the same as src */ if (srcFileName != NULL && UTIL_isSameFile(srcFileName, dstFileName)) { DISPLAYLEVEL(1, "zstd: Refusing to open an output file which will overwrite the input file \n"); return NULL; @@ -610,6 +610,7 @@ typedef struct { size_t srcBufferSize; void* dstBuffer; size_t dstBufferSize; + const char* dictFileName; ZSTD_CStream* cctx; } cRess_t; @@ -637,6 +638,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */ if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "allocation error : can't create dictBuffer"); + ress.dictFileName = dictFileName; if (prefs->adaptiveMode && !prefs->ldmFlag && !comprParams.windowLog) comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT; @@ -1290,12 +1292,18 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, { int result; - /* File check */ + /* ensure src is not a directory */ if (UTIL_isDirectory(srcFileName)) { DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); return 1; } + /* ensure src is not the same as dict */ + if (UTIL_isSameFile(srcFileName, ress.dictFileName)) { + DISPLAYLEVEL(1, "zstd: Refusing to use %s as an input file and dictionary \n", srcFileName); + return 1; + } + ress.srcFile = FIO_openSrcFile(srcFileName); if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ From 8ea219d8c663264e57f4882ce1d9401dab0769c9 Mon Sep 17 00:00:00 2001 From: shakeelrao Date: Sat, 23 Mar 2019 21:59:30 -0700 Subject: [PATCH 3/7] Modify error msg --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 5672de02d..7da7baaff 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1300,7 +1300,7 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, /* ensure src is not the same as dict */ if (UTIL_isSameFile(srcFileName, ress.dictFileName)) { - DISPLAYLEVEL(1, "zstd: Refusing to use %s as an input file and dictionary \n", srcFileName); + DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName); return 1; } From 5333e41ab38668732b6403cf5a170e6c92d5b45c Mon Sep 17 00:00:00 2001 From: shakeelrao Date: Sun, 24 Mar 2019 00:23:50 -0700 Subject: [PATCH 4/7] Add NULL check for dict --- programs/fileio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 7da7baaff..cbd7b92bc 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1298,8 +1298,8 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, return 1; } - /* ensure src is not the same as dict */ - if (UTIL_isSameFile(srcFileName, ress.dictFileName)) { + /* ensure src is not the same as dict (if present) */ + if (ress.dictFileName != NULL && UTIL_isSameFile(srcFileName, ress.dictFileName)) { DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName); return 1; } From 2b4491d81a524ab7e5307ffb430d2f7d0c4b1e9f Mon Sep 17 00:00:00 2001 From: shakeelrao Date: Sun, 24 Mar 2019 00:47:13 -0700 Subject: [PATCH 5/7] Add CLI test to validate error --- tests/playTests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/playTests.sh b/tests/playTests.sh index 493542960..137f1a441 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -410,6 +410,8 @@ $ECHO "- Create first dictionary " TESTFILE=../programs/zstdcli.c $ZSTD --train *.c ../programs/*.c -o tmpDict cp $TESTFILE tmp +$ECHO "- Compress dictionary with itself" +$ZSTD -f tmpDict -D tmpDict | grep "cannot use" && die "expected error : cannot compress dictionary with itself" $ECHO "- Dictionary compression roundtrip" $ZSTD -f tmp -D tmpDict $ZSTD -d tmp.zst -D tmpDict -fo result From b25d7eacf27bce19ea823e1baba898f3553e3047 Mon Sep 17 00:00:00 2001 From: shakeelrao Date: Sun, 24 Mar 2019 03:40:03 -0700 Subject: [PATCH 6/7] Rename test --- tests/playTests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index 137f1a441..f50bc9030 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -410,8 +410,8 @@ $ECHO "- Create first dictionary " TESTFILE=../programs/zstdcli.c $ZSTD --train *.c ../programs/*.c -o tmpDict cp $TESTFILE tmp -$ECHO "- Compress dictionary with itself" -$ZSTD -f tmpDict -D tmpDict | grep "cannot use" && die "expected error : cannot compress dictionary with itself" +$ECHO "- Test dictionary compression with tmpDict as an input file and dictionary" +$ZSTD -f tmpDict -D tmpDict&& die "compression error not detected!" $ECHO "- Dictionary compression roundtrip" $ZSTD -f tmp -D tmpDict $ZSTD -d tmp.zst -D tmpDict -fo result From 44f77b5c71d5dae08f3c50e495184d422135997c Mon Sep 17 00:00:00 2001 From: shakeelrao Date: Sun, 24 Mar 2019 03:42:11 -0700 Subject: [PATCH 7/7] Add whitespace to test case --- tests/playTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index f50bc9030..d22f617e4 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -411,7 +411,7 @@ TESTFILE=../programs/zstdcli.c $ZSTD --train *.c ../programs/*.c -o tmpDict cp $TESTFILE tmp $ECHO "- Test dictionary compression with tmpDict as an input file and dictionary" -$ZSTD -f tmpDict -D tmpDict&& die "compression error not detected!" +$ZSTD -f tmpDict -D tmpDict && die "compression error not detected!" $ECHO "- Dictionary compression roundtrip" $ZSTD -f tmp -D tmpDict $ZSTD -d tmp.zst -D tmpDict -fo result