ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'dev' into ahmed_file

Browse Source
This commit is contained in:
Yann Collet
2019-11-26 11:20:26 -08:00
parent 5e657aca90 d6e0a44576
commit c71bd45a3b
52 changed files with 1614 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/.gitignore
+1
View File
@@ -7,6 +7,7 @@ zstd-decompress
zstd-frugal
zstd-small
zstd-nolegacy
zstd-dictBuilder
# Object files
*.o
programs/Makefile
+6 -2
View File
@@ -158,7 +158,7 @@ default: zstd-release
all: zstd
.PHONY: allVariants
allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy
allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy zstd-dictBuilder
$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
@@ -231,6 +231,10 @@ zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c util.c timefn
zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c util.c timefn.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
zstd-dictBuilder: CPPFLAGS += -DZSTD_NOBENCH -DZSTD_NODECOMPRESS
zstd-dictBuilder: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) $(ZDICT_FILES) zstdcli.c util.c timefn.c fileio.c dibio.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstdmt: zstd
ln -sf zstd zstdmt
@@ -245,7 +249,7 @@ clean:
@$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \
*.gcda default.profraw have_zlib$(EXT)
zstd-dictBuilder$(EXT) *.gcda default.profraw have_zlib$(EXT)
@echo Cleaning completed
MD2ROFF = ronn
programs/fileio.c
+13 -7
View File
@@ -502,7 +502,7 @@ static int FIO_remove(const char* path)
#if defined(_WIN32) || defined(WIN32)
/* windows doesn't allow remove read-only files,
* so try to make it writable first */
chmod(path, _S_IWRITE);
UTIL_chmod(path, _S_IWRITE);
#endif
return remove(path);
}
@@ -526,9 +526,7 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
}
if (!UTIL_isRegularFile(srcFileName)
#ifndef _MSC_VER
&& !UTIL_isFIFO(srcFileName)
#endif /* _MSC_VER */
&& !UTIL_isFIFO(srcFileName)
) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
srcFileName);
@@ -609,8 +607,11 @@ FIO_openDstFile(FIO_prefs_t* const prefs,
{ FILE* const f = fopen( dstFileName, "wb" );
if (f == NULL) {
DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
} else if(srcFileName != NULL && strcmp (srcFileName, stdinmark)) {
chmod(dstFileName, 00600);
} else if (srcFileName != NULL
&& strcmp (srcFileName, stdinmark)
&& strcmp(dstFileName, nulmark) ) {
/* reduce rights on newly created dst file while compression is ongoing */
UTIL_chmod(dstFileName, 00600);
}
return f;
}
@@ -1393,7 +1394,7 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
assert(ress.srcFile != NULL);
if (ress.dstFile == NULL) {
closeDstFile = 1;
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName);
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName);
ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName);
if (ress.dstFile==NULL) return 1; /* could not open dstFileName */
/* Must only be added after FIO_openDstFile() succeeds.
@@ -1415,6 +1416,7 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
clearHandler();
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: closing dst: %s \n", dstFileName);
if (fclose(dstFile)) { /* error closing dstFile */
DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
result=1;
@@ -1427,7 +1429,10 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
} else if ( strcmp(dstFileName, stdoutmark)
&& strcmp(dstFileName, nulmark)
&& transfer_permissions) {
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: transfering permissions into dst: %s \n", dstFileName);
UTIL_setFileStat(dstFileName, &statbuf);
} else {
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: do not transfer permissions into dst: %s \n", dstFileName);
}
}
@@ -1462,6 +1467,7 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
int compressionLevel)
{
int result;
DISPLAYLEVEL(6, "FIO_compressFilename_srcFile: %s \n", srcFileName);
/* ensure src is not a directory */
if (UTIL_isDirectory(srcFileName)) {
programs/util.c
+31 -18
View File
@@ -44,7 +44,6 @@ extern "C" {
exit(1); \
} }
/*
* A modified version of realloc().
* If UTIL_realloc() fails the original block is freed.
@@ -57,6 +56,10 @@ UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size)
return NULL;
}
#if defined(_MSC_VER)
#define chmod _chmod
#endif
/*-****************************************
* Console log
@@ -64,9 +67,16 @@ UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size)
int g_utilDisplayLevel;
/*-****************************************
* Public API
******************************************/
/*-*************************************
* Constants
***************************************/
#define LIST_SIZE_INCREASE (8*1024)
#define MAX_FILE_OF_FILE_NAMES_SIZE (1<<20)*50
/*-*************************************
* Functions
***************************************/
int UTIL_fileExist(const char* filename)
{
@@ -98,6 +108,13 @@ int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
return 1;
}
/* like chmod, but avoid changing permission of /dev/null */
int UTIL_chmod(char const* filename, mode_t permissions)
{
if (!strcmp(filename, "/dev/null")) return 0; /* pretend success, but don't change anything */
return chmod(filename, permissions);
}
int UTIL_setFileStat(const char *filename, stat_t *statbuf)
{
int res = 0;
@@ -117,7 +134,7 @@ int UTIL_setFileStat(const char *filename, stat_t *statbuf)
{
/* (atime, mtime) */
struct timespec timebuf[2] = { {0, UTIME_NOW} };
timebuf[1] = statbuf->st_mtim;
timebuf[1].tv_sec = statbuf->st_mtime;
res += utimensat(AT_FDCWD, filename, timebuf, 0);
}
#endif
@@ -126,21 +143,20 @@ int UTIL_setFileStat(const char *filename, stat_t *statbuf)
res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
#endif
res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */
res += UTIL_chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */
errno = 0;
return -res; /* number of errors is returned */
}
U32 UTIL_isDirectory(const char* infilename)
int UTIL_isDirectory(const char* infilename)
{
int r;
stat_t statbuf;
#if defined(_MSC_VER)
r = _stat64(infilename, &statbuf);
int const r = _stat64(infilename, &statbuf);
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
#else
r = stat(infilename, &statbuf);
int const r = stat(infilename, &statbuf);
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
#endif
return 0;
@@ -170,28 +186,25 @@ int UTIL_isSameFile(const char* fName1, const char* fName2)
#endif
}
#ifndef _MSC_VER
/* Using this to distinguish named pipes */
U32 UTIL_isFIFO(const char* infilename)
/* UTIL_isFIFO : distinguish named pipes */
int UTIL_isFIFO(const char* infilename)
{
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#if PLATFORM_POSIX_VERSION >= 200112L
stat_t statbuf;
int r = UTIL_getFileStat(infilename, &statbuf);
int const r = UTIL_getFileStat(infilename, &statbuf);
if (!r && S_ISFIFO(statbuf.st_mode)) return 1;
#endif
(void)infilename;
return 0;
}
#endif
U32 UTIL_isLink(const char* infilename)
int UTIL_isLink(const char* infilename)
{
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#if PLATFORM_POSIX_VERSION >= 200112L
int r;
stat_t statbuf;
r = lstat(infilename, &statbuf);
int const r = lstat(infilename, &statbuf);
if (!r && S_ISLNK(statbuf.st_mode)) return 1;
#endif
(void)infilename;
programs/util.h
+16 -24
View File
@@ -30,12 +30,12 @@ extern "C" {
# include <io.h> /* _chmod */
#else
# include <unistd.h> /* chown, stat */
#if PLATFORM_POSIX_VERSION < 200809L
# include <utime.h> /* utime */
#else
# include <fcntl.h> /* AT_FDCWD */
# include <sys/stat.h> /* utimensat */
#endif
# if PLATFORM_POSIX_VERSION < 200809L
# include <utime.h> /* utime */
# else
# include <fcntl.h> /* AT_FDCWD */
# include <sys/stat.h> /* utimensat */
# endif
#endif
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
#include "mem.h" /* U32, U64 */
@@ -85,12 +85,6 @@ extern "C" {
#endif
/*-*************************************
* Constants
***************************************/
#define LIST_SIZE_INCREASE (8*1024)
#define MAX_FILE_OF_FILE_NAMES_SIZE (1<<20)*50
/*-****************************************
* Compiler specifics
******************************************/
@@ -120,8 +114,8 @@ extern int g_utilDisplayLevel;
* File functions
******************************************/
#if defined(_MSC_VER)
#define chmod _chmod
typedef struct __stat64 stat_t;
typedef int mode_t;
#else
typedef struct stat stat_t;
#endif
@@ -129,22 +123,20 @@ extern int g_utilDisplayLevel;
int UTIL_fileExist(const char* filename);
int UTIL_isRegularFile(const char* infilename);
int UTIL_setFileStat(const char* filename, stat_t* statbuf);
U32 UTIL_isDirectory(const char* infilename);
int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
int UTIL_isDirectory(const char* infilename);
int UTIL_isSameFile(const char* file1, const char* file2);
int UTIL_compareStr(const void *p1, const void *p2);
int UTIL_isCompressedFile(const char* infilename, const char *extensionList[]);
const char* UTIL_getFileExtension(const char* infilename);
int UTIL_isLink(const char* infilename);
int UTIL_isFIFO(const char* infilename);
#ifndef _MSC_VER
U32 UTIL_isFIFO(const char* infilename);
#endif
U32 UTIL_isLink(const char* infilename);
#define UTIL_FILESIZE_UNKNOWN ((U64)(-1))
U64 UTIL_getFileSize(const char* infilename);
U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles);
U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles);
int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
int UTIL_setFileStat(const char* filename, stat_t* statbuf);
int UTIL_chmod(char const* filename, mode_t permissions); /*< like chmod, but avoid changing permission of /dev/null */
int UTIL_compareStr(const void *p1, const void *p2);
const char* UTIL_getFileExtension(const char* infilename);
/*-****************************************
programs/zstd.1.md
+7 -6
View File
@@ -405,7 +405,7 @@ The list of available _options_:
Bigger hash tables cause less collisions which usually makes compression
faster, but requires more memory during compression.
The minimum _hlog_ is 6 (64 B) and the maximum is 26 (128 MiB).
The minimum _hlog_ is 6 (64 B) and the maximum is 30 (1 GiB).
- `chainLog`=_clog_, `clog`=_clog_:
Specify the maximum number of bits for a hash chain or a binary tree.
@@ -416,7 +416,8 @@ The list of available _options_:
compression.
This option is ignored for the ZSTD_fast strategy.
The minimum _clog_ is 6 (64 B) and the maximum is 28 (256 MiB).
The minimum _clog_ is 6 (64 B) and the maximum is 29 (524 Mib) on 32-bit platforms
and 30 (1 Gib) on 64-bit platforms.
- `searchLog`=_slog_, `slog`=_slog_:
Specify the maximum number of searches in a hash chain or a binary tree
@@ -425,7 +426,7 @@ The list of available _options_:
More searches increases the chance to find a match which usually increases
compression ratio but decreases compression speed.
The minimum _slog_ is 1 and the maximum is 26.
The minimum _slog_ is 1 and the maximum is 'windowLog' - 1.
- `minMatch`=_mml_, `mml`=_mml_:
Specify the minimum searched length of a match in a hash table.
@@ -450,7 +451,7 @@ The list of available _options_:
For all other strategies, this field has no impact.
The minimum _tlen_ is 0 and the maximum is 999.
The minimum _tlen_ is 0 and the maximum is 128 Kib.
- `overlapLog`=_ovlog_, `ovlog`=_ovlog_:
Determine `overlapSize`, amount of data reloaded from previous job.
@@ -473,7 +474,7 @@ The list of available _options_:
Bigger hash tables usually improve compression ratio at the expense of more
memory during compression and a decrease in compression speed.
The minimum _lhlog_ is 6 and the maximum is 26 (default: 20).
The minimum _lhlog_ is 6 and the maximum is 30 (default: 20).
- `ldmMinMatch`=_lmml_, `lmml`=_lmml_:
Specify the minimum searched length of a match for long distance matching.
@@ -493,7 +494,7 @@ The list of available _options_:
Larger bucket sizes improve collision resolution but decrease compression
speed.
The minimum _lblog_ is 0 and the maximum is 8 (default: 3).
The minimum _lblog_ is 1 and the maximum is 8 (default: 3).
- `ldmHashRateLog`=_lhrlog_, `lhrlog`=_lhrlog_:
Specify the frequency of inserting entries into the long distance matching
programs/zstdcli.c
+3 -5
View File
@@ -1005,12 +1005,10 @@ int main(int argCount, const char* argv[])
unsigned u, fileNamesNb;
unsigned const nbFilenames = (unsigned)filenames->tableSize;
for (u=0, fileNamesNb=0; u<nbFilenames; u++) {
if (UTIL_isLink(filenames->fileNames[u])
#ifndef _MSC_VER
&& !UTIL_isFIFO(filenames->fileNames[u])
#endif /* _MSC_VER */
if ( UTIL_isLink(filenames->fileNames[u])
&& !UTIL_isFIFO(filenames->fileNames[u])
) {
DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenames->fileNames[u]);
DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring \n", filenames->fileNames[u]);
} else {
filenames->fileNames[fileNamesNb++] = filenames->fileNames[u];
} }