ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cli : added ability to compress multiple files into stdout (-c)

Browse Source
This commit is contained in:
Yann Collet
2016-02-12 15:56:46 +01:00
parent 492a9bb88e
commit f062436fa6
6 changed files with 144 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+103 -57
View File
@@ -186,54 +186,73 @@ static U64 FIO_getFileSize(const char* infilename)
}
static FILE* FIO_openSrcFile(const char* srcFileName)
{
FILE* f;
if (!strcmp (srcFileName, stdinmark)) {
DISPLAYLEVEL(4,"Using stdin for input\n");
f = stdin;
SET_BINARY_MODE(stdin);
} else {
f = fopen(srcFileName, "rb");
}
if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: No such file\n", srcFileName);
return f;
}
static FILE* FIO_openDstFile(const char* dstFileName)
{
FILE* f;
if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output\n");
f = stdout;
SET_BINARY_MODE(stdout);
} else {
if (!g_overwrite) { /* Check if destination file already exists */
f = fopen( dstFileName, "rb" );
if (f != 0) { /* dest file exists, prompt for overwrite authorization */
fclose(f);
if (g_displayLevel <= 1) {
/* No interaction possible */
DISPLAY("zstd: %s already exists; not overwritten \n", dstFileName);
return 0;
}
DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName);
{
int ch = getchar();
if ((ch!='Y') && (ch!='y')) {
DISPLAY(" not overwritten \n");
return 0;
}
while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */
} } }
f = fopen( dstFileName, "wb" );
}
return f;
}
static int FIO_getFiles(FILE** fileOutPtr, FILE** fileInPtr,
const char* dstFileName, const char* srcFileName)
{
if (!strcmp (srcFileName, stdinmark)) {
DISPLAYLEVEL(4,"Using stdin for input\n");
*fileInPtr = stdin;
SET_BINARY_MODE(stdin);
} else {
*fileInPtr = fopen(srcFileName, "rb");
}
*fileInPtr = FIO_openSrcFile(srcFileName);
if ( *fileInPtr==0 ) {
DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", srcFileName);
return 1;
}
if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output\n");
*fileOutPtr = stdout;
SET_BINARY_MODE(stdout);
} else {
if (!g_overwrite) { /* Check if destination file already exists */
*fileOutPtr = fopen( dstFileName, "rb" );
if (*fileOutPtr != 0) { /* dest file exists, prompt for overwrite authorization */
fclose(*fileOutPtr);
DISPLAY("Warning : %s already exists \n", dstFileName);
if ((g_displayLevel <= 1) || (*fileInPtr == stdin)) {
/* No interaction possible */
DISPLAY("Operation aborted : %s already exists \n", dstFileName);
return 1;
}
DISPLAY("Overwrite ? (y/N) : ");
{
int ch = getchar();
if ((ch!='Y') && (ch!='y')) {
DISPLAY("No. Operation aborted : %s already exists \n", dstFileName);
return 1;
}
while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */
} } }
*fileOutPtr = fopen( dstFileName, "wb" );
}
if (*fileOutPtr==0) EXM_THROW(13, "Pb opening %s", dstFileName);
*fileOutPtr = FIO_openDstFile(dstFileName);
if (*fileOutPtr==0) return 1;
return 0;
}
/*!FIO_loadFile
* creates a buffer, pointed by *bufferPtr,
* loads "filename" content into it
@@ -280,6 +299,7 @@ typedef struct {
void* dictBuffer;
size_t dictBufferSize;
ZBUFF_CCtx* ctx;
FILE* dstFile;
} cRess_t;
static cRess_t FIO_createCResources(const char* dictFileName)
@@ -313,24 +333,25 @@ static void FIO_freeCResources(cRess_t ress)
}
/*!
* FIO_compressFilename_extRess() :
* @result : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName
/*! FIO_compressFilename_internal() :
* same as FIO_compressFilename_extRess(), with ress.desFile already opened
* @return : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName
*/
static int FIO_compressFilename_extRess(cRess_t ress,
const char* dstFileName, const char* srcFileName,
int cLevel)
static int FIO_compressFilename_internal(cRess_t ress,
const char* dstFileName, const char* srcFileName,
int cLevel)
{
FILE* srcFile;
FILE* dstFile;
FILE* dstFile = ress.dstFile;
U64 filesize = 0;
U64 compressedfilesize = 0;
size_t dictSize = ress.dictBufferSize;
size_t sizeCheck, errorCode;
/* File check */
if (FIO_getFiles(&dstFile, &srcFile, dstFileName, srcFileName)) return 1;
srcFile = FIO_openSrcFile(srcFileName);
if (!srcFile) return 1; /* srcFile could not be opened */
/* init */
filesize = FIO_getFileSize(srcFileName) + dictSize;
@@ -346,8 +367,7 @@ static int FIO_compressFilename_extRess(cRess_t ress,
filesize += inSize;
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
{
/* Compress (buffered streaming ensures appropriate formatting) */
{ /* Compress using buffered streaming */
size_t usedInSize = inSize;
size_t cSize = ress.dstBufferSize;
size_t result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize);
@@ -362,7 +382,6 @@ static int FIO_compressFilename_extRess(cRess_t ress,
if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
compressedfilesize += cSize;
}
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(filesize>>20), (double)compressedfilesize/filesize*100);
}
@@ -384,12 +403,31 @@ static int FIO_compressFilename_extRess(cRess_t ress,
/* clean */
fclose(srcFile);
if (fclose(dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
return 0;
}
/*! FIO_compressFilename_extRess() :
* @return : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName
*/
static int FIO_compressFilename_extRess(cRess_t ress,
const char* dstFileName, const char* srcFileName,
int cLevel)
{
int result;
ress.dstFile = FIO_openDstFile(dstFileName);
if (ress.dstFile==0) return 1;
result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel);
if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
return result;
}
int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
const char* dictFileName, int compressionLevel)
{
@@ -427,20 +465,28 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
int missed_files = 0;
char* dstFileName = (char*)malloc(FNSPACE);
size_t dfnSize = FNSPACE;
const size_t suffixSize = strlen(suffix);
const size_t suffixSize = suffix ? strlen(suffix) : 0;
cRess_t ress;
/* init */
ress = FIO_createCResources(dictFileName);
/* loop on each file */
for (u=0; u<nbFiles; u++) {
size_t ifnSize = strlen(inFileNamesTable[u]);
if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
strcpy(dstFileName, inFileNamesTable[u]);
strcat(dstFileName, suffix);
missed_files += FIO_compressFilename_extRess(ress, dstFileName, inFileNamesTable[u], compressionLevel);
if (suffix) {
for (u=0; u<nbFiles; u++) {
size_t ifnSize = strlen(inFileNamesTable[u]);
if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
strcpy(dstFileName, inFileNamesTable[u]);
strcat(dstFileName, suffix);
missed_files += FIO_compressFilename_extRess(ress, dstFileName,
inFileNamesTable[u], compressionLevel);
}
} else {
ress.dstFile = stdout;
for (u=0; u<nbFiles; u++)
missed_files += FIO_compressFilename_internal(ress, stdoutmark,
inFileNamesTable[u], compressionLevel);
if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close %s", stdoutmark);
}
/* Close & Free */
programs/fileio.h
+2 -1
View File
@@ -64,13 +64,14 @@ int FIO_decompressFilename (const char* outfilename, const char* infilename, con
* Multiple File functions
***************************************/
/** FIO_compressMultipleFilenames() :
if `suffix == NULL`, output is stdout.
@return : nb of missing files */
int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
const char* suffix,
const char* dictFileName, int compressionLevel);
/** FIO_decompressMultipleFilenames() :
@result : nb of missing or skipped files */
@return : nb of missing or skipped files */
int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
const char* suffix,
const char* dictFileName);
programs/playTests.sh
+13 -6
View File
@@ -22,7 +22,8 @@ roundTripTest() {
[ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
printf "\n**** frame concatenation **** "
echo "\n**** frame concatenation **** "
echo "hello " > hello.tmp
echo "world!" > world.tmp
@@ -37,7 +38,8 @@ rm ./*.tmp ./*.zstd
echo frame concatenation test completed
echo "**** flush write error test **** "
echo "\n**** flush write error test **** "
echo "echo foo | $ZSTD > /dev/full"
echo foo | $ZSTD > /dev/full && die "write error not detected!"
@@ -45,27 +47,32 @@ echo "echo foo | $ZSTD | $ZSTD -d > /dev/full"
echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
echo "*** dictionary tests *** "
echo "\n**** dictionary tests **** "
./datagen > tmpDict
./datagen -g1M | md5sum > tmp1
./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dv | md5sum > tmp2
./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2
diff -q tmp1 tmp2
echo "*** multiple files tests *** "
echo "\n**** multiple files tests **** "
./datagen -s1 > tmp1 2> /dev/null
./datagen -s2 -g100K > tmp2 2> /dev/null
./datagen -s3 -g1M > tmp3 2> /dev/null
$ZSTD -f tmp*
echo "compress tmp* : "
ls -ls tmp*
rm tmp1 tmp2 tmp3
echo "decompress tmp* : "
$ZSTD -df *.zst
ls -ls tmp*
echo "compress tmp* into stdout : "
$ZSTD -c tmp1 tmp2 tmp3 > tmpall
ls -ls tmp*
$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
rm tmp*
echo "**** zstd round-trip tests **** "
echo "\n**** zstd round-trip tests **** "
roundTripTest
roundTripTest '' 6
programs/zstdcli.c
+9 -9
View File
@@ -379,28 +379,28 @@ int main(int argCount, const char** argv)
if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) return badusage(programName);
if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) return badusage(programName);
/* No warning message in pipe mode (stdin + stdout) or multiple mode */
if (!strcmp(filenameTable[0], stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
if ((filenameIdx>1) && (displayLevel==2)) displayLevel=1;
/* user-selected output filename only possible with a single file */
if ((outFileName) && (filenameIdx>1)) {
/* user-selected output filename, only possible with a single file */
if (outFileName && strcmp(outFileName,stdoutmark) && (filenameIdx>1)) {
DISPLAY("Too many files (%u) on the command line. \n", filenameIdx);
return filenameIdx;
}
/* No warning message in pipe mode (stdin + stdout) or multiple mode */
if (!strcmp(filenameTable[0], stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
if ((filenameIdx>1) && (displayLevel==2)) displayLevel=1;
/* IO Stream/File */
FIO_setNotificationLevel(displayLevel);
if (decode) {
if (outFileName)
if (filenameIdx==1)
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
else
operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, ZSTD_EXTENSION, dictFileName);
} else { /* compression */
if (outFileName)
if (filenameIdx==1)
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel);
else
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, ZSTD_EXTENSION, dictFileName, cLevel);
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, forceStdout ? NULL : ZSTD_EXTENSION, dictFileName, cLevel);
}
_end: