ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/Cyan4973/dev' into dev

Browse Source 
# Conflicts:
#	lib/common/util.h
#	programs/paramgrill.c
#	visual/2013/fullbench/fullbench.vcxproj.filters
#	visual/2013/fuzzer/fuzzer.vcxproj.filters
This commit is contained in:
inikep
2016-05-10 13:20:01 +02:00
parent 349fcf7ea2 d693117113
commit ed9a08538c
28 changed files with 392 additions and 1442 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+59 -72
View File
@@ -116,6 +116,11 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
static const unsigned refreshRate = 150;
static clock_t g_time = 0;
static unsigned FIO_GetMilliSpan(clock_t nPrevious)
{
clock_t const nCurrent = clock();
return (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC);
}
/*-*************************************
@@ -147,14 +152,6 @@ void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; }
/*-*************************************
* Functions
***************************************/
static unsigned FIO_GetMilliSpan(clock_t nPrevious)
{
clock_t nCurrent = clock();
unsigned nSpan = (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC);
return nSpan;
}
static FILE* FIO_openSrcFile(const char* srcFileName)
{
FILE* f;
@@ -297,21 +294,20 @@ static int FIO_compressFilename_internal(cRess_t ress,
const char* dstFileName, const char* srcFileName,
int cLevel)
{
FILE* srcFile = ress.srcFile;
FILE* dstFile = ress.dstFile;
FILE* const srcFile = ress.srcFile;
FILE* const dstFile = ress.dstFile;
U64 readsize = 0;
U64 compressedfilesize = 0;
size_t dictSize = ress.dictBufferSize;
size_t sizeCheck;
ZSTD_parameters params;
U64 const fileSize = UTIL_getFileSize(srcFileName);
/* init */
params.cParams = ZSTD_getCParams(cLevel, fileSize, dictSize);
params.fParams.contentSizeFlag = 1;
if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog;
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); }
{ ZSTD_parameters params;
params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize);
params.fParams.contentSizeFlag = 1;
if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog;
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); }
}
/* Main compression loop */
readsize = 0;
@@ -325,16 +321,15 @@ static int FIO_compressFilename_internal(cRess_t ress,
{ /* Compress using buffered streaming */
size_t usedInSize = inSize;
size_t cSize = ress.dstBufferSize;
size_t result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize);
if (ZBUFF_isError(result))
EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result));
{ size_t const result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize);
if (ZBUFF_isError(result)) EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result)); }
if (inSize != usedInSize)
/* inBuff should be entirely consumed since buffer sizes are recommended ones */
EXM_THROW(24, "Compression error : input block not fully consumed");
/* Write cBlock */
sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); }
compressedfilesize += cSize;
}
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(readsize>>20), (double)compressedfilesize/readsize*100);
@@ -345,8 +340,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
size_t const result = ZBUFF_compressEnd(ress.ctx, ress.dstBuffer, &cSize);
if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName);
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
compressedfilesize += cSize;
}
@@ -376,7 +371,6 @@ static int FIO_compressFilename_srcFile(cRess_t ress,
result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel);
/* clean */
fclose(ress.srcFile);
return result;
}
@@ -409,7 +403,7 @@ static int FIO_compressFilename_extRess(cRess_t ress,
int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
const char* dictFileName, int compressionLevel)
{
clock_t start, end;
clock_t start;
cRess_t ress;
int issueWithSrcFile = 0;
@@ -417,19 +411,13 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
start = clock();
ress = FIO_createCResources(dictFileName);
/* Compress File */
issueWithSrcFile += FIO_compressFilename_extRess(ress, dstFileName, srcFileName, compressionLevel);
/* Free resources */
FIO_freeCResources(ress);
/* Final Status */
end = clock();
{
double seconds = (double)(end - start) / CLOCKS_PER_SEC;
{ double seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
}
return issueWithSrcFile;
}
@@ -438,11 +426,10 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
const char* suffix,
const char* dictFileName, int compressionLevel)
{
unsigned u;
int missed_files = 0;
char* dstFileName = (char*)malloc(FNSPACE);
char* dstFileName = (char*)malloc(FNSPACE);
size_t dfnSize = FNSPACE;
const size_t suffixSize = suffix ? strlen(suffix) : 0;
size_t const suffixSize = suffix ? strlen(suffix) : 0;
cRess_t ress;
/* init */
@@ -450,12 +437,14 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
/* loop on each file */
if (!strcmp(suffix, stdoutmark)) {
unsigned u;
ress.dstFile = stdout;
for (u=0; u<nbFiles; u++)
missed_files += FIO_compressFilename_srcFile(ress, stdoutmark,
inFileNamesTable[u], compressionLevel);
if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close %s", stdoutmark);
} else {
unsigned u;
for (u=0; u<nbFiles; u++) {
size_t ifnSize = strlen(inFileNamesTable[u]);
if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
@@ -472,7 +461,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
return missed_files;
}
#endif // #ifndef ZSTD_NOCOMPRESS
#endif /* #ifndef ZSTD_NOCOMPRESS */
@@ -515,7 +504,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
static void FIO_freeDResources(dRess_t ress)
{
size_t errorCode = ZBUFF_freeDCtx(ress.dctx);
size_t const errorCode = ZBUFF_freeDCtx(ress.dctx);
if (ZBUFF_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode));
free(ress.srcBuffer);
free(ress.dstBuffer);
@@ -536,16 +525,15 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
/* Complete Header loading */
{ size_t const toLoad = ZSTD_frameHeaderSize_max - alreadyLoaded; /* assumption : alreadyLoaded <= ZSTD_frameHeaderSize_max */
size_t const checkSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
if (checkSize != toLoad) EXM_THROW(32, "Read error"); /* assumption : srcSize >= ZSTD_frameHeaderSize_max */
size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput); /* can be <= toLoad (null string) */
readSize = alreadyLoaded + loadedSize;
}
readSize = ZSTD_frameHeaderSize_max;
/* Main decompression Loop */
while (1) {
/* Decode */
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
size_t toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
size_t const toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
readSize -= inSize;
@@ -577,8 +565,8 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
{
unsigned long long filesize = 0;
FILE* dstFile = ress.dstFile;
FILE* srcFile = FIO_openSrcFile(srcFileName);
FILE* const dstFile = ress.dstFile;
FILE* const srcFile = FIO_openSrcFile(srcFileName);
if (srcFile==0) return 1;
/* for each frame */
@@ -588,16 +576,17 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile);
if (sizeCheck==0) break; /* no more input */
if (sizeCheck != toRead) EXM_THROW(31, "zstd: %s read error : cannot read header", srcFileName);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(MEM_readLE32(ress.srcBuffer))) {
filesize += FIO_decompressLegacyFrame(dstFile, srcFile, MEM_readLE32(ress.srcBuffer));
continue;
}
#endif /* ZSTD_LEGACY_SUPPORT */
if (MEM_readLE32(ress.srcBuffer) != ZSTD_MAGICNUMBER) {
DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName);
return 1;
}
{ U32 const magic = MEM_readLE32(ress.srcBuffer);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
if (ZSTD_isLegacy(magic)) {
filesize += FIO_decompressLegacyFrame(dstFile, srcFile, ress.dictBuffer, ress.dictBufferSize, magic);
continue;
}
#endif
if (magic != ZSTD_MAGICNUMBER) {
DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName);
return 1;
} }
filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead);
}
@@ -624,9 +613,7 @@ static int FIO_decompressFile_extRess(dRess_t ress,
if (ress.dstFile==0) return 1;
result = FIO_decompressSrcFile(ress, srcFileName);
if (result != 0) {
remove(dstFileName);
}
if (result != 0) remove(dstFileName);
if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName);
return result;
@@ -651,28 +638,27 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
const char* suffix,
const char* dictFileName)
{
unsigned u;
int skippedFiles = 0;
int missingFiles = 0;
char* dstFileName = (char*)malloc(FNSPACE);
size_t dfnSize = FNSPACE;
const size_t suffixSize = suffix ? strlen(suffix) : 0;
dRess_t ress;
if (dstFileName==NULL) EXM_THROW(70, "not enough memory for dstFileName");
ress = FIO_createDResources(dictFileName);
dRess_t ress = FIO_createDResources(dictFileName);
if (!strcmp(suffix, stdoutmark) || !strcmp(suffix, nulmark)) {
unsigned u;
ress.dstFile = FIO_openDstFile(suffix);
if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", suffix);
for (u=0; u<nbFiles; u++)
missingFiles += FIO_decompressSrcFile(ress, srcNamesTable[u]);
if (fclose(ress.dstFile)) EXM_THROW(39, "Write error : cannot properly close %s", stdoutmark);
} else {
size_t const suffixSize = suffix ? strlen(suffix) : 0;
size_t dfnSize = FNSPACE;
unsigned u;
char* dstFileName = (char*)malloc(FNSPACE);
if (dstFileName==NULL) EXM_THROW(70, "not enough memory for dstFileName");
for (u=0; u<nbFiles; u++) { /* create dstFileName */
const char* srcFileName = srcNamesTable[u];
size_t sfnSize = strlen(srcFileName);
const char* suffixPtr = srcFileName + sfnSize - suffixSize;
const char* const srcFileName = srcNamesTable[u];
size_t const sfnSize = strlen(srcFileName);
const char* const suffixPtr = srcFileName + sfnSize - suffixSize;
if (dfnSize+suffixSize <= sfnSize+1) {
free(dstFileName);
dfnSize = sfnSize + 20;
@@ -688,11 +674,12 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
dstFileName[sfnSize-suffixSize] = '\0';
missingFiles += FIO_decompressFile_extRess(ress, dstFileName, srcFileName);
} }
}
free(dstFileName);
}
FIO_freeDResources(ress);
free(dstFileName);
return missingFiles + skippedFiles;
}
#endif // #ifndef ZSTD_NODECOMPRESS
#endif /* #ifndef ZSTD_NODECOMPRESS */