ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HC modes can be selected from command line

Browse Source
This commit is contained in:
Yann Collet
2015-10-29 16:49:43 +01:00
parent 76c5c2a4a0
commit 2acb5d3d48
12 changed files with 426 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/Makefile
+1 -1
View File
@@ -58,7 +58,7 @@ default: zstd
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 paramgrill datagen
zstd : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
zstd: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
programs/bench.c
+7 -10
View File
@@ -229,9 +229,9 @@ typedef struct
size_t resSize;
} blockParam_t;
typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel);
typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
static size_t local_compress_fast (void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel)
static size_t local_compress_fast (void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
{
(void)compressionLevel;
return ZSTD_compress(dst, maxDstSize, src, srcSize);
@@ -323,10 +323,10 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
for (blockNb=0; blockNb<nbBlocks; blockNb++)
cSize += blockTable[blockNb].cSize;
if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops;
ratio = (double)cSize / (double)srcSize*100.;
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.);
ratio = (double)srcSize / (double)cSize;
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.3f),%7.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.);
#if 0
#if 1
/* Decompression */
memset(resultBuffer, 0xD6, srcSize);
@@ -343,7 +343,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
milliTime = BMK_GetMilliSpan(milliTime);
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.3f),%7.1f MB/s ,%7.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
/* CRC Checking */
crcCheck = XXH64(resultBuffer, srcSize, 0);
@@ -367,10 +367,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
if (crcOrig == crcCheck)
{
if (ratio<100.)
DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
else
DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
DISPLAY("%-16.16s : %9i -> %9i (%5.3f),%7.1f MB/s ,%7.1f MB/s \n", fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
}
}
programs/fileio.c
+68 -8
View File
@@ -66,6 +66,7 @@
#include "mem.h"
#include "fileio.h"
#include "zstd_static.h"
#include "zstdhc_static.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
# include "zstd_v01.h" /* legacy */
@@ -213,8 +214,47 @@ static void FIO_getFileHandles(FILE** pfinput, FILE** pfoutput, const char* inpu
if ( *pfoutput==0) EXM_THROW(13, "Pb opening dst : %s", output_filename);
}
typedef void* (*FIO_createC) (void);
static void* local_ZSTD_createCCtx(void) { return (void*) ZSTD_createCCtx(); }
static void* local_ZSTD_HC_createCCtx(void) { return (void*) ZSTD_HC_createCCtx(); }
unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, unsigned cLevel)
typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel);
static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel)
{
(void)cLevel;
return ZSTD_compressBegin((ZSTD_CCtx*)ctx, dst, maxDstSize);
}
static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel)
{
return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel);
}
typedef size_t (*FIO_continueC) (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
static size_t local_ZSTD_compressContinue (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
return ZSTD_compressContinue((ZSTD_CCtx*)ctx, dst, maxDstSize, src, srcSize);
}
static size_t local_ZSTD_HC_compressContinue (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
return ZSTD_HC_compressContinue((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, src, srcSize);
}
typedef size_t (*FIO_endC) (void* ctx, void* dst, size_t maxDstSize);
static size_t local_ZSTD_compressEnd (void* ctx, void* dst, size_t maxDstSize)
{
return ZSTD_compressEnd((ZSTD_CCtx*)ctx, dst, maxDstSize);
}
static size_t local_ZSTD_HC_compressEnd (void* ctx, void* dst, size_t maxDstSize)
{
return ZSTD_HC_compressEnd((ZSTD_HC_CCtx*)ctx, dst, maxDstSize);
}
typedef void (*FIO_freeC) (void* ctx);
static void local_ZSTD_freeCCtx(void* ctx) { ZSTD_freeCCtx((ZSTD_CCtx*)ctx); }
static void local_ZSTD_HC_freeCCtx(void* ctx) { ZSTD_HC_freeCCtx((ZSTD_HC_CCtx*)ctx); }
unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, int cLevel)
{
U64 filesize = 0;
U64 compressedfilesize = 0;
@@ -228,14 +268,34 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
FILE* finput;
FILE* foutput;
size_t sizeCheck, cSize;
ZSTD_CCtx* ctx = ZSTD_createCCtx();
void* ctx;
FIO_createC createC=NULL;
FIO_initC initC=NULL;
FIO_continueC continueC = NULL;
FIO_endC endC = NULL;
FIO_freeC freeC = NULL;
/* Init */
(void)cLevel;
if (cLevel <= 1)
{
createC = local_ZSTD_createCCtx;
initC = local_ZSTD_compressBegin;
continueC = local_ZSTD_compressContinue;
endC = local_ZSTD_compressEnd;
freeC = local_ZSTD_freeCCtx;
}
else
{
createC = local_ZSTD_HC_createCCtx;
initC = local_ZSTD_HC_compressBegin;
continueC = local_ZSTD_HC_compressContinue;
endC = local_ZSTD_HC_compressEnd;
freeC = local_ZSTD_HC_freeCCtx;
}
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
/* Allocate Memory */
ctx = createC();
inBuff = (BYTE*)malloc(inBuffSize);
outBuff = (BYTE*)malloc(outBuffSize);
if (!inBuff || !outBuff || !ctx) EXM_THROW(21, "Allocation error : not enough memory");
@@ -243,7 +303,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
inEnd = inBuff + inBuffSize;
/* Write Frame Header */
cSize = ZSTD_compressBegin(ctx, outBuff, outBuffSize);
cSize = initC(ctx, outBuff, outBuffSize, cLevel);
if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header");
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
@@ -263,7 +323,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
/* Compress Block */
cSize = ZSTD_compressContinue(ctx, outBuff, outBuffSize, inSlot, inSize);
cSize = continueC(ctx, outBuff, outBuffSize, inSlot, inSize);
if (ZSTD_isError(cSize))
EXM_THROW(24, "Compression error : %s ", ZSTD_getErrorName(cSize));
@@ -277,7 +337,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
}
/* End of Frame */
cSize = ZSTD_compressEnd(ctx, outBuff, outBuffSize);
cSize = endC(ctx, outBuff, outBuffSize);
if (ZSTD_isError(cSize)) EXM_THROW(26, "Compression error : cannot create frame end");
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
@@ -292,7 +352,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
/* clean */
free(inBuff);
free(outBuff);
ZSTD_freeCCtx(ctx);
freeC(ctx);
fclose(finput);
if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename);
programs/fileio.h
+1 -1
View File
@@ -52,7 +52,7 @@ void FIO_setNotificationLevel(unsigned level);
/* *************************************
* Stream/File functions
***************************************/
unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename, unsigned cLevel);
unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename, int compressionLevel);
unsigned long long FIO_decompressFilename (const char* outfilename, const char* infilename);
/**
FIO_compressFilename :
programs/paramgrill.c
+1 -1
View File
@@ -446,7 +446,7 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi
int cLevel;
fprintf(f, "\n /* Selected configurations : */ \n");
fprintf(f, "#define ZSTD_HC_MAX_CLEVEL 20 \n");
fprintf(f, "#define ZSTD_HC_MAX_CLEVEL %2u \n", ZSTD_HC_MAX_CLEVEL);
fprintf(f, "static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = {\n");
fprintf(f, " /* W, C, H, S */ \n");
programs/zstdcli.c
+2
View File
@@ -118,6 +118,8 @@ static int usage(const char* programName)
DISPLAY( "input : a filename\n");
DISPLAY( " with no FILE, or when FILE is - , read standard input\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -1 : Fast compression (default) \n");
DISPLAY( " -9 : High compression \n");
DISPLAY( " -d : decompression (default for %s extension)\n", ZSTD_EXTENSION);
//DISPLAY( " -z : force compression\n");
DISPLAY( " -f : overwrite output without prompting \n");