ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1 from Cyan4973/dev

Browse Source 
Dev
This commit is contained in:
Yann Collet
2015-01-24 13:47:43 +01:00
parent 04d8e15e57 759433d679
commit adcffd085c
4 changed files with 98 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/zstd.c
+7 -7
View File
@@ -1004,9 +1004,9 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
//U32 limit = 4 * BLOCKSIZE; //U32 limit = 4 * BLOCKSIZE;
//const U32 updateRate = 2 * BLOCKSIZE; //const U32 updateRate = 2 * BLOCKSIZE;
// Init /* Init */
if (ctx->base==NULL) ctx->base = src, ctx->current=0; if (ctx->base==NULL) ctx->base = src, ctx->current=0;
if (src != ctx->base + ctx->current) // not contiguous if (src != ctx->base + ctx->current) /* not contiguous */
{ {
ZSTD_resetCCtx(ctx); ZSTD_resetCCtx(ctx);
ctx->base = src; ctx->base = src;
@@ -1035,12 +1035,12 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
} }
*/ */
// compress /* compress */
if (maxDstSize < ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; if (maxDstSize < ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
cSize = ZSTD_compressBlock(ctx, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize); cSize = ZSTD_compressBlock(ctx, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize);
if (cSize == 0) if (cSize == 0)
{ {
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize)) return cSize;
} }
else else
@@ -1084,7 +1084,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
BYTE* const ostart = dst; BYTE* const ostart = dst;
BYTE* op = ostart; BYTE* op = ostart;
// Header /* Header */
{ {
size_t headerSize = ZSTD_compressBegin(ctx, dst, maxDstSize); size_t headerSize = ZSTD_compressBegin(ctx, dst, maxDstSize);
if(ZSTD_isError(headerSize)) return headerSize; if(ZSTD_isError(headerSize)) return headerSize;
@@ -1092,7 +1092,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
maxDstSize -= headerSize; maxDstSize -= headerSize;
} }
// Compression /* Compression */
{ {
size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize); size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize);
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize)) return cSize;
@@ -1100,7 +1100,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
maxDstSize -= cSize; maxDstSize -= cSize;
} }
// Close frame /* Close frame */
{ {
size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize); size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize);
if(ZSTD_isError(endSize)) return endSize; if(ZSTD_isError(endSize)) return endSize;
programs/Makefile
+8 -2
View File
@@ -127,11 +127,17 @@ test-zstd32: zstd32 datagen
./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID) ./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID)
./datagen -g6GB | ./zstd32 -vq | ./zstd32 -d > $(VOID) ./datagen -g6GB | ./zstd32 -vq | ./zstd32 -d > $(VOID)
test-fullbench: fullbench test-fullbench: fullbench datagen
./fullbench -i1 ./fullbench -i1
./datagen -P0 -g516K > tmp
./fullbench -i1 tmp
@rm tmp
test-fullbench32: fullbench32 test-fullbench32: fullbench32 datagen
./fullbench32 -i1 ./fullbench32 -i1
./datagen -P0 -g516K > tmp
./fullbench32 -i1 tmp
@rm tmp
test-fuzzer: fuzzer test-fuzzer: fuzzer
./fuzzer ./fuzzer
programs/datagen.c
+62 -25
View File
@@ -56,6 +56,24 @@
#endif #endif
/**************************************
* OS-specific Includes
**************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _isatty */
# ifdef __MINGW32__
int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
# endif
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#else
# include <unistd.h> /* isatty */
# define SET_BINARY_MODE(file)
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#endif
/************************************** /**************************************
* Constants * Constants
**************************************/ **************************************/
@@ -85,7 +103,6 @@
* Local Parameters * Local Parameters
**************************************/ **************************************/
static unsigned no_prompt = 0; static unsigned no_prompt = 0;
static char* programName;
static unsigned displayLevel = 2; static unsigned displayLevel = 2;
@@ -97,7 +114,7 @@ static unsigned int CDG_rand(U32* src)
{ {
U32 rand32 = *src; U32 rand32 = *src;
rand32 *= PRIME1; rand32 *= PRIME1;
rand32 += PRIME2; rand32 ^= PRIME2;
rand32 = CDG_rotl32(rand32, 13); rand32 = CDG_rotl32(rand32, 13);
*src = rand32; *src = rand32;
return rand32; return rand32;
@@ -106,14 +123,20 @@ static unsigned int CDG_rand(U32* src)
#define LTSIZE 8192 #define LTSIZE 8192
#define LTMASK (LTSIZE-1) #define LTMASK (LTSIZE-1)
static const char firstChar = '(';
static const char lastChar = '}';
static void* CDG_createLiteralDistrib(double ld) static void* CDG_createLiteralDistrib(double ld)
{ {
char* lt = malloc(LTSIZE); BYTE* lt = malloc(LTSIZE);
U32 i = 0; U32 i = 0;
char character = '0'; BYTE character = '0';
BYTE firstChar = '(';
BYTE lastChar = '}';
if (ld==0.0)
{
character = 0;
firstChar = 0;
lastChar =255;
}
while (i<LTSIZE) while (i<LTSIZE)
{ {
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1; U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
@@ -137,7 +160,7 @@ static char CDG_genChar(U32* seed, const void* ltctx)
#define CDG_RAND15BITS ((CDG_rand(seed) >> 3) & 32767) #define CDG_RAND15BITS ((CDG_rand(seed) >> 3) & 32767)
#define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 7) ? (CDG_rand(seed) & 15) : (CDG_rand(seed) & 511) + 15) #define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 7) ? (CDG_rand(seed) & 15) : (CDG_rand(seed) & 511) + 15)
#define CDG_DICTSIZE (32 KB) #define CDG_DICTSIZE (32 KB)
static void CDG_generate(U64 size, U32* seed, double matchProba) static void CDG_generate(U64 size, U32* seed, double matchProba, double litProba)
{ {
BYTE fullbuff[CDG_DICTSIZE + 128 KB + 1]; BYTE fullbuff[CDG_DICTSIZE + 128 KB + 1];
BYTE* buff = fullbuff + CDG_DICTSIZE; BYTE* buff = fullbuff + CDG_DICTSIZE;
@@ -145,10 +168,11 @@ static void CDG_generate(U64 size, U32* seed, double matchProba)
U32 P32 = (U32)(32768 * matchProba); U32 P32 = (U32)(32768 * matchProba);
U32 pos=1; U32 pos=1;
U32 genBlockSize = 128 KB; U32 genBlockSize = 128 KB;
double literalDistrib = 0.13; void* ldctx = CDG_createLiteralDistrib(litProba);
void* ldctx = CDG_createLiteralDistrib(literalDistrib); FILE* fout = stdout;
/* Build initial prefix */ /* init */
SET_BINARY_MODE(stdout);
fullbuff[0] = CDG_genChar(seed, ldctx); fullbuff[0] = CDG_genChar(seed, ldctx);
while (pos<32 KB) while (pos<32 KB)
{ {
@@ -207,11 +231,8 @@ static void CDG_generate(U64 size, U32* seed, double matchProba)
} }
} }
/* output datagen */ /* output generated data */
pos=0; fwrite(buff, 1, genBlockSize, fout);
for (;pos+512<=genBlockSize;pos+=512)
printf("%512.512s", buff+pos);
for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
/* Regenerate prefix */ /* Regenerate prefix */
memcpy(fullbuff, buff + 96 KB, 32 KB); memcpy(fullbuff, buff + 96 KB, 32 KB);
} }
@@ -221,7 +242,7 @@ static void CDG_generate(U64 size, U32* seed, double matchProba)
/********************************************************* /*********************************************************
* Command line * Command line
*********************************************************/ *********************************************************/
static int CDG_usage(void) static int CDG_usage(char* programName)
{ {
DISPLAY( "Compressible data generator\n"); DISPLAY( "Compressible data generator\n");
DISPLAY( "Usage :\n"); DISPLAY( "Usage :\n");
@@ -239,9 +260,11 @@ static int CDG_usage(void)
int main(int argc, char** argv) int main(int argc, char** argv)
{ {
int argNb; int argNb;
int proba = CDG_COMPRESSIBILITY_DEFAULT; double proba = (double)CDG_COMPRESSIBILITY_DEFAULT / 100;
double litProba = proba / 3.6;
U64 size = CDG_SIZE_DEFAULT; U64 size = CDG_SIZE_DEFAULT;
U32 seed = CDG_SEED_DEFAULT; U32 seed = CDG_SEED_DEFAULT;
char* programName;
/* Check command line */ /* Check command line */
programName = argv[0]; programName = argv[0];
@@ -262,7 +285,7 @@ int main(int argc, char** argv)
switch(*argument) switch(*argument)
{ {
case 'h': case 'h':
return CDG_usage(); return CDG_usage(programName);
case 'g': case 'g':
argument++; argument++;
size=0; size=0;
@@ -287,23 +310,37 @@ int main(int argc, char** argv)
argument++; argument++;
} }
break; break;
case 'p': case 'P':
argument++; argument++;
proba=0; proba=0.0;
while ((*argument>='0') && (*argument<='9')) while ((*argument>='0') && (*argument<='9'))
{ {
proba *= 10; proba *= 10;
proba += *argument - '0'; proba += *argument - '0';
argument++; argument++;
} }
if (proba<0) proba=0; if (proba>100.) proba=100.;
if (proba>100) proba=100; proba /= 100.;
litProba = proba / 4.;
break;
case 'L':
argument++;
litProba=0.;
while ((*argument>='0') && (*argument<='9'))
{
litProba *= 10;
litProba += *argument - '0';
argument++;
}
if (litProba>100.) litProba=100.;
litProba /= 100.;
break; break;
case 'v': case 'v':
displayLevel = 4; displayLevel = 4;
argument++; argument++;
break; break;
default: ; default:
return CDG_usage(programName);
} }
} }
@@ -312,9 +349,9 @@ int main(int argc, char** argv)
DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION); DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION);
DISPLAYLEVEL(3, "Seed = %u \n", seed); DISPLAYLEVEL(3, "Seed = %u \n", seed);
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba); if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100));
CDG_generate(size, &seed, ((double)proba) / 100); CDG_generate(size, &seed, proba, litProba);
return 0; return 0;
} }
programs/fullbench.c
+21 -5
View File
@@ -383,7 +383,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
} }
/* Allocation */ /* Allocation */
dstBuffSize = srcSize + 512; dstBuffSize = ZSTD_compressBound(srcSize);
dstBuff = malloc(dstBuffSize); dstBuff = malloc(dstBuffSize);
buff2 = malloc(dstBuffSize); buff2 = malloc(dstBuffSize);
if ((!dstBuff) || (!buff2)) if ((!dstBuff) || (!buff2))
@@ -403,6 +403,14 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
{ {
blockProperties_t bp; blockProperties_t bp;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize); ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); // Get first block compressed size
if (bp.blockType != bt_compressed)
{
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
free(dstBuff);
free(buff2);
return 0;
}
g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp) + 3; g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp) + 3;
memcpy(buff2, dstBuff+7, g_cSize); memcpy(buff2, dstBuff+7, g_cSize);
//srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed //srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed
@@ -418,6 +426,13 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize); ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
ip += 4; // Jump magic Number ip += 4; // Jump magic Number
blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block compressed size blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block compressed size
if (bp.blockType != bt_compressed)
{
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
free(dstBuff);
free(buff2);
return 0;
}
iend = ip + 3 + blockSize; // Get end of first block iend = ip + 3 + blockSize; // Get end of first block
ip += 3; // jump first block header ip += 3; // jump first block header
ip += ZSTD_getcBlockSize(ip, iend - ip, &bp) + 3; // jump literal sub block and its header ip += ZSTD_getcBlockSize(ip, iend - ip, &bp) + 3; // jump literal sub block and its header
@@ -450,6 +465,8 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
default : ; default : ;
} }
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */
for (loopNb = 1; loopNb <= nbIterations; loopNb++) for (loopNb = 1; loopNb <= nbIterations; loopNb++)
{ {
double averageTime; double averageTime;
@@ -457,7 +474,6 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
U32 nbRounds=0; U32 nbRounds=0;
DISPLAY("%2i- %-30.30s : \r", loopNb, benchName); DISPLAY("%2i- %-30.30s : \r", loopNb, benchName);
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */
milliTime = BMK_GetMilliStart(); milliTime = BMK_GetMilliStart();
while(BMK_GetMilliStart() == milliTime); while(BMK_GetMilliStart() == milliTime);
@@ -545,7 +561,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
} }
// Alloc /* Alloc */
origBuff = (char*) malloc((size_t)benchedSize); origBuff = (char*) malloc((size_t)benchedSize);
if(!origBuff) if(!origBuff)
{ {
@@ -554,7 +570,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
return 12; return 12;
} }
// Fill input buffer /* Fill input buffer */
DISPLAY("Loading %s... \r", inFileName); DISPLAY("Loading %s... \r", inFileName);
readSize = fread(origBuff, 1, benchedSize, inFile); readSize = fread(origBuff, 1, benchedSize, inFile);
fclose(inFile); fclose(inFile);
@@ -566,7 +582,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
return 13; return 13;
} }
// bench /* bench */
DISPLAY("\r%79s\r", ""); DISPLAY("\r%79s\r", "");
DISPLAY(" %s : \n", inFileName); DISPLAY(" %s : \n", inFileName);
if (benchNb) if (benchNb)