ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

datagen refactoring

Browse Source
This commit is contained in:
Yann Collet
2015-02-10 18:15:20 +01:00
parent 6610cb381c
commit fb98fd0bd4
5 changed files with 84 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/datagen.c
+55 -54
View File
@@ -122,84 +122,85 @@ static char RDG_genChar(U32* seed, const void* ltctx)
return lt[id];
}
#define RDG_DICTSIZE (32 KB)
#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 32767)
#define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15)
#define RDG_DICTSIZE (32 KB)
void RDG_generate(U64 size, U32 seedInit, double matchProba, double litProba)
void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, void* litTable, unsigned* seedPtr)
{
BYTE fullbuff[RDG_DICTSIZE + 128 KB + 1];
BYTE* buff = fullbuff + RDG_DICTSIZE;
U64 total=0;
U32 P32 = (U32)(32768 * matchProba);
U32 pos=1;
U32 genBlockSize = 128 KB;
void* ldctx = RDG_createLiteralDistrib(litProba);
FILE* fout = stdout;
U32* seed = &seedInit;;
BYTE* buffPtr = ((BYTE*)buffer) - prefixSize;
const U32 matchProba32 = (U32)(32768 * matchProba);
size_t pos = prefixSize;
void* ldctx = litTable;
U32* seed = seedPtr;
/* init */
SET_BINARY_MODE(stdout);
fullbuff[0] = RDG_genChar(seed, ldctx);
while (pos<32 KB)
if (pos==0) buffPtr[0] = RDG_genChar(seed, ldctx), pos=1;
/* Generate compressible data */
while (pos < buffSize)
{
/* Select : Literal (char) or Match (within 32K) */
if (RDG_RAND15BITS < P32)
if (RDG_RAND15BITS < matchProba32)
{
/* Copy (within 64K) */
/* Copy (within 32K) */
int match;
U32 d;
int ref;
int length = RDG_RANDLENGTH + 4;
U32 offset = RDG_RAND15BITS + 1;
if (offset > pos) offset = pos;
ref = pos - offset;
if (pos + length > buffSize) length = buffSize - pos;
match = pos - offset;
d = pos + length;
while (pos < d) fullbuff[pos++] = fullbuff[ref++];
while (pos < d) buffPtr[pos++] = buffPtr[match++];
}
else
{
/* Literal (noise) */
U32 d = pos + RDG_RANDLENGTH;
while (pos < d) fullbuff[pos++] = RDG_genChar(seed, ldctx);
U32 d;
int length = RDG_RANDLENGTH;
if (pos + length > buffSize) length = buffSize - pos;
d = pos + length;
while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldctx);
}
}
}
void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
{
void* ldctx;
if (litProba==0.0) litProba = matchProba / 3.6;
ldctx = RDG_createLiteralDistrib(litProba);
RDG_genBlock(buffer, size, 0, matchProba, ldctx, &seed);
}
#define RDG_BLOCKSIZE (128 KB)
void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed)
{
BYTE fullbuff[RDG_DICTSIZE + RDG_BLOCKSIZE + 1];
BYTE* buff = fullbuff + RDG_DICTSIZE;
U64 total = 0;
U32 genBlockSize = RDG_BLOCKSIZE;
void* ldctx;
/* init */
if (litProba==0.0) litProba = matchProba / 3.6;
ldctx = RDG_createLiteralDistrib(litProba);
SET_BINARY_MODE(stdout);
/* Generate dict */
RDG_genBlock(fullbuff, RDG_DICTSIZE, 0, matchProba, ldctx, &seed);
/* Generate compressible data */
pos = 0;
while (total < size)
{
if (size-total < 128 KB) genBlockSize = (U32)(size-total);
RDG_genBlock(buff, RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, ldctx, &seed);
if (size-total < RDG_BLOCKSIZE) genBlockSize = (U32)(size-total);
total += genBlockSize;
buff[genBlockSize] = 0;
pos = 0;
while (pos<genBlockSize)
{
/* Select : Literal (char) or Match (within 32K) */
if (RDG_RAND15BITS < P32)
{
/* Copy (within 64K) */
int ref;
U32 d;
int length = RDG_RANDLENGTH + 4;
U32 offset = RDG_RAND15BITS + 1;
if (pos + length > genBlockSize ) length = genBlockSize - pos;
ref = pos - offset;
d = pos + length;
while (pos < d) buff[pos++] = buff[ref++];
}
else
{
/* Literal (noise) */
U32 d;
int length = RDG_RANDLENGTH;
if (pos + length > genBlockSize) length = genBlockSize - pos;
d = pos + length;
while (pos < d) buff[pos++] = RDG_genChar(seed, ldctx);
}
}
/* output generated data */
fwrite(buff, 1, genBlockSize, fout);
/* Regenerate prefix */
memcpy(fullbuff, buff + 96 KB, 32 KB);
fwrite(buff, 1, genBlockSize, stdout);
/* update dict */
memcpy(fullbuff, buff + (RDG_BLOCKSIZE - RDG_DICTSIZE), RDG_DICTSIZE);
}
}