ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

datagen generates lorem ipsum by default

Browse Source
This commit is contained in:
Yann Collet
2024-02-20 15:24:25 -08:00
parent 7003c9905e
commit 83598aa106
5 changed files with 177 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/lorem.c
+5 -5
View File
@@ -23,11 +23,11 @@
* and lacks a regularity more representative of text. * and lacks a regularity more representative of text.
* *
* The compression ratio achievable on the generated lorem ipsum * The compression ratio achievable on the generated lorem ipsum
* is still a bit too good, presumably because the dictionary is too small. * is still a bit too good, presumably because the dictionary is a bit too
* It would be possible to create some more complex scheme, * small. It would be possible to create some more complex scheme, notably by
* notably by enlarging the dictionary with a word generator, * enlarging the dictionary with a word generator, and adding grammatical rules
* and adding grammatical rules (composition) and syntax rules. * (composition) and syntax rules. But that's probably overkill for the intended
* But that's probably overkill for the intended goal. * goal.
*/ */
#include "lorem.h" #include "lorem.h"
tests/Makefile
+1 -1
View File
@@ -206,7 +206,7 @@ paramgrill : LDLIBS += -lm
paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c $(PRGDIR)/lorem.c paramgrill.c paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c $(PRGDIR)/lorem.c paramgrill.c
CLEAN += datagen CLEAN += datagen
datagen : $(PRGDIR)/datagen.c datagencli.c datagen : $(PRGDIR)/datagen.c $(PRGDIR)/lorem.c loremOut.c datagencli.c
$(LINK.c) $^ -o $@$(EXT) $(LINK.c) $^ -o $@$(EXT)
CLEAN += roundTripCrash CLEAN += roundTripCrash
tests/datagencli.c
+106 -86
View File
@@ -8,122 +8,142 @@
* You may select, at your option, one of the above-listed licenses. * You may select, at your option, one of the above-listed licenses.
*/ */
/*-************************************
* Dependencies
**************************************/
#include <stdio.h> /* fprintf, stderr */
#include "datagen.h" /* RDG_generate */
#include "loremOut.h" /* LOREM_genOut */
#include "util.h" /* Compiler options */
/*-************************************ /*-************************************
* Dependencies * Constants
**************************************/ **************************************/
#include "util.h" /* Compiler options */ #define KB *(1 << 10)
#include <stdio.h> /* fprintf, stderr */ #define MB *(1 << 20)
#include "datagen.h" /* RDG_generate */ #define GB *(1U << 30)
/*-************************************
* Constants
**************************************/
#define KB *(1 <<10)
#define MB *(1 <<20)
#define GB *(1U<<30)
#define SIZE_DEFAULT ((64 KB) + 1) #define SIZE_DEFAULT ((64 KB) + 1)
#define SEED_DEFAULT 0 #define SEED_DEFAULT 0
#define COMPRESSIBILITY_DEFAULT 50 #define COMPRESSIBILITY_DEFAULT 9999
/*-************************************ /*-************************************
* Macros * Macros
**************************************/ **************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } #define DISPLAYLEVEL(l, ...) \
if (displayLevel >= l) { \
DISPLAY(__VA_ARGS__); \
}
static unsigned displayLevel = 2; static unsigned displayLevel = 2;
/*-******************************************************* /*-*******************************************************
* Command line * Command line
*********************************************************/ *********************************************************/
static int usage(const char* programName) static int usage(const char* programName)
{ {
DISPLAY( "Compressible data generator\n"); DISPLAY("Compressible data generator\n");
DISPLAY( "Usage :\n"); DISPLAY("Usage :\n");
DISPLAY( " %s [args]\n", programName); DISPLAY(" %s [args]\n", programName);
DISPLAY( "\n"); DISPLAY("\n");
DISPLAY( "Arguments :\n"); DISPLAY("Arguments :\n");
DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT); DISPLAY(" -g# : generate # data (default:%i)\n", SIZE_DEFAULT);
DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT); DISPLAY(" -s# : Select seed (default:%i)\n", SEED_DEFAULT);
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", DISPLAY(" -P# : Select compressibility in %% (range [0-100])\n");
COMPRESSIBILITY_DEFAULT); DISPLAY(" -h : display help and exit\n");
DISPLAY( " -h : display help and exit\n");
return 0; return 0;
} }
int main(int argc, const char** argv) int main(int argc, const char** argv)
{ {
unsigned probaU32 = COMPRESSIBILITY_DEFAULT; unsigned probaU32 = COMPRESSIBILITY_DEFAULT;
double litProba = 0.0; double litProba = 0.0;
U64 size = SIZE_DEFAULT; U64 size = SIZE_DEFAULT;
U32 seed = SEED_DEFAULT; U32 seed = SEED_DEFAULT;
const char* const programName = argv[0]; const char* const programName = argv[0];
int argNb; int argNb;
for(argNb=1; argNb<argc; argNb++) { for (argNb = 1; argNb < argc; argNb++) {
const char* argument = argv[argNb]; const char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */ if (!argument)
continue; /* Protection if argument empty */
/* Handle commands. Aggregated commands are allowed */ /* Handle commands. Aggregated commands are allowed */
if (*argument=='-') { if (*argument == '-') {
argument++; argument++;
while (*argument!=0) { while (*argument != 0) {
switch(*argument) switch (*argument) {
{ case 'h':
case 'h': return usage(programName);
return usage(programName); case 'g':
case 'g': argument++;
argument++; size = 0;
size=0; while ((*argument >= '0') && (*argument <= '9'))
while ((*argument>='0') && (*argument<='9')) size *= 10, size += (U64)(*argument++ - '0');
size *= 10, size += *argument++ - '0'; if (*argument == 'K') {
if (*argument=='K') { size <<= 10; argument++; } size <<= 10;
if (*argument=='M') { size <<= 20; argument++; } argument++;
if (*argument=='G') { size <<= 30; argument++; } }
if (*argument=='B') { argument++; } if (*argument == 'M') {
break; size <<= 20;
case 's': argument++;
argument++; }
seed=0; if (*argument == 'G') {
while ((*argument>='0') && (*argument<='9')) size <<= 30;
seed *= 10, seed += *argument++ - '0'; argument++;
break; }
case 'P': if (*argument == 'B') {
argument++; argument++;
probaU32 = 0; }
while ((*argument>='0') && (*argument<='9')) break;
probaU32 *= 10, probaU32 += *argument++ - '0'; case 's':
if (probaU32>100) probaU32 = 100; argument++;
break; seed = 0;
case 'L': /* hidden argument : Literal distribution probability */ while ((*argument >= '0') && (*argument <= '9'))
argument++; seed *= 10, seed += (U32)(*argument++ - '0');
litProba=0.; break;
while ((*argument>='0') && (*argument<='9')) case 'P':
litProba *= 10, litProba += *argument++ - '0'; argument++;
if (litProba>100.) litProba=100.; probaU32 = 0;
litProba /= 100.; while ((*argument >= '0') && (*argument <= '9'))
break; probaU32 *= 10,
case 'v': probaU32 += (U32)(*argument++ - '0');
displayLevel = 4; if (probaU32 > 100)
argument++; probaU32 = 100;
break; break;
default: case 'L': /* hidden argument : Literal distribution
return usage(programName); probability */
argument++;
litProba = 0.;
while ((*argument >= '0') && (*argument <= '9'))
litProba *= 10, litProba += *argument++ - '0';
if (litProba > 100.)
litProba = 100.;
litProba /= 100.;
break;
case 'v':
displayLevel = 4;
argument++;
break;
default:
return usage(programName);
} }
} } } /* for(argNb=1; argNb<argc; argNb++) */ }
}
} /* for(argNb=1; argNb<argc; argNb++) */
DISPLAYLEVEL(4, "Compressible data Generator \n"); DISPLAYLEVEL(4, "Compressible data Generator \n");
if (probaU32!=COMPRESSIBILITY_DEFAULT)
DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32);
DISPLAYLEVEL(3, "Seed = %u \n", (unsigned)seed); DISPLAYLEVEL(3, "Seed = %u \n", (unsigned)seed);
RDG_genStdout(size, (double)probaU32/100, litProba, seed); if (probaU32 != COMPRESSIBILITY_DEFAULT) {
DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32);
RDG_genStdout(size, (double)probaU32 / 100, litProba, seed);
} else {
LOREM_genOut(size, seed);
}
RDG_genStdout(size, (double)probaU32 / 100, litProba, seed);
DISPLAYLEVEL(3, "\n"); DISPLAYLEVEL(3, "\n");
return 0; return 0;
tests/loremOut.c
+50
View File
@@ -0,0 +1,50 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* Implementation notes:
* Generates a stream of Lorem ipsum paragraphs to stdout,
* up to the requested size, which can be very large (> 4 GB).
* Note that, beyond 1 paragraph, this generator produces
* a different content than LOREM_genBuffer (even when using same seed).
*/
#include "loremOut.h"
#include <assert.h>
#include <stdio.h>
#include "lorem.h" /* LOREM_genBlock */
#include "platform.h" /* Compiler options, SET_BINARY_MODE */
#define MIN(a, b) ((a) < (b) ? (a) : (b))
#define LOREM_BLOCKSIZE (1 << 10)
void LOREM_genOut(unsigned long long size, unsigned seed)
{
char buff[LOREM_BLOCKSIZE] = { 0 };
unsigned long long total = 0;
size_t genBlockSize = (size_t)MIN(size, LOREM_BLOCKSIZE);
/* init */
SET_BINARY_MODE(stdout);
/* Generate Ipsum text, one paragraph at a time */
while (total < size) {
size_t generated =
LOREM_genBlock(buff, genBlockSize, seed++, total == 0, 0);
assert(generated <= genBlockSize);
total += generated;
assert(total <= size);
fwrite(buff,
1,
generated,
stdout); /* note: should check potential write error */
if (size - total < genBlockSize)
genBlockSize = (size_t)(size - total);
}
assert(total == size);
}
tests/loremOut.h
+15
View File
@@ -0,0 +1,15 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* LOREM_genOut():
* Generate @size bytes of compressible data using lorem ipsum generator into
* stdout.
*/
void LOREM_genOut(unsigned long long size, unsigned seed);