datagen generates lorem ipsum by default

programs/lorem.c

@@ -23,11 +23,11 @@
  * and lacks a regularity more representative of text.
  *
  * The compression ratio achievable on the generated lorem ipsum
- * is still a bit too good, presumably because the dictionary is too small.
+ * is still a bit too good, presumably because the dictionary is a bit too
- * It would be possible to create some more complex scheme,
+ * small. It would be possible to create some more complex scheme, notably by
- * notably by enlarging the dictionary with a word generator,
+ * enlarging the dictionary with a word generator, and adding grammatical rules
- * and adding grammatical rules (composition) and syntax rules.
+ * (composition) and syntax rules. But that's probably overkill for the intended
- * But that's probably overkill for the intended goal.
+ * goal.
  */
 
 #include "lorem.h"

tests/Makefile

@@ -206,7 +206,7 @@ paramgrill : LDLIBS += -lm
 paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c $(PRGDIR)/lorem.c paramgrill.c
 
 CLEAN += datagen
-datagen : $(PRGDIR)/datagen.c datagencli.c
+datagen : $(PRGDIR)/datagen.c $(PRGDIR)/lorem.c loremOut.c datagencli.c
 	$(LINK.c) $^ -o $@$(EXT)
 
 CLEAN += roundTripCrash

tests/datagencli.c

@@ -8,54 +8,52 @@
  * You may select, at your option, one of the above-listed licenses.
  */
 
-
 /*-************************************
-*  Dependencies
+ *  Dependencies
-**************************************/
+ **************************************/
-#include "util.h"      /* Compiler options */
 #include <stdio.h>    /* fprintf, stderr */
 #include "datagen.h"  /* RDG_generate */
-
+#include "loremOut.h" /* LOREM_genOut */
+#include "util.h"     /* Compiler options */
 
 /*-************************************
-*  Constants
+ *  Constants
-**************************************/
+ **************************************/
-#define KB *(1 <<10)
+#define KB *(1 << 10)
-#define MB *(1 <<20)
+#define MB *(1 << 20)
-#define GB *(1U<<30)
+#define GB *(1U << 30)
 
 #define SIZE_DEFAULT ((64 KB) + 1)
 #define SEED_DEFAULT 0
-#define COMPRESSIBILITY_DEFAULT 50
+#define COMPRESSIBILITY_DEFAULT 9999
-
 
 /*-************************************
-*  Macros
+ *  Macros
-**************************************/
+ **************************************/
 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+#define DISPLAYLEVEL(l, ...)  \
+    if (displayLevel >= l) {  \
+        DISPLAY(__VA_ARGS__); \
+    }
 static unsigned displayLevel = 2;
 
-
 /*-*******************************************************
-*  Command line
+ *  Command line
-*********************************************************/
+ *********************************************************/
 static int usage(const char* programName)
 {
-    DISPLAY( "Compressible data generator\n");
+    DISPLAY("Compressible data generator\n");
-    DISPLAY( "Usage :\n");
+    DISPLAY("Usage :\n");
-    DISPLAY( "      %s [args]\n", programName);
+    DISPLAY("      %s [args]\n", programName);
-    DISPLAY( "\n");
+    DISPLAY("\n");
-    DISPLAY( "Arguments :\n");
+    DISPLAY("Arguments :\n");
-    DISPLAY( " -g#    : generate # data (default:%i)\n", SIZE_DEFAULT);
+    DISPLAY(" -g#    : generate # data (default:%i)\n", SIZE_DEFAULT);
-    DISPLAY( " -s#    : Select seed (default:%i)\n", SEED_DEFAULT);
+    DISPLAY(" -s#    : Select seed (default:%i)\n", SEED_DEFAULT);
-    DISPLAY( " -P#    : Select compressibility in %% (default:%i%%)\n",
+    DISPLAY(" -P#    : Select compressibility in %% (range [0-100])\n");
-                        COMPRESSIBILITY_DEFAULT);
+    DISPLAY(" -h     : display help and exit\n");
-    DISPLAY( " -h     : display help and exit\n");
     return 0;
 }
 
-
 int main(int argc, const char** argv)
 {
     unsigned probaU32             = COMPRESSIBILITY_DEFAULT;
@@ -65,48 +63,63 @@ int main(int argc, const char** argv)
     const char* const programName = argv[0];
 
     int argNb;
-    for(argNb=1; argNb<argc; argNb++) {
+    for (argNb = 1; argNb < argc; argNb++) {
         const char* argument = argv[argNb];
 
-        if(!argument) continue;   /* Protection if argument empty */
+        if (!argument)
+            continue; /* Protection if argument empty */
 
         /* Handle commands. Aggregated commands are allowed */
-        if (*argument=='-') {
+        if (*argument == '-') {
             argument++;
-            while (*argument!=0) {
+            while (*argument != 0) {
-                switch(*argument)
+                switch (*argument) {
-                {
                     case 'h':
                         return usage(programName);
                     case 'g':
                         argument++;
-                    size=0;
+                        size = 0;
-                    while ((*argument>='0') && (*argument<='9'))
+                        while ((*argument >= '0') && (*argument <= '9'))
-                        size *= 10, size += *argument++ - '0';
+                            size *= 10, size += (U64)(*argument++ - '0');
-                    if (*argument=='K') { size <<= 10; argument++; }
+                        if (*argument == 'K') {
-                    if (*argument=='M') { size <<= 20; argument++; }
+                            size <<= 10;
-                    if (*argument=='G') { size <<= 30; argument++; }
+                            argument++;
-                    if (*argument=='B') { argument++; }
+                        }
+                        if (*argument == 'M') {
+                            size <<= 20;
+                            argument++;
+                        }
+                        if (*argument == 'G') {
+                            size <<= 30;
+                            argument++;
+                        }
+                        if (*argument == 'B') {
+                            argument++;
+                        }
                         break;
                     case 's':
                         argument++;
-                    seed=0;
+                        seed = 0;
-                    while ((*argument>='0') && (*argument<='9'))
+                        while ((*argument >= '0') && (*argument <= '9'))
-                        seed *= 10, seed += *argument++ - '0';
+                            seed *= 10, seed += (U32)(*argument++ - '0');
                         break;
                     case 'P':
                         argument++;
                         probaU32 = 0;
-                    while ((*argument>='0') && (*argument<='9'))
+                        while ((*argument >= '0') && (*argument <= '9'))
-                        probaU32 *= 10, probaU32 += *argument++ - '0';
+                            probaU32 *= 10,
-                    if (probaU32>100) probaU32 = 100;
+                                    probaU32 += (U32)(*argument++ - '0');
+                        if (probaU32 > 100)
+                            probaU32 = 100;
                         break;
-                case 'L':   /* hidden argument : Literal distribution probability */
+                    case 'L': /* hidden argument : Literal distribution
+                                 probability */
                         argument++;
-                    litProba=0.;
+                        litProba = 0.;
-                    while ((*argument>='0') && (*argument<='9'))
+                        while ((*argument >= '0') && (*argument <= '9'))
                             litProba *= 10, litProba += *argument++ - '0';
-                    if (litProba>100.) litProba=100.;
+                        if (litProba > 100.)
+                            litProba = 100.;
                         litProba /= 100.;
                         break;
                     case 'v':
@@ -116,14 +129,21 @@ int main(int argc, const char** argv)
                     default:
                         return usage(programName);
                 }
-    }   }   }   /* for(argNb=1; argNb<argc; argNb++) */
+            }
+        }
+    } /* for(argNb=1; argNb<argc; argNb++) */
 
     DISPLAYLEVEL(4, "Compressible data Generator \n");
-    if (probaU32!=COMPRESSIBILITY_DEFAULT)
-        DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32);
     DISPLAYLEVEL(3, "Seed = %u \n", (unsigned)seed);
 
-    RDG_genStdout(size, (double)probaU32/100, litProba, seed);
+    if (probaU32 != COMPRESSIBILITY_DEFAULT) {
+        DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32);
+        RDG_genStdout(size, (double)probaU32 / 100, litProba, seed);
+    } else {
+        LOREM_genOut(size, seed);
+    }
+
+    RDG_genStdout(size, (double)probaU32 / 100, litProba, seed);
     DISPLAYLEVEL(3, "\n");
 
     return 0;

tests/loremOut.c

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* Implementation notes:
+ * Generates a stream of Lorem ipsum paragraphs to stdout,
+ * up to the requested size, which can be very large (> 4 GB).
+ * Note that, beyond 1 paragraph, this generator produces
+ * a different content than LOREM_genBuffer (even when using same seed).
+ */
+
+#include "loremOut.h"
+#include <assert.h>
+#include <stdio.h>
+#include "lorem.h"    /* LOREM_genBlock */
+#include "platform.h" /* Compiler options, SET_BINARY_MODE */
+
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+#define LOREM_BLOCKSIZE (1 << 10)
+void LOREM_genOut(unsigned long long size, unsigned seed)
+{
+    char buff[LOREM_BLOCKSIZE] = { 0 };
+    unsigned long long total   = 0;
+    size_t genBlockSize        = (size_t)MIN(size, LOREM_BLOCKSIZE);
+
+    /* init */
+    SET_BINARY_MODE(stdout);
+
+    /* Generate Ipsum text, one paragraph at a time */
+    while (total < size) {
+        size_t generated =
+                LOREM_genBlock(buff, genBlockSize, seed++, total == 0, 0);
+        assert(generated <= genBlockSize);
+        total += generated;
+        assert(total <= size);
+        fwrite(buff,
+               1,
+               generated,
+               stdout); /* note: should check potential write error */
+        if (size - total < genBlockSize)
+            genBlockSize = (size_t)(size - total);
+    }
+    assert(total == size);
+}

tests/loremOut.h

@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* LOREM_genOut():
+ * Generate @size bytes of compressible data using lorem ipsum generator into
+ * stdout.
+ */
+void LOREM_genOut(unsigned long long size, unsigned seed);