build(rust): link Rust CLI generators into program tests
Keep the original C translation units as declaration-only build shims while the Rust CLI helper archive owns datagen and lorem. Track both Rust sources in the program and test archive prerequisites, and link the helpers-only archive into C test binaries that still provide their own file I/O and orchestration. This preserves the existing C test harnesses while making the migrated helper implementations the single definitions used by the binary and tests. Test Plan: - make -C programs zstd V=1 - make -C tests datagen V=1 - make -C tests test-rust-lib-smoke V=1 - git diff --check Refs: programs/datagen.c, programs/lorem.c, tests/Makefile
This commit is contained in:
+2
-1
@@ -33,7 +33,8 @@ RUST_SOURCES := $(RUST_MANIFEST) $(RUST_DIR)/Cargo.lock \
|
|||||||
$(shell find $(RUST_DIR)/src -type f -name '*.rs' -print)
|
$(shell find $(RUST_DIR)/src -type f -name '*.rs' -print)
|
||||||
RUST_CLI_SOURCES := $(RUST_CLI_MANIFEST) $(RUST_CLI_DIR)/Cargo.lock \
|
RUST_CLI_SOURCES := $(RUST_CLI_MANIFEST) $(RUST_CLI_DIR)/Cargo.lock \
|
||||||
$(RUST_CLI_DIR)/src/lib.rs $(RUST_DIR)/src/zstd_cli.rs \
|
$(RUST_CLI_DIR)/src/lib.rs $(RUST_DIR)/src/zstd_cli.rs \
|
||||||
$(RUST_DIR)/src/timefn.rs $(RUST_DIR)/src/benchfn.rs
|
$(RUST_DIR)/src/timefn.rs $(RUST_DIR)/src/benchfn.rs \
|
||||||
|
$(RUST_DIR)/src/datagen.rs $(RUST_DIR)/src/lorem.rs
|
||||||
|
|
||||||
# Keep Rust's HUF implementation in lockstep with libzstd.mk's C selection.
|
# Keep Rust's HUF implementation in lockstep with libzstd.mk's C selection.
|
||||||
# Forced modes may arrive as libzstd.mk variables or as direct -D flags in
|
# Forced modes may arrive as libzstd.mk variables or as direct -D flags in
|
||||||
|
|||||||
+3
-174
@@ -8,179 +8,8 @@
|
|||||||
* You may select, at your option, one of the above-listed licenses.
|
* You may select, at your option, one of the above-listed licenses.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
/* The implementation lives in rust/src/datagen.rs, built into the Rust CLI
|
||||||
|
* static archive. This translation unit stays in the original source lists so
|
||||||
|
* build configuration keeps working while the implementation is in Rust. */
|
||||||
|
|
||||||
|
|
||||||
/*-************************************
|
|
||||||
* Dependencies
|
|
||||||
**************************************/
|
|
||||||
#include "datagen.h"
|
#include "datagen.h"
|
||||||
#include "platform.h" /* SET_BINARY_MODE */
|
|
||||||
#include <stdlib.h> /* malloc, free */
|
|
||||||
#include <stdio.h> /* FILE, fwrite, fprintf */
|
|
||||||
#include <string.h> /* memcpy */
|
|
||||||
#include "../lib/common/mem.h" /* U32 */
|
|
||||||
|
|
||||||
|
|
||||||
/*-************************************
|
|
||||||
* Macros
|
|
||||||
**************************************/
|
|
||||||
#define KB *(1 <<10)
|
|
||||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
|
||||||
|
|
||||||
#define RDG_DEBUG 0
|
|
||||||
#define TRACE(...) if (RDG_DEBUG) fprintf(stderr, __VA_ARGS__ )
|
|
||||||
|
|
||||||
|
|
||||||
/*-************************************
|
|
||||||
* Local constants
|
|
||||||
**************************************/
|
|
||||||
#define LTLOG 13
|
|
||||||
#define LTSIZE (1<<LTLOG)
|
|
||||||
#define LTMASK (LTSIZE-1)
|
|
||||||
|
|
||||||
|
|
||||||
/*-*******************************************************
|
|
||||||
* Local Functions
|
|
||||||
*********************************************************/
|
|
||||||
#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
|
||||||
static U32 RDG_rand(U32* src)
|
|
||||||
{
|
|
||||||
static const U32 prime1 = 2654435761U;
|
|
||||||
static const U32 prime2 = 2246822519U;
|
|
||||||
U32 rand32 = *src;
|
|
||||||
rand32 *= prime1;
|
|
||||||
rand32 ^= prime2;
|
|
||||||
rand32 = RDG_rotl32(rand32, 13);
|
|
||||||
*src = rand32;
|
|
||||||
return rand32 >> 5;
|
|
||||||
}
|
|
||||||
|
|
||||||
typedef U32 fixedPoint_24_8;
|
|
||||||
|
|
||||||
static void RDG_fillLiteralDistrib(BYTE* ldt, fixedPoint_24_8 ld)
|
|
||||||
{
|
|
||||||
BYTE const firstChar = (ld<=0.0) ? 0 : '(';
|
|
||||||
BYTE const lastChar = (ld<=0.0) ? 255 : '}';
|
|
||||||
BYTE character = (ld<=0.0) ? 0 : '0';
|
|
||||||
U32 u;
|
|
||||||
|
|
||||||
if (ld<=0) ld = 0;
|
|
||||||
for (u=0; u<LTSIZE; ) {
|
|
||||||
U32 const weight = (((LTSIZE - u) * ld) >> 8) + 1;
|
|
||||||
U32 const end = MIN ( u + weight , LTSIZE);
|
|
||||||
while (u < end) ldt[u++] = character;
|
|
||||||
character++;
|
|
||||||
if (character > lastChar) character = firstChar;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
static BYTE RDG_genChar(U32* seed, const BYTE* ldt)
|
|
||||||
{
|
|
||||||
U32 const id = RDG_rand(seed) & LTMASK;
|
|
||||||
return ldt[id]; /* memory-sanitizer fails here, stating "uninitialized value" when table initialized with P==0.0. Checked : table is fully initialized */
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
static U32 RDG_rand15Bits (U32* seedPtr)
|
|
||||||
{
|
|
||||||
return RDG_rand(seedPtr) & 0x7FFF;
|
|
||||||
}
|
|
||||||
|
|
||||||
static U32 RDG_randLength(U32* seedPtr)
|
|
||||||
{
|
|
||||||
if (RDG_rand(seedPtr) & 7) return (RDG_rand(seedPtr) & 0xF); /* small length */
|
|
||||||
return (RDG_rand(seedPtr) & 0x1FF) + 0xF;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize,
|
|
||||||
double matchProba, const BYTE* ldt, U32* seedPtr)
|
|
||||||
{
|
|
||||||
BYTE* const buffPtr = (BYTE*)buffer;
|
|
||||||
U32 const matchProba32 = (U32)(32768 * matchProba);
|
|
||||||
size_t pos = prefixSize;
|
|
||||||
U32 prevOffset = 1;
|
|
||||||
|
|
||||||
/* special case : sparse content */
|
|
||||||
while (matchProba >= 1.0) {
|
|
||||||
size_t size0 = RDG_rand(seedPtr) & 3;
|
|
||||||
size0 = (size_t)1 << (16 + size0 * 2);
|
|
||||||
size0 += RDG_rand(seedPtr) & (size0-1); /* because size0 is power of 2*/
|
|
||||||
if (buffSize < pos + size0) {
|
|
||||||
memset(buffPtr+pos, 0, buffSize-pos);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
memset(buffPtr+pos, 0, size0);
|
|
||||||
pos += size0;
|
|
||||||
buffPtr[pos-1] = RDG_genChar(seedPtr, ldt);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* init */
|
|
||||||
if (pos==0) buffPtr[0] = RDG_genChar(seedPtr, ldt), pos=1;
|
|
||||||
|
|
||||||
/* Generate compressible data */
|
|
||||||
while (pos < buffSize) {
|
|
||||||
/* Select : Literal (char) or Match (within 32K) */
|
|
||||||
if (RDG_rand15Bits(seedPtr) < matchProba32) {
|
|
||||||
/* Copy (within 32K) */
|
|
||||||
U32 const length = RDG_randLength(seedPtr) + 4;
|
|
||||||
U32 const d = (U32) MIN(pos + length , buffSize);
|
|
||||||
U32 const repeatOffset = (RDG_rand(seedPtr) & 15) == 2;
|
|
||||||
U32 const randOffset = RDG_rand15Bits(seedPtr) + 1;
|
|
||||||
U32 const offset = repeatOffset ? prevOffset : (U32) MIN(randOffset , pos);
|
|
||||||
size_t match = pos - offset;
|
|
||||||
while (pos < d) { buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */ }
|
|
||||||
prevOffset = offset;
|
|
||||||
} else {
|
|
||||||
/* Literal (noise) */
|
|
||||||
U32 const length = RDG_randLength(seedPtr);
|
|
||||||
U32 const d = (U32) MIN(pos + length, buffSize);
|
|
||||||
while (pos < d) { buffPtr[pos++] = RDG_genChar(seedPtr, ldt); }
|
|
||||||
} }
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
|
|
||||||
{
|
|
||||||
U32 seed32 = seed;
|
|
||||||
BYTE ldt[LTSIZE];
|
|
||||||
memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */
|
|
||||||
if (litProba<=0.0) litProba = matchProba / 4.5;
|
|
||||||
RDG_fillLiteralDistrib(ldt, (fixedPoint_24_8)(litProba * 256 + 0.001));
|
|
||||||
RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed32);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed)
|
|
||||||
{
|
|
||||||
U32 seed32 = seed;
|
|
||||||
size_t const stdBlockSize = 128 KB;
|
|
||||||
size_t const stdDictSize = 32 KB;
|
|
||||||
BYTE* const buff = (BYTE*)malloc(stdDictSize + stdBlockSize);
|
|
||||||
U64 total = 0;
|
|
||||||
BYTE ldt[LTSIZE]; /* literals distribution table */
|
|
||||||
|
|
||||||
/* init */
|
|
||||||
if (buff==NULL) { perror("datagen"); exit(1); }
|
|
||||||
if (litProba<=0.0) litProba = matchProba / 4.5;
|
|
||||||
memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */
|
|
||||||
RDG_fillLiteralDistrib(ldt, (fixedPoint_24_8)(litProba * 256 + 0.001));
|
|
||||||
SET_BINARY_MODE(stdout);
|
|
||||||
|
|
||||||
/* Generate initial dict */
|
|
||||||
RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed32);
|
|
||||||
|
|
||||||
/* Generate compressible data */
|
|
||||||
while (total < size) {
|
|
||||||
size_t const genBlockSize = (size_t) (MIN (stdBlockSize, size-total));
|
|
||||||
RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed32);
|
|
||||||
total += genBlockSize;
|
|
||||||
{ size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; }
|
|
||||||
/* update dict */
|
|
||||||
memcpy(buff, buff + stdBlockSize, stdDictSize);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* cleanup */
|
|
||||||
free(buff);
|
|
||||||
}
|
|
||||||
|
|||||||
+3
-273
@@ -8,278 +8,8 @@
|
|||||||
* You may select, at your option, one of the above-listed licenses.
|
* You may select, at your option, one of the above-listed licenses.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Implementation notes:
|
/* The implementation lives in rust/src/lorem.rs, built into the Rust CLI
|
||||||
*
|
* static archive. This translation unit stays in the original source lists so
|
||||||
* This is a very simple lorem ipsum generator
|
* build configuration keeps working while the implementation is in Rust. */
|
||||||
* which features a static list of words
|
|
||||||
* and print them one after another randomly
|
|
||||||
* with a fake sentence / paragraph structure.
|
|
||||||
*
|
|
||||||
* The goal is to generate a printable text
|
|
||||||
* that can be used to fake a text compression scenario.
|
|
||||||
* The resulting compression / ratio curve of the lorem ipsum generator
|
|
||||||
* is more satisfying than the previous statistical generator,
|
|
||||||
* which was initially designed for entropy compression,
|
|
||||||
* and lacks a regularity more representative of text.
|
|
||||||
*
|
|
||||||
* The compression ratio achievable on the generated lorem ipsum
|
|
||||||
* is still a bit too good, presumably because the dictionary is a bit too
|
|
||||||
* small. It would be possible to create some more complex scheme, notably by
|
|
||||||
* enlarging the dictionary with a word generator, and adding grammatical rules
|
|
||||||
* (composition) and syntax rules. But that's probably overkill for the intended
|
|
||||||
* goal.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#include "lorem.h"
|
#include "lorem.h"
|
||||||
#include <assert.h>
|
|
||||||
#include <limits.h> /* INT_MAX */
|
|
||||||
#include <string.h> /* memcpy */
|
|
||||||
|
|
||||||
#define WORD_MAX_SIZE 20
|
|
||||||
|
|
||||||
/* Define the word pool */
|
|
||||||
static const char* kWords[] = {
|
|
||||||
"lorem", "ipsum", "dolor", "sit", "amet",
|
|
||||||
"consectetur", "adipiscing", "elit", "sed", "do",
|
|
||||||
"eiusmod", "tempor", "incididunt", "ut", "labore",
|
|
||||||
"et", "dolore", "magna", "aliqua", "dis",
|
|
||||||
"lectus", "vestibulum", "mattis", "ullamcorper", "velit",
|
|
||||||
"commodo", "a", "lacus", "arcu", "magnis",
|
|
||||||
"parturient", "montes", "nascetur", "ridiculus", "mus",
|
|
||||||
"mauris", "nulla", "malesuada", "pellentesque", "eget",
|
|
||||||
"gravida", "in", "dictum", "non", "erat",
|
|
||||||
"nam", "voluptat", "maecenas", "blandit", "aliquam",
|
|
||||||
"etiam", "enim", "lobortis", "scelerisque", "fermentum",
|
|
||||||
"dui", "faucibus", "ornare", "at", "elementum",
|
|
||||||
"eu", "facilisis", "odio", "morbi", "quis",
|
|
||||||
"eros", "donec", "ac", "orci", "purus",
|
|
||||||
"turpis", "cursus", "leo", "vel", "porta",
|
|
||||||
"consequat", "interdum", "varius", "vulputate", "aliquet",
|
|
||||||
"pharetra", "nunc", "auctor", "urna", "id",
|
|
||||||
"metus", "viverra", "nibh", "cras", "mi",
|
|
||||||
"unde", "omnis", "iste", "natus", "error",
|
|
||||||
"perspiciatis", "voluptatem", "accusantium", "doloremque", "laudantium",
|
|
||||||
"totam", "rem", "aperiam", "eaque", "ipsa",
|
|
||||||
"quae", "ab", "illo", "inventore", "veritatis",
|
|
||||||
"quasi", "architecto", "beatae", "vitae", "dicta",
|
|
||||||
"sunt", "explicabo", "nemo", "ipsam", "quia",
|
|
||||||
"voluptas", "aspernatur", "aut", "odit", "fugit",
|
|
||||||
"consequuntur", "magni", "dolores", "eos", "qui",
|
|
||||||
"ratione", "sequi", "nesciunt", "neque", "porro",
|
|
||||||
"quisquam", "est", "dolorem", "adipisci", "numquam",
|
|
||||||
"eius", "modi", "tempora", "incidunt", "magnam",
|
|
||||||
"quaerat", "ad", "minima", "veniam", "nostrum",
|
|
||||||
"ullam", "corporis", "suscipit", "laboriosam", "nisi",
|
|
||||||
"aliquid", "ex", "ea", "commodi", "consequatur",
|
|
||||||
"autem", "eum", "iure", "voluptate", "esse",
|
|
||||||
"quam", "nihil", "molestiae", "illum", "fugiat",
|
|
||||||
"quo", "pariatur", "vero", "accusamus", "iusto",
|
|
||||||
"dignissimos", "ducimus", "blanditiis", "praesentium", "voluptatum",
|
|
||||||
"deleniti", "atque", "corrupti", "quos", "quas",
|
|
||||||
"molestias", "excepturi", "sint", "occaecati", "cupiditate",
|
|
||||||
"provident", "similique", "culpa", "officia", "deserunt",
|
|
||||||
"mollitia", "animi", "laborum", "dolorum", "fuga",
|
|
||||||
"harum", "quidem", "rerum", "facilis", "expedita",
|
|
||||||
"distinctio", "libero", "tempore", "cum", "soluta",
|
|
||||||
"nobis", "eligendi", "optio", "cumque", "impedit",
|
|
||||||
"minus", "quod", "maxime", "placeat", "facere",
|
|
||||||
"possimus", "assumenda", "repellendus", "temporibus", "quibusdam",
|
|
||||||
"officiis", "debitis", "saepe", "eveniet", "voluptates",
|
|
||||||
"repudiandae", "recusandae", "itaque", "earum", "hic",
|
|
||||||
"tenetur", "sapiente", "delectus", "reiciendis", "cillum",
|
|
||||||
"maiores", "alias", "perferendis", "doloribus", "asperiores",
|
|
||||||
"repellat", "minim", "nostrud", "exercitation", "ullamco",
|
|
||||||
"laboris", "aliquip", "duis", "aute", "irure",
|
|
||||||
};
|
|
||||||
static const unsigned kNbWords = sizeof(kWords) / sizeof(kWords[0]);
|
|
||||||
|
|
||||||
/* simple 1-dimension distribution, based on word's length, favors small words
|
|
||||||
*/
|
|
||||||
static const int kWeights[] = { 0, 8, 6, 4, 3, 2 };
|
|
||||||
static const size_t kNbWeights = sizeof(kWeights) / sizeof(kWeights[0]);
|
|
||||||
|
|
||||||
#define DISTRIB_SIZE_MAX 650
|
|
||||||
static int g_distrib[DISTRIB_SIZE_MAX] = { 0 };
|
|
||||||
static unsigned g_distribCount = 0;
|
|
||||||
|
|
||||||
static void countFreqs(
|
|
||||||
const char* words[],
|
|
||||||
size_t nbWords,
|
|
||||||
const int* weights,
|
|
||||||
size_t nbWeights)
|
|
||||||
{
|
|
||||||
unsigned total = 0;
|
|
||||||
size_t w;
|
|
||||||
for (w = 0; w < nbWords; w++) {
|
|
||||||
size_t len = strlen(words[w]);
|
|
||||||
int lmax;
|
|
||||||
if (len >= nbWeights)
|
|
||||||
len = nbWeights - 1;
|
|
||||||
lmax = weights[len];
|
|
||||||
total += (unsigned)lmax;
|
|
||||||
}
|
|
||||||
g_distribCount = total;
|
|
||||||
assert(g_distribCount <= DISTRIB_SIZE_MAX);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void init_word_distrib(
|
|
||||||
const char* words[],
|
|
||||||
size_t nbWords,
|
|
||||||
const int* weights,
|
|
||||||
size_t nbWeights)
|
|
||||||
{
|
|
||||||
size_t w, d = 0;
|
|
||||||
countFreqs(words, nbWords, weights, nbWeights);
|
|
||||||
for (w = 0; w < nbWords; w++) {
|
|
||||||
size_t len = strlen(words[w]);
|
|
||||||
int l, lmax;
|
|
||||||
if (len >= nbWeights)
|
|
||||||
len = nbWeights - 1;
|
|
||||||
lmax = weights[len];
|
|
||||||
for (l = 0; l < lmax; l++) {
|
|
||||||
g_distrib[d++] = (int)w;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Note: this unit only works when invoked sequentially.
|
|
||||||
* No concurrent access is allowed */
|
|
||||||
static char* g_ptr = NULL;
|
|
||||||
static size_t g_nbChars = 0;
|
|
||||||
static size_t g_maxChars = 10000000;
|
|
||||||
static unsigned g_randRoot = 0;
|
|
||||||
|
|
||||||
#define RDG_rotl32(x, r) ((x << r) | (x >> (32 - r)))
|
|
||||||
static unsigned LOREM_rand(unsigned range)
|
|
||||||
{
|
|
||||||
static const unsigned prime1 = 2654435761U;
|
|
||||||
static const unsigned prime2 = 2246822519U;
|
|
||||||
unsigned rand32 = g_randRoot;
|
|
||||||
rand32 *= prime1;
|
|
||||||
rand32 ^= prime2;
|
|
||||||
rand32 = RDG_rotl32(rand32, 13);
|
|
||||||
g_randRoot = rand32;
|
|
||||||
return (unsigned)(((unsigned long long)rand32 * range) >> 32);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void writeLastCharacters(void)
|
|
||||||
{
|
|
||||||
size_t lastChars = g_maxChars - g_nbChars;
|
|
||||||
assert(g_maxChars >= g_nbChars);
|
|
||||||
if (lastChars == 0)
|
|
||||||
return;
|
|
||||||
g_ptr[g_nbChars++] = '.';
|
|
||||||
if (lastChars > 2) {
|
|
||||||
memset(g_ptr + g_nbChars, ' ', lastChars - 2);
|
|
||||||
}
|
|
||||||
if (lastChars > 1) {
|
|
||||||
g_ptr[g_maxChars - 1] = '\n';
|
|
||||||
}
|
|
||||||
g_nbChars = g_maxChars;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void generateWord(const char* word, const char* separator, int upCase)
|
|
||||||
{
|
|
||||||
size_t const len = strlen(word) + strlen(separator);
|
|
||||||
if (g_nbChars + len > g_maxChars) {
|
|
||||||
writeLastCharacters();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
memcpy(g_ptr + g_nbChars, word, strlen(word));
|
|
||||||
if (upCase) {
|
|
||||||
static const char toUp = 'A' - 'a';
|
|
||||||
g_ptr[g_nbChars] = (char)(g_ptr[g_nbChars] + toUp);
|
|
||||||
}
|
|
||||||
g_nbChars += strlen(word);
|
|
||||||
memcpy(g_ptr + g_nbChars, separator, strlen(separator));
|
|
||||||
g_nbChars += strlen(separator);
|
|
||||||
}
|
|
||||||
|
|
||||||
static int about(unsigned target)
|
|
||||||
{
|
|
||||||
return (int)(LOREM_rand(target) + LOREM_rand(target) + 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Function to generate a random sentence */
|
|
||||||
static void generateSentence(int nbWords)
|
|
||||||
{
|
|
||||||
int commaPos = about(9);
|
|
||||||
int comma2 = commaPos + about(7);
|
|
||||||
int qmark = (LOREM_rand(11) == 7);
|
|
||||||
const char* endSep = qmark ? "? " : ". ";
|
|
||||||
int i;
|
|
||||||
for (i = 0; i < nbWords; i++) {
|
|
||||||
int const wordID = g_distrib[LOREM_rand(g_distribCount)];
|
|
||||||
const char* const word = kWords[wordID];
|
|
||||||
const char* sep = " ";
|
|
||||||
if (i == commaPos)
|
|
||||||
sep = ", ";
|
|
||||||
if (i == comma2)
|
|
||||||
sep = ", ";
|
|
||||||
if (i == nbWords - 1)
|
|
||||||
sep = endSep;
|
|
||||||
generateWord(word, sep, i == 0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static void generateParagraph(int nbSentences)
|
|
||||||
{
|
|
||||||
int i;
|
|
||||||
for (i = 0; i < nbSentences; i++) {
|
|
||||||
int wordsPerSentence = about(11);
|
|
||||||
generateSentence(wordsPerSentence);
|
|
||||||
}
|
|
||||||
if (g_nbChars < g_maxChars) {
|
|
||||||
g_ptr[g_nbChars++] = '\n';
|
|
||||||
}
|
|
||||||
if (g_nbChars < g_maxChars) {
|
|
||||||
g_ptr[g_nbChars++] = '\n';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* It's "common" for lorem ipsum generators to start with the same first
|
|
||||||
* pre-defined sentence */
|
|
||||||
static void generateFirstSentence(void)
|
|
||||||
{
|
|
||||||
int i;
|
|
||||||
for (i = 0; i < 18; i++) {
|
|
||||||
const char* word = kWords[i];
|
|
||||||
const char* separator = " ";
|
|
||||||
if (i == 4)
|
|
||||||
separator = ", ";
|
|
||||||
if (i == 7)
|
|
||||||
separator = ", ";
|
|
||||||
generateWord(word, separator, i == 0);
|
|
||||||
}
|
|
||||||
generateWord(kWords[18], ". ", 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t
|
|
||||||
LOREM_genBlock(void* buffer, size_t size, unsigned seed, int first, int fill)
|
|
||||||
{
|
|
||||||
g_ptr = (char*)buffer;
|
|
||||||
assert(size < INT_MAX);
|
|
||||||
g_maxChars = size;
|
|
||||||
g_nbChars = 0;
|
|
||||||
g_randRoot = seed;
|
|
||||||
if (g_distribCount == 0) {
|
|
||||||
init_word_distrib(kWords, kNbWords, kWeights, kNbWeights);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (first) {
|
|
||||||
generateFirstSentence();
|
|
||||||
}
|
|
||||||
while (g_nbChars < g_maxChars) {
|
|
||||||
int sentencePerParagraph = about(7);
|
|
||||||
generateParagraph(sentencePerParagraph);
|
|
||||||
if (!fill)
|
|
||||||
break; /* only generate one paragraph in not-fill mode */
|
|
||||||
}
|
|
||||||
g_ptr = NULL;
|
|
||||||
return g_nbChars;
|
|
||||||
}
|
|
||||||
|
|
||||||
void LOREM_genBuffer(void* buffer, size_t size, unsigned seed)
|
|
||||||
{
|
|
||||||
LOREM_genBlock(buffer, size, seed, 1, 1);
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,5 +1,9 @@
|
|||||||
#[path = "../../src/benchfn.rs"]
|
#[path = "../../src/benchfn.rs"]
|
||||||
mod benchfn;
|
mod benchfn;
|
||||||
|
#[path = "../../src/datagen.rs"]
|
||||||
|
mod datagen;
|
||||||
|
#[path = "../../src/lorem.rs"]
|
||||||
|
mod lorem;
|
||||||
#[path = "../../src/timefn.rs"]
|
#[path = "../../src/timefn.rs"]
|
||||||
mod timefn;
|
mod timefn;
|
||||||
#[cfg(feature = "cli")]
|
#[cfg(feature = "cli")]
|
||||||
|
|||||||
+4
-2
@@ -110,7 +110,8 @@ RUST_CLI_DIR := $(RUST_DIR)/cli
|
|||||||
RUST_CLI_MANIFEST := $(RUST_CLI_DIR)/Cargo.toml
|
RUST_CLI_MANIFEST := $(RUST_CLI_DIR)/Cargo.toml
|
||||||
RUST_CLI_HELPER_SOURCES := $(RUST_CLI_MANIFEST) $(RUST_CLI_DIR)/Cargo.lock \
|
RUST_CLI_HELPER_SOURCES := $(RUST_CLI_MANIFEST) $(RUST_CLI_DIR)/Cargo.lock \
|
||||||
$(RUST_CLI_DIR)/src/lib.rs \
|
$(RUST_CLI_DIR)/src/lib.rs \
|
||||||
$(RUST_DIR)/src/timefn.rs $(RUST_DIR)/src/benchfn.rs
|
$(RUST_DIR)/src/timefn.rs $(RUST_DIR)/src/benchfn.rs \
|
||||||
|
$(RUST_DIR)/src/datagen.rs $(RUST_DIR)/src/lorem.rs
|
||||||
RUST_CLI_HELPERS_TARGET_DIR := $(RUST_DIR)/target/cli-helpers
|
RUST_CLI_HELPERS_TARGET_DIR := $(RUST_DIR)/target/cli-helpers
|
||||||
RUST_CLI_HELPERS_STATICLIB := $(RUST_CLI_HELPERS_TARGET_DIR)/release/libzstd_cli_rs.a
|
RUST_CLI_HELPERS_STATICLIB := $(RUST_CLI_HELPERS_TARGET_DIR)/release/libzstd_cli_rs.a
|
||||||
RUST_CLI_HELPERS_STATICLIB_32 := $(RUST_CLI_HELPERS_TARGET_DIR)/$(RUST_TARGET_32)/release/libzstd_cli_rs.a
|
RUST_CLI_HELPERS_STATICLIB_32 := $(RUST_CLI_HELPERS_TARGET_DIR)/$(RUST_TARGET_32)/release/libzstd_cli_rs.a
|
||||||
@@ -380,7 +381,8 @@ $(RUST_LINK_TARGETS) $(RUST_LINK_TARGETS_32): $(RUST_HUF_C_MODE_STAMP)
|
|||||||
# prerequisite so `$^` places it after every C object referencing its symbols.
|
# prerequisite so `$^` places it after every C object referencing its symbols.
|
||||||
RUST_CLI_LINK_TARGETS := fullbench fullbench-lib fullbench-dll fuzzer \
|
RUST_CLI_LINK_TARGETS := fullbench fullbench-lib fullbench-dll fuzzer \
|
||||||
zstreamtest zstreamtest_asan zstreamtest_tsan \
|
zstreamtest zstreamtest_asan zstreamtest_tsan \
|
||||||
zstreamtest_ubsan paramgrill decodecorpus poolTests
|
zstreamtest_ubsan paramgrill decodecorpus poolTests \
|
||||||
|
datagen largeDictionary
|
||||||
$(RUST_CLI_LINK_TARGETS): $(RUST_CLI_HELPERS_STATICLIB)
|
$(RUST_CLI_LINK_TARGETS): $(RUST_CLI_HELPERS_STATICLIB)
|
||||||
|
|
||||||
RUST_CLI_LINK_TARGETS_32 := fullbench32 fuzzer32 zstreamtest32
|
RUST_CLI_LINK_TARGETS_32 := fullbench32 fuzzer32 zstreamtest32
|
||||||
|
|||||||
Reference in New Issue
Block a user