diff --git a/programs/Makefile b/programs/Makefile index 36bf6c0db..de711edfa 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -33,7 +33,8 @@ RUST_SOURCES := $(RUST_MANIFEST) $(RUST_DIR)/Cargo.lock \ $(shell find $(RUST_DIR)/src -type f -name '*.rs' -print) RUST_CLI_SOURCES := $(RUST_CLI_MANIFEST) $(RUST_CLI_DIR)/Cargo.lock \ $(RUST_CLI_DIR)/src/lib.rs $(RUST_DIR)/src/zstd_cli.rs \ - $(RUST_DIR)/src/timefn.rs $(RUST_DIR)/src/benchfn.rs + $(RUST_DIR)/src/timefn.rs $(RUST_DIR)/src/benchfn.rs \ + $(RUST_DIR)/src/datagen.rs $(RUST_DIR)/src/lorem.rs # Keep Rust's HUF implementation in lockstep with libzstd.mk's C selection. # Forced modes may arrive as libzstd.mk variables or as direct -D flags in diff --git a/programs/datagen.c b/programs/datagen.c index ddc690bb1..ad817b37d 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -8,179 +8,8 @@ * You may select, at your option, one of the above-listed licenses. */ +/* The implementation lives in rust/src/datagen.rs, built into the Rust CLI + * static archive. This translation unit stays in the original source lists so + * build configuration keeps working while the implementation is in Rust. */ - -/*-************************************ -* Dependencies -**************************************/ #include "datagen.h" -#include "platform.h" /* SET_BINARY_MODE */ -#include /* malloc, free */ -#include /* FILE, fwrite, fprintf */ -#include /* memcpy */ -#include "../lib/common/mem.h" /* U32 */ - - -/*-************************************ -* Macros -**************************************/ -#define KB *(1 <<10) -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) - -#define RDG_DEBUG 0 -#define TRACE(...) if (RDG_DEBUG) fprintf(stderr, __VA_ARGS__ ) - - -/*-************************************ -* Local constants -**************************************/ -#define LTLOG 13 -#define LTSIZE (1<> (32 - r))) -static U32 RDG_rand(U32* src) -{ - static const U32 prime1 = 2654435761U; - static const U32 prime2 = 2246822519U; - U32 rand32 = *src; - rand32 *= prime1; - rand32 ^= prime2; - rand32 = RDG_rotl32(rand32, 13); - *src = rand32; - return rand32 >> 5; -} - -typedef U32 fixedPoint_24_8; - -static void RDG_fillLiteralDistrib(BYTE* ldt, fixedPoint_24_8 ld) -{ - BYTE const firstChar = (ld<=0.0) ? 0 : '('; - BYTE const lastChar = (ld<=0.0) ? 255 : '}'; - BYTE character = (ld<=0.0) ? 0 : '0'; - U32 u; - - if (ld<=0) ld = 0; - for (u=0; u> 8) + 1; - U32 const end = MIN ( u + weight , LTSIZE); - while (u < end) ldt[u++] = character; - character++; - if (character > lastChar) character = firstChar; - } -} - - -static BYTE RDG_genChar(U32* seed, const BYTE* ldt) -{ - U32 const id = RDG_rand(seed) & LTMASK; - return ldt[id]; /* memory-sanitizer fails here, stating "uninitialized value" when table initialized with P==0.0. Checked : table is fully initialized */ -} - - -static U32 RDG_rand15Bits (U32* seedPtr) -{ - return RDG_rand(seedPtr) & 0x7FFF; -} - -static U32 RDG_randLength(U32* seedPtr) -{ - if (RDG_rand(seedPtr) & 7) return (RDG_rand(seedPtr) & 0xF); /* small length */ - return (RDG_rand(seedPtr) & 0x1FF) + 0xF; -} - -static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, - double matchProba, const BYTE* ldt, U32* seedPtr) -{ - BYTE* const buffPtr = (BYTE*)buffer; - U32 const matchProba32 = (U32)(32768 * matchProba); - size_t pos = prefixSize; - U32 prevOffset = 1; - - /* special case : sparse content */ - while (matchProba >= 1.0) { - size_t size0 = RDG_rand(seedPtr) & 3; - size0 = (size_t)1 << (16 + size0 * 2); - size0 += RDG_rand(seedPtr) & (size0-1); /* because size0 is power of 2*/ - if (buffSize < pos + size0) { - memset(buffPtr+pos, 0, buffSize-pos); - return; - } - memset(buffPtr+pos, 0, size0); - pos += size0; - buffPtr[pos-1] = RDG_genChar(seedPtr, ldt); - continue; - } - - /* init */ - if (pos==0) buffPtr[0] = RDG_genChar(seedPtr, ldt), pos=1; - - /* Generate compressible data */ - while (pos < buffSize) { - /* Select : Literal (char) or Match (within 32K) */ - if (RDG_rand15Bits(seedPtr) < matchProba32) { - /* Copy (within 32K) */ - U32 const length = RDG_randLength(seedPtr) + 4; - U32 const d = (U32) MIN(pos + length , buffSize); - U32 const repeatOffset = (RDG_rand(seedPtr) & 15) == 2; - U32 const randOffset = RDG_rand15Bits(seedPtr) + 1; - U32 const offset = repeatOffset ? prevOffset : (U32) MIN(randOffset , pos); - size_t match = pos - offset; - while (pos < d) { buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */ } - prevOffset = offset; - } else { - /* Literal (noise) */ - U32 const length = RDG_randLength(seedPtr); - U32 const d = (U32) MIN(pos + length, buffSize); - while (pos < d) { buffPtr[pos++] = RDG_genChar(seedPtr, ldt); } - } } -} - - -void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed) -{ - U32 seed32 = seed; - BYTE ldt[LTSIZE]; - memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */ - if (litProba<=0.0) litProba = matchProba / 4.5; - RDG_fillLiteralDistrib(ldt, (fixedPoint_24_8)(litProba * 256 + 0.001)); - RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed32); -} - - -void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed) -{ - U32 seed32 = seed; - size_t const stdBlockSize = 128 KB; - size_t const stdDictSize = 32 KB; - BYTE* const buff = (BYTE*)malloc(stdDictSize + stdBlockSize); - U64 total = 0; - BYTE ldt[LTSIZE]; /* literals distribution table */ - - /* init */ - if (buff==NULL) { perror("datagen"); exit(1); } - if (litProba<=0.0) litProba = matchProba / 4.5; - memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */ - RDG_fillLiteralDistrib(ldt, (fixedPoint_24_8)(litProba * 256 + 0.001)); - SET_BINARY_MODE(stdout); - - /* Generate initial dict */ - RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed32); - - /* Generate compressible data */ - while (total < size) { - size_t const genBlockSize = (size_t) (MIN (stdBlockSize, size-total)); - RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed32); - total += genBlockSize; - { size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; } - /* update dict */ - memcpy(buff, buff + stdBlockSize, stdDictSize); - } - - /* cleanup */ - free(buff); -} diff --git a/programs/lorem.c b/programs/lorem.c index 79030c92f..f53c227d8 100644 --- a/programs/lorem.c +++ b/programs/lorem.c @@ -8,278 +8,8 @@ * You may select, at your option, one of the above-listed licenses. */ -/* Implementation notes: - * - * This is a very simple lorem ipsum generator - * which features a static list of words - * and print them one after another randomly - * with a fake sentence / paragraph structure. - * - * The goal is to generate a printable text - * that can be used to fake a text compression scenario. - * The resulting compression / ratio curve of the lorem ipsum generator - * is more satisfying than the previous statistical generator, - * which was initially designed for entropy compression, - * and lacks a regularity more representative of text. - * - * The compression ratio achievable on the generated lorem ipsum - * is still a bit too good, presumably because the dictionary is a bit too - * small. It would be possible to create some more complex scheme, notably by - * enlarging the dictionary with a word generator, and adding grammatical rules - * (composition) and syntax rules. But that's probably overkill for the intended - * goal. - */ +/* The implementation lives in rust/src/lorem.rs, built into the Rust CLI + * static archive. This translation unit stays in the original source lists so + * build configuration keeps working while the implementation is in Rust. */ #include "lorem.h" -#include -#include /* INT_MAX */ -#include /* memcpy */ - -#define WORD_MAX_SIZE 20 - -/* Define the word pool */ -static const char* kWords[] = { - "lorem", "ipsum", "dolor", "sit", "amet", - "consectetur", "adipiscing", "elit", "sed", "do", - "eiusmod", "tempor", "incididunt", "ut", "labore", - "et", "dolore", "magna", "aliqua", "dis", - "lectus", "vestibulum", "mattis", "ullamcorper", "velit", - "commodo", "a", "lacus", "arcu", "magnis", - "parturient", "montes", "nascetur", "ridiculus", "mus", - "mauris", "nulla", "malesuada", "pellentesque", "eget", - "gravida", "in", "dictum", "non", "erat", - "nam", "voluptat", "maecenas", "blandit", "aliquam", - "etiam", "enim", "lobortis", "scelerisque", "fermentum", - "dui", "faucibus", "ornare", "at", "elementum", - "eu", "facilisis", "odio", "morbi", "quis", - "eros", "donec", "ac", "orci", "purus", - "turpis", "cursus", "leo", "vel", "porta", - "consequat", "interdum", "varius", "vulputate", "aliquet", - "pharetra", "nunc", "auctor", "urna", "id", - "metus", "viverra", "nibh", "cras", "mi", - "unde", "omnis", "iste", "natus", "error", - "perspiciatis", "voluptatem", "accusantium", "doloremque", "laudantium", - "totam", "rem", "aperiam", "eaque", "ipsa", - "quae", "ab", "illo", "inventore", "veritatis", - "quasi", "architecto", "beatae", "vitae", "dicta", - "sunt", "explicabo", "nemo", "ipsam", "quia", - "voluptas", "aspernatur", "aut", "odit", "fugit", - "consequuntur", "magni", "dolores", "eos", "qui", - "ratione", "sequi", "nesciunt", "neque", "porro", - "quisquam", "est", "dolorem", "adipisci", "numquam", - "eius", "modi", "tempora", "incidunt", "magnam", - "quaerat", "ad", "minima", "veniam", "nostrum", - "ullam", "corporis", "suscipit", "laboriosam", "nisi", - "aliquid", "ex", "ea", "commodi", "consequatur", - "autem", "eum", "iure", "voluptate", "esse", - "quam", "nihil", "molestiae", "illum", "fugiat", - "quo", "pariatur", "vero", "accusamus", "iusto", - "dignissimos", "ducimus", "blanditiis", "praesentium", "voluptatum", - "deleniti", "atque", "corrupti", "quos", "quas", - "molestias", "excepturi", "sint", "occaecati", "cupiditate", - "provident", "similique", "culpa", "officia", "deserunt", - "mollitia", "animi", "laborum", "dolorum", "fuga", - "harum", "quidem", "rerum", "facilis", "expedita", - "distinctio", "libero", "tempore", "cum", "soluta", - "nobis", "eligendi", "optio", "cumque", "impedit", - "minus", "quod", "maxime", "placeat", "facere", - "possimus", "assumenda", "repellendus", "temporibus", "quibusdam", - "officiis", "debitis", "saepe", "eveniet", "voluptates", - "repudiandae", "recusandae", "itaque", "earum", "hic", - "tenetur", "sapiente", "delectus", "reiciendis", "cillum", - "maiores", "alias", "perferendis", "doloribus", "asperiores", - "repellat", "minim", "nostrud", "exercitation", "ullamco", - "laboris", "aliquip", "duis", "aute", "irure", -}; -static const unsigned kNbWords = sizeof(kWords) / sizeof(kWords[0]); - -/* simple 1-dimension distribution, based on word's length, favors small words - */ -static const int kWeights[] = { 0, 8, 6, 4, 3, 2 }; -static const size_t kNbWeights = sizeof(kWeights) / sizeof(kWeights[0]); - -#define DISTRIB_SIZE_MAX 650 -static int g_distrib[DISTRIB_SIZE_MAX] = { 0 }; -static unsigned g_distribCount = 0; - -static void countFreqs( - const char* words[], - size_t nbWords, - const int* weights, - size_t nbWeights) -{ - unsigned total = 0; - size_t w; - for (w = 0; w < nbWords; w++) { - size_t len = strlen(words[w]); - int lmax; - if (len >= nbWeights) - len = nbWeights - 1; - lmax = weights[len]; - total += (unsigned)lmax; - } - g_distribCount = total; - assert(g_distribCount <= DISTRIB_SIZE_MAX); -} - -static void init_word_distrib( - const char* words[], - size_t nbWords, - const int* weights, - size_t nbWeights) -{ - size_t w, d = 0; - countFreqs(words, nbWords, weights, nbWeights); - for (w = 0; w < nbWords; w++) { - size_t len = strlen(words[w]); - int l, lmax; - if (len >= nbWeights) - len = nbWeights - 1; - lmax = weights[len]; - for (l = 0; l < lmax; l++) { - g_distrib[d++] = (int)w; - } - } -} - -/* Note: this unit only works when invoked sequentially. - * No concurrent access is allowed */ -static char* g_ptr = NULL; -static size_t g_nbChars = 0; -static size_t g_maxChars = 10000000; -static unsigned g_randRoot = 0; - -#define RDG_rotl32(x, r) ((x << r) | (x >> (32 - r))) -static unsigned LOREM_rand(unsigned range) -{ - static const unsigned prime1 = 2654435761U; - static const unsigned prime2 = 2246822519U; - unsigned rand32 = g_randRoot; - rand32 *= prime1; - rand32 ^= prime2; - rand32 = RDG_rotl32(rand32, 13); - g_randRoot = rand32; - return (unsigned)(((unsigned long long)rand32 * range) >> 32); -} - -static void writeLastCharacters(void) -{ - size_t lastChars = g_maxChars - g_nbChars; - assert(g_maxChars >= g_nbChars); - if (lastChars == 0) - return; - g_ptr[g_nbChars++] = '.'; - if (lastChars > 2) { - memset(g_ptr + g_nbChars, ' ', lastChars - 2); - } - if (lastChars > 1) { - g_ptr[g_maxChars - 1] = '\n'; - } - g_nbChars = g_maxChars; -} - -static void generateWord(const char* word, const char* separator, int upCase) -{ - size_t const len = strlen(word) + strlen(separator); - if (g_nbChars + len > g_maxChars) { - writeLastCharacters(); - return; - } - memcpy(g_ptr + g_nbChars, word, strlen(word)); - if (upCase) { - static const char toUp = 'A' - 'a'; - g_ptr[g_nbChars] = (char)(g_ptr[g_nbChars] + toUp); - } - g_nbChars += strlen(word); - memcpy(g_ptr + g_nbChars, separator, strlen(separator)); - g_nbChars += strlen(separator); -} - -static int about(unsigned target) -{ - return (int)(LOREM_rand(target) + LOREM_rand(target) + 1); -} - -/* Function to generate a random sentence */ -static void generateSentence(int nbWords) -{ - int commaPos = about(9); - int comma2 = commaPos + about(7); - int qmark = (LOREM_rand(11) == 7); - const char* endSep = qmark ? "? " : ". "; - int i; - for (i = 0; i < nbWords; i++) { - int const wordID = g_distrib[LOREM_rand(g_distribCount)]; - const char* const word = kWords[wordID]; - const char* sep = " "; - if (i == commaPos) - sep = ", "; - if (i == comma2) - sep = ", "; - if (i == nbWords - 1) - sep = endSep; - generateWord(word, sep, i == 0); - } -} - -static void generateParagraph(int nbSentences) -{ - int i; - for (i = 0; i < nbSentences; i++) { - int wordsPerSentence = about(11); - generateSentence(wordsPerSentence); - } - if (g_nbChars < g_maxChars) { - g_ptr[g_nbChars++] = '\n'; - } - if (g_nbChars < g_maxChars) { - g_ptr[g_nbChars++] = '\n'; - } -} - -/* It's "common" for lorem ipsum generators to start with the same first - * pre-defined sentence */ -static void generateFirstSentence(void) -{ - int i; - for (i = 0; i < 18; i++) { - const char* word = kWords[i]; - const char* separator = " "; - if (i == 4) - separator = ", "; - if (i == 7) - separator = ", "; - generateWord(word, separator, i == 0); - } - generateWord(kWords[18], ". ", 0); -} - -size_t -LOREM_genBlock(void* buffer, size_t size, unsigned seed, int first, int fill) -{ - g_ptr = (char*)buffer; - assert(size < INT_MAX); - g_maxChars = size; - g_nbChars = 0; - g_randRoot = seed; - if (g_distribCount == 0) { - init_word_distrib(kWords, kNbWords, kWeights, kNbWeights); - } - - if (first) { - generateFirstSentence(); - } - while (g_nbChars < g_maxChars) { - int sentencePerParagraph = about(7); - generateParagraph(sentencePerParagraph); - if (!fill) - break; /* only generate one paragraph in not-fill mode */ - } - g_ptr = NULL; - return g_nbChars; -} - -void LOREM_genBuffer(void* buffer, size_t size, unsigned seed) -{ - LOREM_genBlock(buffer, size, seed, 1, 1); -} diff --git a/rust/cli/src/lib.rs b/rust/cli/src/lib.rs index 3b800f5c7..5a6740c29 100644 --- a/rust/cli/src/lib.rs +++ b/rust/cli/src/lib.rs @@ -1,5 +1,9 @@ #[path = "../../src/benchfn.rs"] mod benchfn; +#[path = "../../src/datagen.rs"] +mod datagen; +#[path = "../../src/lorem.rs"] +mod lorem; #[path = "../../src/timefn.rs"] mod timefn; #[cfg(feature = "cli")] diff --git a/tests/Makefile b/tests/Makefile index 52e589a69..e34c24aaf 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -110,7 +110,8 @@ RUST_CLI_DIR := $(RUST_DIR)/cli RUST_CLI_MANIFEST := $(RUST_CLI_DIR)/Cargo.toml RUST_CLI_HELPER_SOURCES := $(RUST_CLI_MANIFEST) $(RUST_CLI_DIR)/Cargo.lock \ $(RUST_CLI_DIR)/src/lib.rs \ - $(RUST_DIR)/src/timefn.rs $(RUST_DIR)/src/benchfn.rs + $(RUST_DIR)/src/timefn.rs $(RUST_DIR)/src/benchfn.rs \ + $(RUST_DIR)/src/datagen.rs $(RUST_DIR)/src/lorem.rs RUST_CLI_HELPERS_TARGET_DIR := $(RUST_DIR)/target/cli-helpers RUST_CLI_HELPERS_STATICLIB := $(RUST_CLI_HELPERS_TARGET_DIR)/release/libzstd_cli_rs.a RUST_CLI_HELPERS_STATICLIB_32 := $(RUST_CLI_HELPERS_TARGET_DIR)/$(RUST_TARGET_32)/release/libzstd_cli_rs.a @@ -380,7 +381,8 @@ $(RUST_LINK_TARGETS) $(RUST_LINK_TARGETS_32): $(RUST_HUF_C_MODE_STAMP) # prerequisite so `$^` places it after every C object referencing its symbols. RUST_CLI_LINK_TARGETS := fullbench fullbench-lib fullbench-dll fuzzer \ zstreamtest zstreamtest_asan zstreamtest_tsan \ - zstreamtest_ubsan paramgrill decodecorpus poolTests + zstreamtest_ubsan paramgrill decodecorpus poolTests \ + datagen largeDictionary $(RUST_CLI_LINK_TARGETS): $(RUST_CLI_HELPERS_STATICLIB) RUST_CLI_LINK_TARGETS_32 := fullbench32 fuzzer32 zstreamtest32