diff --git a/.gitignore b/.gitignore
index 048a4c4e4..34e18b443 100644
--- a/.gitignore
+++ b/.gitignore
@@ -39,12 +39,15 @@ buck-out/
build-*
*.gcda
+# IDE
+.clang_complete
+compile_flags.txt
+.clang-format
+
# Other files
.directory
_codelite/
_zstdbench/
-.clang_complete
-compile_flags.txt
*.idea
*.swp
.DS_Store
diff --git a/build/VS2010/datagen/datagen.vcxproj b/build/VS2010/datagen/datagen.vcxproj
index a66358a0d..aaba4788b 100644
--- a/build/VS2010/datagen/datagen.vcxproj
+++ b/build/VS2010/datagen/datagen.vcxproj
@@ -157,6 +157,8 @@
+
+
diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt
index 3226374a5..3ead07010 100644
--- a/build/cmake/tests/CMakeLists.txt
+++ b/build/cmake/tests/CMakeLists.txt
@@ -50,7 +50,7 @@ set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
set(TESTS_DIR ${ZSTD_SOURCE_DIR}/tests)
include_directories(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
-add_executable(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
+add_executable(datagen ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/lorem.c ${TESTS_DIR}/loremOut.c ${TESTS_DIR}/datagencli.c)
target_link_libraries(datagen libzstd_static)
#
diff --git a/build/meson/tests/meson.build b/build/meson/tests/meson.build
index 03480d205..9847ab037 100644
--- a/build/meson/tests/meson.build
+++ b/build/meson/tests/meson.build
@@ -44,7 +44,8 @@ testcommon_dep = declare_dependency(link_with: testcommon,
dependencies: libzstd_deps,
include_directories: libzstd_includes)
-datagen_sources = [join_paths(zstd_rootdir, 'tests/datagencli.c')]
+datagen_sources = [join_paths(zstd_rootdir, 'tests/datagencli.c'),
+ join_paths(zstd_rootdir, 'tests/loremOut.c')]
datagen = executable('datagen',
datagen_sources,
c_args: [ '-DNDEBUG' ],
diff --git a/lib/common/xxhash.h b/lib/common/xxhash.h
index 424ed19b8..e5ed3dc04 100644
--- a/lib/common/xxhash.h
+++ b/lib/common/xxhash.h
@@ -630,7 +630,6 @@ XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, X
* @brief The opaque state struct for the XXH32 streaming API.
*
* @see XXH32_state_s for details.
- * @see @ref streaming_example "Streaming Example"
*/
typedef struct XXH32_state_s XXH32_state_t;
@@ -641,8 +640,6 @@ typedef struct XXH32_state_s XXH32_state_t;
* @return `NULL` on failure.
*
* @note Must be freed with XXH32_freeState().
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void);
/*!
@@ -654,8 +651,6 @@ XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void);
*
* @note @p statePtr must be allocated with XXH32_createState().
*
- * @see @ref streaming_example "Streaming Example"
- *
*/
XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
/*!
@@ -681,8 +676,6 @@ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_
* @return @ref XXH_ERROR on failure.
*
* @note This function resets and seeds a state. Call it before @ref XXH32_update().
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed);
@@ -704,8 +697,6 @@ XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t
* @return @ref XXH_ERROR on failure.
*
* @note Call this to incrementally consume blocks of data.
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
@@ -722,8 +713,6 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void*
* @note
* Calling XXH32_digest() will not affect @p statePtr, so you can update,
* digest, and update again.
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
#endif /* !XXH_NO_STREAM */
@@ -900,7 +889,6 @@ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size
* @brief The opaque state struct for the XXH64 streaming API.
*
* @see XXH64_state_s for details.
- * @see @ref streaming_example "Streaming Example"
*/
typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
@@ -911,8 +899,6 @@ typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
* @return `NULL` on failure.
*
* @note Must be freed with XXH64_freeState().
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void);
@@ -924,8 +910,6 @@ XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void);
* @return @ref XXH_OK.
*
* @note @p statePtr must be allocated with XXH64_createState().
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
@@ -952,8 +936,6 @@ XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const
* @return @ref XXH_ERROR on failure.
*
* @note This function resets and seeds a state. Call it before @ref XXH64_update().
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed);
@@ -975,8 +957,6 @@ XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr,
* @return @ref XXH_ERROR on failure.
*
* @note Call this to incrementally consume blocks of data.
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
@@ -993,8 +973,6 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr,
* @note
* Calling XXH64_digest() will not affect @p statePtr, so you can update,
* digest, and update again.
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr);
#endif /* !XXH_NO_STREAM */
@@ -1199,7 +1177,6 @@ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const
* @brief The opaque state struct for the XXH3 streaming API.
*
* @see XXH3_state_s for details.
- * @see @ref streaming_example "Streaming Example"
*/
typedef struct XXH3_state_s XXH3_state_t;
XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void);
@@ -1231,8 +1208,6 @@ XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOE
* - Call this function before @ref XXH3_64bits_update().
* - Digest will be equivalent to `XXH3_64bits()`.
*
- * @see @ref streaming_example "Streaming Example"
- *
*/
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);
@@ -1253,8 +1228,6 @@ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* stateP
* - Call this function before @ref XXH3_64bits_update().
* - Digest will be equivalent to `XXH3_64bits_withSeed()`.
*
- * @see @ref streaming_example "Streaming Example"
- *
*/
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);
@@ -1279,8 +1252,6 @@ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_
* (secret's content should look like a bunch of random bytes).
* When in doubt about the randomness of a candidate `secret`,
* consider employing `XXH3_generateSecret()` instead (see below).
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);
@@ -1302,8 +1273,6 @@ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_stat
* @return @ref XXH_ERROR on failure.
*
* @note Call this to incrementally consume blocks of data.
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
@@ -1320,8 +1289,6 @@ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* stat
* @note
* Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update,
* digest, and update again.
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr);
#endif /* !XXH_NO_STREAM */
@@ -1442,8 +1409,6 @@ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE cons
* - This function resets `statePtr` and generate a secret with default parameters.
* - Call it before @ref XXH3_128bits_update().
* - Digest will be equivalent to `XXH3_128bits()`.
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);
@@ -1463,8 +1428,6 @@ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* state
* - This function resets `statePtr` and generate a secret from `seed`.
* - Call it before @ref XXH3_128bits_update().
* - Digest will be equivalent to `XXH3_128bits_withSeed()`.
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);
/*!
@@ -1486,8 +1449,6 @@ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state
* (secret's content should look like a bunch of random bytes).
* When in doubt about the randomness of a candidate `secret`,
* consider employing `XXH3_generateSecret()` instead (see below).
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);
@@ -6072,8 +6033,6 @@ static void XXH_alignedFree(void* p)
* @return `NULL` on failure.
*
* @note Must be freed with XXH3_freeState().
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void)
{
@@ -6092,8 +6051,6 @@ XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void)
* @return @ref XXH_OK.
*
* @note Must be allocated with XXH3_createState().
- *
- * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr)
{
diff --git a/programs/benchzstd.c b/programs/benchzstd.c
index b3af4c332..32227669f 100644
--- a/programs/benchzstd.c
+++ b/programs/benchzstd.c
@@ -8,198 +8,283 @@
* You may select, at your option, one of the above-listed licenses.
*/
-
/* **************************************
-* Tuning parameters
-****************************************/
-#ifndef BMK_TIMETEST_DEFAULT_S /* default minimum time per test */
-# define BMK_TIMETEST_DEFAULT_S 3
+ * Tuning parameters
+ ****************************************/
+#ifndef BMK_TIMETEST_DEFAULT_S /* default minimum time per test */
+# define BMK_TIMETEST_DEFAULT_S 3
#endif
-
/* *************************************
-* Includes
-***************************************/
-#include "platform.h" /* Large Files support */
-#include "util.h" /* UTIL_getFileSize, UTIL_sleep */
-#include /* malloc, free */
-#include /* memset, strerror */
-#include /* fprintf, fopen */
-#include
-#include /* assert */
+ * Includes
+ ***************************************/
+/* this must be included first */
+#include "platform.h" /* Large Files support, compiler specifics */
-#include "timefn.h" /* UTIL_time_t */
-#include "benchfn.h"
+/* then following system includes */
+#include /* assert */
+#include
+#include /* fprintf, fopen */
+#include /* malloc, free */
+#include /* memset, strerror */
+#include "util.h" /* UTIL_getFileSize, UTIL_sleep */
#include "../lib/common/mem.h"
+#include "benchfn.h"
+#include "timefn.h" /* UTIL_time_t */
#ifndef ZSTD_STATIC_LINKING_ONLY
-# define ZSTD_STATIC_LINKING_ONLY
+# define ZSTD_STATIC_LINKING_ONLY
#endif
#include "../lib/zstd.h"
-#include "datagen.h" /* RDG_genBuffer */
-#include "lorem.h" /* LOREM_genBuffer */
+#include "datagen.h" /* RDG_genBuffer */
+#include "lorem.h" /* LOREM_genBuffer */
#ifndef XXH_INLINE_ALL
-# define XXH_INLINE_ALL
+# define XXH_INLINE_ALL
#endif
#include "../lib/common/xxhash.h"
-#include "benchzstd.h"
#include "../lib/zstd_errors.h"
-
+#include "benchzstd.h"
/* *************************************
-* Constants
-***************************************/
+ * Constants
+ ***************************************/
#ifndef ZSTD_GIT_COMMIT
-# define ZSTD_GIT_COMMIT_STRING ""
+# define ZSTD_GIT_COMMIT_STRING ""
#else
-# define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT)
+# define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT)
#endif
-#define TIMELOOP_MICROSEC (1*1000000ULL) /* 1 second */
-#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */
-#define ACTIVEPERIOD_MICROSEC (70*TIMELOOP_MICROSEC) /* 70 seconds */
-#define COOLPERIOD_SEC 10
+#define TIMELOOP_MICROSEC (1 * 1000000ULL) /* 1 second */
+#define TIMELOOP_NANOSEC (1 * 1000000000ULL) /* 1 second */
+#define ACTIVEPERIOD_MICROSEC (70 * TIMELOOP_MICROSEC) /* 70 seconds */
+#define COOLPERIOD_SEC 10
-#define KB *(1 <<10)
-#define MB *(1 <<20)
-#define GB *(1U<<30)
+#define KB *(1 << 10)
+#define MB *(1 << 20)
+#define GB *(1U << 30)
#define BMK_RUNTEST_DEFAULT_MS 1000
-static const size_t maxMemory = (sizeof(size_t)==4) ?
- /* 32-bit */ (2 GB - 64 MB) :
- /* 64-bit */ (size_t)(1ULL << ((sizeof(size_t)*8)-31));
-
+static const size_t maxMemory = (sizeof(size_t) == 4)
+ ?
+ /* 32-bit */ (2 GB - 64 MB)
+ :
+ /* 64-bit */ (size_t)(1ULL << ((sizeof(size_t) * 8) - 31));
/* *************************************
-* console display
-***************************************/
-#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush(NULL); }
-#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
-/* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
-#define OUTPUT(...) { fprintf(stdout, __VA_ARGS__); fflush(NULL); }
-#define OUTPUTLEVEL(l, ...) if (displayLevel>=l) { OUTPUT(__VA_ARGS__); }
-
+ * console display
+ ***************************************/
+#define DISPLAY(...) \
+ { \
+ fprintf(stderr, __VA_ARGS__); \
+ fflush(NULL); \
+ }
+#define DISPLAYLEVEL(l, ...) \
+ if (displayLevel >= l) { \
+ DISPLAY(__VA_ARGS__); \
+ }
+/* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : +
+ * progression; 4 : + information */
+#define OUTPUT(...) \
+ { \
+ fprintf(stdout, __VA_ARGS__); \
+ fflush(NULL); \
+ }
+#define OUTPUTLEVEL(l, ...) \
+ if (displayLevel >= l) { \
+ OUTPUT(__VA_ARGS__); \
+ }
/* *************************************
-* Exceptions
-***************************************/
+ * Exceptions
+ ***************************************/
#ifndef DEBUG
-# define DEBUG 0
+# define DEBUG 0
#endif
-#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
+#define DEBUGOUTPUT(...) \
+ { \
+ if (DEBUG) \
+ DISPLAY(__VA_ARGS__); \
+ }
-#define RETURN_ERROR_INT(errorNum, ...) { \
- DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
- DISPLAYLEVEL(1, "Error %i : ", errorNum); \
- DISPLAYLEVEL(1, __VA_ARGS__); \
- DISPLAYLEVEL(1, " \n"); \
- return errorNum; \
+#define RETURN_ERROR_INT(errorNum, ...) \
+ { \
+ DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
+ DISPLAYLEVEL(1, "Error %i : ", errorNum); \
+ DISPLAYLEVEL(1, __VA_ARGS__); \
+ DISPLAYLEVEL(1, " \n"); \
+ return errorNum; \
+ }
+
+#define CHECK_Z(zf) \
+ { \
+ size_t const zerr = zf; \
+ if (ZSTD_isError(zerr)) { \
+ DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
+ DISPLAY("Error : "); \
+ DISPLAY("%s failed : %s", #zf, ZSTD_getErrorName(zerr)); \
+ DISPLAY(" \n"); \
+ exit(1); \
+ } \
+ }
+
+#define RETURN_ERROR(errorNum, retType, ...) \
+ { \
+ retType r; \
+ memset(&r, 0, sizeof(retType)); \
+ DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
+ DISPLAYLEVEL(1, "Error %i : ", errorNum); \
+ DISPLAYLEVEL(1, __VA_ARGS__); \
+ DISPLAYLEVEL(1, " \n"); \
+ r.tag = errorNum; \
+ return r; \
+ }
+
+/* replacement for snprintf(), which is not supported by C89
+ * sprintf() would be the supported one, but it's labelled unsafe,
+ * so some modern static analyzer will flag it as such, making it unusable.
+ * formatString_u() replaces snprintf() for the specific case where there are only %u arguments */
+static int formatString_u(char* buffer, size_t buffer_size, const char* formatString, unsigned int value)
+{
+ size_t written = 0;
+ int i;
+ assert(value <= 100);
+
+ for (i = 0; formatString[i] != '\0' && written < buffer_size - 1; ++i) {
+ if (formatString[i] != '%') {
+ buffer[written++] = formatString[i];
+ continue;
+ }
+
+ if (formatString[++i] == 'u') {
+ /* Handle single digit */
+ if (value < 10) {
+ buffer[written++] = '0' + (char)value;
+ } else if (value < 100) {
+ /* Handle two digits */
+ if (written >= buffer_size - 2) {
+ return -1; /* buffer overflow */
+ }
+ buffer[written++] = '0' + (char)(value / 10);
+ buffer[written++] = '0' + (char)(value % 10);
+ } else { /* 100 */
+ if (written >= buffer_size - 3) {
+ return -1; /* buffer overflow */
+ }
+ buffer[written++] = '1';
+ buffer[written++] = '0';
+ buffer[written++] = '0';
+ }
+ } else if (formatString[i] == '%') { /* Check for escaped percent sign */
+ buffer[written++] = '%';
+ } else {
+ return -1; /* unsupported format */
+ }
+ }
+
+ if (written < buffer_size) {
+ buffer[written] = '\0';
+ } else {
+ buffer[0] = '\0'; /* Handle truncation */
+ }
+
+ return (int)written;
}
-#define CHECK_Z(zf) { \
- size_t const zerr = zf; \
- if (ZSTD_isError(zerr)) { \
- DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
- DISPLAY("Error : "); \
- DISPLAY("%s failed : %s", \
- #zf, ZSTD_getErrorName(zerr)); \
- DISPLAY(" \n"); \
- exit(1); \
- } \
-}
-
-#define RETURN_ERROR(errorNum, retType, ...) { \
- retType r; \
- memset(&r, 0, sizeof(retType)); \
- DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
- DISPLAYLEVEL(1, "Error %i : ", errorNum); \
- DISPLAYLEVEL(1, __VA_ARGS__); \
- DISPLAYLEVEL(1, " \n"); \
- r.tag = errorNum; \
- return r; \
-}
-
-
/* *************************************
-* Benchmark Parameters
-***************************************/
+ * Benchmark Parameters
+ ***************************************/
-BMK_advancedParams_t BMK_initAdvancedParams(void) {
+BMK_advancedParams_t BMK_initAdvancedParams(void)
+{
BMK_advancedParams_t const res = {
- BMK_both, /* mode */
+ BMK_both, /* mode */
BMK_TIMETEST_DEFAULT_S, /* nbSeconds */
- 0, /* blockSize */
- 0, /* nbWorkers */
- 0, /* realTime */
- 0, /* additionalParam */
- 0, /* ldmFlag */
- 0, /* ldmMinMatch */
- 0, /* ldmHashLog */
- 0, /* ldmBuckSizeLog */
- 0, /* ldmHashRateLog */
- ZSTD_ps_auto, /* literalCompressionMode */
- 0 /* useRowMatchFinder */
+ 0, /* blockSize */
+ 0, /* nbWorkers */
+ 0, /* realTime */
+ 0, /* additionalParam */
+ 0, /* ldmFlag */
+ 0, /* ldmMinMatch */
+ 0, /* ldmHashLog */
+ 0, /* ldmBuckSizeLog */
+ 0, /* ldmHashRateLog */
+ ZSTD_ps_auto, /* literalCompressionMode */
+ 0 /* useRowMatchFinder */
};
return res;
}
-
/* ********************************************************
-* Bench functions
-**********************************************************/
+ * Bench functions
+ **********************************************************/
typedef struct {
const void* srcPtr;
size_t srcSize;
- void* cPtr;
+ void* cPtr;
size_t cRoom;
size_t cSize;
- void* resPtr;
+ void* resPtr;
size_t resSize;
} blockParam_t;
#undef MIN
#undef MAX
-#define MIN(a,b) ((a) < (b) ? (a) : (b))
-#define MAX(a,b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
-static void
-BMK_initCCtx(ZSTD_CCtx* ctx,
- const void* dictBuffer, size_t dictBufferSize,
- int cLevel,
- const ZSTD_compressionParameters* comprParams,
- const BMK_advancedParams_t* adv)
+static void BMK_initCCtx(
+ ZSTD_CCtx* ctx,
+ const void* dictBuffer,
+ size_t dictBufferSize,
+ int cLevel,
+ const ZSTD_compressionParameters* comprParams,
+ const BMK_advancedParams_t* adv)
{
ZSTD_CCtx_reset(ctx, ZSTD_reset_session_and_parameters);
- if (adv->nbWorkers==1) {
+ if (adv->nbWorkers == 1) {
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, 0));
} else {
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, adv->nbWorkers));
}
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, cLevel));
- CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_useRowMatchFinder, adv->useRowMatchFinder));
- CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_enableLongDistanceMatching, adv->ldmFlag));
+ CHECK_Z(ZSTD_CCtx_setParameter(
+ ctx, ZSTD_c_useRowMatchFinder, adv->useRowMatchFinder));
+ CHECK_Z(ZSTD_CCtx_setParameter(
+ ctx, ZSTD_c_enableLongDistanceMatching, adv->ldmFlag));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmMinMatch, adv->ldmMinMatch));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashLog, adv->ldmHashLog));
- CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmBucketSizeLog, adv->ldmBucketSizeLog));
- CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashRateLog, adv->ldmHashRateLog));
- CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, (int)comprParams->windowLog));
- CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, (int)comprParams->hashLog));
- CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, (int)comprParams->chainLog));
- CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, (int)comprParams->searchLog));
- CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, (int)comprParams->minMatch));
- CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, (int)comprParams->targetLength));
- CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_literalCompressionMode, (int)adv->literalCompressionMode));
- CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, (int)comprParams->strategy));
+ CHECK_Z(ZSTD_CCtx_setParameter(
+ ctx, ZSTD_c_ldmBucketSizeLog, adv->ldmBucketSizeLog));
+ CHECK_Z(ZSTD_CCtx_setParameter(
+ ctx, ZSTD_c_ldmHashRateLog, adv->ldmHashRateLog));
+ CHECK_Z(ZSTD_CCtx_setParameter(
+ ctx, ZSTD_c_windowLog, (int)comprParams->windowLog));
+ CHECK_Z(ZSTD_CCtx_setParameter(
+ ctx, ZSTD_c_hashLog, (int)comprParams->hashLog));
+ CHECK_Z(ZSTD_CCtx_setParameter(
+ ctx, ZSTD_c_chainLog, (int)comprParams->chainLog));
+ CHECK_Z(ZSTD_CCtx_setParameter(
+ ctx, ZSTD_c_searchLog, (int)comprParams->searchLog));
+ CHECK_Z(ZSTD_CCtx_setParameter(
+ ctx, ZSTD_c_minMatch, (int)comprParams->minMatch));
+ CHECK_Z(ZSTD_CCtx_setParameter(
+ ctx, ZSTD_c_targetLength, (int)comprParams->targetLength));
+ CHECK_Z(ZSTD_CCtx_setParameter(
+ ctx,
+ ZSTD_c_literalCompressionMode,
+ (int)adv->literalCompressionMode));
+ CHECK_Z(ZSTD_CCtx_setParameter(
+ ctx, ZSTD_c_strategy, (int)comprParams->strategy));
CHECK_Z(ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize));
}
-static void BMK_initDCtx(ZSTD_DCtx* dctx,
- const void* dictBuffer, size_t dictBufferSize) {
+static void
+BMK_initDCtx(ZSTD_DCtx* dctx, const void* dictBuffer, size_t dictBufferSize)
+{
CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters));
CHECK_Z(ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize));
}
-
typedef struct {
ZSTD_CCtx* cctx;
const void* dictBuffer;
@@ -209,9 +294,16 @@ typedef struct {
const BMK_advancedParams_t* adv;
} BMK_initCCtxArgs;
-static size_t local_initCCtx(void* payload) {
+static size_t local_initCCtx(void* payload)
+{
BMK_initCCtxArgs* ag = (BMK_initCCtxArgs*)payload;
- BMK_initCCtx(ag->cctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv);
+ BMK_initCCtx(
+ ag->cctx,
+ ag->dictBuffer,
+ ag->dictBufferSize,
+ ag->cLevel,
+ ag->comprParams,
+ ag->adv);
return 0;
}
@@ -221,18 +313,20 @@ typedef struct {
size_t dictBufferSize;
} BMK_initDCtxArgs;
-static size_t local_initDCtx(void* payload) {
+static size_t local_initDCtx(void* payload)
+{
BMK_initDCtxArgs* ag = (BMK_initDCtxArgs*)payload;
BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize);
return 0;
}
-
/* `addArgs` is the context */
static size_t local_defaultCompress(
- const void* srcBuffer, size_t srcSize,
- void* dstBuffer, size_t dstSize,
- void* addArgs)
+ const void* srcBuffer,
+ size_t srcSize,
+ void* dstBuffer,
+ size_t dstSize,
+ void* addArgs)
{
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)addArgs;
return ZSTD_compress2(cctx, dstBuffer, dstSize, srcBuffer, srcSize);
@@ -240,18 +334,24 @@ static size_t local_defaultCompress(
/* `addArgs` is the context */
static size_t local_defaultDecompress(
- const void* srcBuffer, size_t srcSize,
- void* dstBuffer, size_t dstCapacity,
- void* addArgs)
+ const void* srcBuffer,
+ size_t srcSize,
+ void* dstBuffer,
+ size_t dstCapacity,
+ void* addArgs)
{
- size_t moreToFlush = 1;
+ size_t moreToFlush = 1;
ZSTD_DCtx* const dctx = (ZSTD_DCtx*)addArgs;
ZSTD_inBuffer in;
ZSTD_outBuffer out;
- in.src = srcBuffer; in.size = srcSize; in.pos = 0;
- out.dst = dstBuffer; out.size = dstCapacity; out.pos = 0;
+ in.src = srcBuffer;
+ in.size = srcSize;
+ in.pos = 0;
+ out.dst = dstBuffer;
+ out.size = dstCapacity;
+ out.pos = 0;
while (moreToFlush) {
- if(out.pos == out.size) {
+ if (out.pos == out.size) {
return (size_t)-ZSTD_error_dstSize_tooSmall;
}
moreToFlush = ZSTD_decompressStream(dctx, &out, &in);
@@ -260,10 +360,8 @@ static size_t local_defaultDecompress(
}
}
return out.pos;
-
}
-
/* ================================================================= */
/* Benchmark Zstandard, mem-to-mem scenarios */
/* ================================================================= */
@@ -287,104 +385,145 @@ static BMK_benchOutcome_t BMK_benchOutcome_error(void)
return b;
}
-static BMK_benchOutcome_t BMK_benchOutcome_setValidResult(BMK_benchResult_t result)
+static BMK_benchOutcome_t BMK_benchOutcome_setValidResult(
+ BMK_benchResult_t result)
{
BMK_benchOutcome_t b;
- b.tag = 0;
+ b.tag = 0;
b.internal_never_use_directly = result;
return b;
}
-
/* benchMem with no allocation */
-static BMK_benchOutcome_t
-BMK_benchMemAdvancedNoAlloc(
- const void** srcPtrs, size_t* srcSizes,
- void** cPtrs, size_t* cCapacities, size_t* cSizes,
- void** resPtrs, size_t* resSizes,
- void** resultBufferPtr, void* compressedBuffer,
- size_t maxCompressedSize,
- BMK_timedFnState_t* timeStateCompress,
- BMK_timedFnState_t* timeStateDecompress,
+static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
+ const void** srcPtrs,
+ size_t* srcSizes,
+ void** cPtrs,
+ size_t* cCapacities,
+ size_t* cSizes,
+ void** resPtrs,
+ size_t* resSizes,
+ void** resultBufferPtr,
+ void* compressedBuffer,
+ size_t maxCompressedSize,
+ BMK_timedFnState_t* timeStateCompress,
+ BMK_timedFnState_t* timeStateDecompress,
- const void* srcBuffer, size_t srcSize,
- const size_t* fileSizes, unsigned nbFiles,
- const int cLevel,
- const ZSTD_compressionParameters* comprParams,
- const void* dictBuffer, size_t dictBufferSize,
- ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
- int displayLevel, const char* displayName,
- const BMK_advancedParams_t* adv)
+ const void* srcBuffer,
+ size_t srcSize,
+ const size_t* fileSizes,
+ unsigned nbFiles,
+ const int cLevel,
+ const ZSTD_compressionParameters* comprParams,
+ const void* dictBuffer,
+ size_t dictBufferSize,
+ ZSTD_CCtx* cctx,
+ ZSTD_DCtx* dctx,
+ int displayLevel,
+ const char* displayName,
+ const BMK_advancedParams_t* adv)
{
- size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
+ size_t const blockSize =
+ ((adv->blockSize >= 32 && (adv->mode != BMK_decodeOnly))
+ ? adv->blockSize
+ : srcSize)
+ + (!srcSize); /* avoid div by 0 */
BMK_benchResult_t benchResult;
size_t const loadedCompressedSize = srcSize;
- size_t cSize = 0;
- double ratio = 0.;
+ size_t cSize = 0;
+ double ratio = 0.;
U32 nbBlocks;
- assert(cctx != NULL); assert(dctx != NULL);
+ assert(cctx != NULL);
+ assert(dctx != NULL);
/* init */
memset(&benchResult, 0, sizeof(benchResult));
- if (strlen(displayName)>17) displayName += strlen(displayName) - 17; /* display last 17 characters */
+ if (strlen(displayName) > 17)
+ displayName +=
+ strlen(displayName) - 17; /* display last 17 characters */
if (adv->mode == BMK_decodeOnly) {
/* benchmark only decompression : source must be already compressed */
const char* srcPtr = (const char*)srcBuffer;
- U64 totalDSize64 = 0;
+ U64 totalDSize64 = 0;
U32 fileNb;
- for (fileNb=0; fileNb decodedSize) { /* size_t overflow */
- RETURN_ERROR(32, BMK_benchOutcome_t, "decompressed size is too large for local system");
+ if (totalDSize64 > decodedSize) { /* size_t overflow */
+ RETURN_ERROR(
+ 32,
+ BMK_benchOutcome_t,
+ "decompressed size is too large for local system");
}
*resultBufferPtr = malloc(decodedSize);
if (!(*resultBufferPtr)) {
- RETURN_ERROR(33, BMK_benchOutcome_t, "allocation error: not enough memory");
+ RETURN_ERROR(
+ 33,
+ BMK_benchOutcome_t,
+ "allocation error: not enough memory");
}
- cSize = srcSize;
+ cSize = srcSize;
srcSize = decodedSize;
- ratio = (double)srcSize / (double)cSize;
+ ratio = (double)srcSize / (double)cSize;
}
}
/* Init data blocks */
- { const char* srcPtr = (const char*)srcBuffer;
- char* cPtr = (char*)compressedBuffer;
- char* resPtr = (char*)(*resultBufferPtr);
+ {
+ const char* srcPtr = (const char*)srcBuffer;
+ char* cPtr = (char*)compressedBuffer;
+ char* resPtr = (char*)(*resultBufferPtr);
U32 fileNb;
- for (nbBlocks=0, fileNb=0; fileNbmode == BMK_decodeOnly) ? 1 : (U32)((remaining + (blockSize-1)) / blockSize);
- U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
- for ( ; nbBlocksmode == BMK_decodeOnly)
+ ? 1
+ : (U32)((remaining + (blockSize - 1)) / blockSize);
+ U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
+ for (; nbBlocks < blockEnd; nbBlocks++) {
size_t const thisBlockSize = MIN(remaining, blockSize);
- srcPtrs[nbBlocks] = srcPtr;
- srcSizes[nbBlocks] = thisBlockSize;
- cPtrs[nbBlocks] = cPtr;
- cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
- resPtrs[nbBlocks] = resPtr;
- resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
+ srcPtrs[nbBlocks] = srcPtr;
+ srcSizes[nbBlocks] = thisBlockSize;
+ cPtrs[nbBlocks] = cPtr;
+ cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly)
+ ? thisBlockSize
+ : ZSTD_compressBound(thisBlockSize);
+ resPtrs[nbBlocks] = resPtr;
+ resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly)
+ ? (size_t)ZSTD_findDecompressedSize(
+ srcPtr, thisBlockSize)
+ : thisBlockSize;
srcPtr += thisBlockSize;
cPtr += cCapacities[nbBlocks];
resPtr += thisBlockSize;
remaining -= thisBlockSize;
if (adv->mode == BMK_decodeOnly) {
- cSizes[nbBlocks] = thisBlockSize;
+ cSizes[nbBlocks] = thisBlockSize;
benchResult.cSize = thisBlockSize;
- } } } }
+ }
+ }
+ }
+ }
/* warming up `compressedBuffer` */
if (adv->mode == BMK_decodeOnly) {
@@ -394,236 +533,329 @@ BMK_benchMemAdvancedNoAlloc(
}
if (!UTIL_support_MT_measurements() && adv->nbWorkers > 1) {
- OUTPUTLEVEL(2, "Warning : time measurements may be incorrect in multithreading mode... \n")
+ OUTPUTLEVEL(
+ 2,
+ "Warning : time measurements may be incorrect in multithreading mode... \n")
}
/* Bench */
- { U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0);
-# define NB_MARKS 4
+ {
+ U64 const crcOrig = (adv->mode == BMK_decodeOnly)
+ ? 0
+ : XXH64(srcBuffer, srcSize, 0);
+#define NB_MARKS 4
const char* marks[NB_MARKS] = { " |", " /", " =", " \\" };
- U32 markNb = 0;
- int compressionCompleted = (adv->mode == BMK_decodeOnly);
- int decompressionCompleted = (adv->mode == BMK_compressOnly);
+ U32 markNb = 0;
+ int compressionCompleted = (adv->mode == BMK_decodeOnly);
+ int decompressionCompleted = (adv->mode == BMK_compressOnly);
BMK_benchParams_t cbp, dbp;
BMK_initCCtxArgs cctxprep;
BMK_initDCtxArgs dctxprep;
- cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */
- cbp.benchPayload = cctx;
- cbp.initFn = local_initCCtx; /* BMK_initCCtx */
- cbp.initPayload = &cctxprep;
- cbp.errorFn = ZSTD_isError;
- cbp.blockCount = nbBlocks;
- cbp.srcBuffers = srcPtrs;
- cbp.srcSizes = srcSizes;
- cbp.dstBuffers = cPtrs;
+ cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */
+ cbp.benchPayload = cctx;
+ cbp.initFn = local_initCCtx; /* BMK_initCCtx */
+ cbp.initPayload = &cctxprep;
+ cbp.errorFn = ZSTD_isError;
+ cbp.blockCount = nbBlocks;
+ cbp.srcBuffers = srcPtrs;
+ cbp.srcSizes = srcSizes;
+ cbp.dstBuffers = cPtrs;
cbp.dstCapacities = cCapacities;
- cbp.blockResults = cSizes;
+ cbp.blockResults = cSizes;
- cctxprep.cctx = cctx;
- cctxprep.dictBuffer = dictBuffer;
+ cctxprep.cctx = cctx;
+ cctxprep.dictBuffer = dictBuffer;
cctxprep.dictBufferSize = dictBufferSize;
- cctxprep.cLevel = cLevel;
- cctxprep.comprParams = comprParams;
- cctxprep.adv = adv;
+ cctxprep.cLevel = cLevel;
+ cctxprep.comprParams = comprParams;
+ cctxprep.adv = adv;
- dbp.benchFn = local_defaultDecompress;
- dbp.benchPayload = dctx;
- dbp.initFn = local_initDCtx;
- dbp.initPayload = &dctxprep;
- dbp.errorFn = ZSTD_isError;
- dbp.blockCount = nbBlocks;
- dbp.srcBuffers = (const void* const *) cPtrs;
- dbp.srcSizes = cSizes;
- dbp.dstBuffers = resPtrs;
+ dbp.benchFn = local_defaultDecompress;
+ dbp.benchPayload = dctx;
+ dbp.initFn = local_initDCtx;
+ dbp.initPayload = &dctxprep;
+ dbp.errorFn = ZSTD_isError;
+ dbp.blockCount = nbBlocks;
+ dbp.srcBuffers = (const void* const*)cPtrs;
+ dbp.srcSizes = cSizes;
+ dbp.dstBuffers = resPtrs;
dbp.dstCapacities = resSizes;
- dbp.blockResults = NULL;
+ dbp.blockResults = NULL;
- dctxprep.dctx = dctx;
- dctxprep.dictBuffer = dictBuffer;
+ dctxprep.dctx = dctx;
+ dctxprep.dictBuffer = dictBuffer;
dctxprep.dictBufferSize = dictBufferSize;
- OUTPUTLEVEL(2, "\r%70s\r", ""); /* blank line */
+ OUTPUTLEVEL(2, "\r%70s\r", ""); /* blank line */
assert(srcSize < UINT_MAX);
- OUTPUTLEVEL(2, "%2s-%-17.17s :%10u -> \r", marks[markNb], displayName, (unsigned)srcSize);
+ OUTPUTLEVEL(
+ 2,
+ "%2s-%-17.17s :%10u -> \r",
+ marks[markNb],
+ displayName,
+ (unsigned)srcSize);
while (!(compressionCompleted && decompressionCompleted)) {
if (!compressionCompleted) {
- BMK_runOutcome_t const cOutcome = BMK_benchTimedFn( timeStateCompress, cbp);
+ BMK_runOutcome_t const cOutcome =
+ BMK_benchTimedFn(timeStateCompress, cbp);
if (!BMK_isSuccessful_runOutcome(cOutcome)) {
RETURN_ERROR(30, BMK_benchOutcome_t, "compression error");
}
- { BMK_runTime_t const cResult = BMK_extract_runTime(cOutcome);
- cSize = cResult.sumOfReturn;
+ {
+ BMK_runTime_t const cResult = BMK_extract_runTime(cOutcome);
+ cSize = cResult.sumOfReturn;
ratio = (double)srcSize / (double)cSize;
- { BMK_benchResult_t newResult;
- newResult.cSpeed = (U64)((double)srcSize * TIMELOOP_NANOSEC / cResult.nanoSecPerRun);
+ {
+ BMK_benchResult_t newResult;
+ newResult.cSpeed =
+ (U64)((double)srcSize * TIMELOOP_NANOSEC
+ / cResult.nanoSecPerRun);
benchResult.cSize = cSize;
if (newResult.cSpeed > benchResult.cSpeed)
benchResult.cSpeed = newResult.cSpeed;
- } }
-
- { int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
- assert(cSize < UINT_MAX);
- OUTPUTLEVEL(2, "%2s-%-17.17s :%10u ->%10u (x%5.*f), %6.*f MB/s \r",
- marks[markNb], displayName,
- (unsigned)srcSize, (unsigned)cSize,
- ratioAccuracy, ratio,
- benchResult.cSpeed < (10 * MB_UNIT) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT);
+ }
}
- compressionCompleted = BMK_isCompleted_TimedFn(timeStateCompress);
+
+ {
+ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
+ assert(cSize < UINT_MAX);
+ OUTPUTLEVEL(
+ 2,
+ "%2s-%-17.17s :%10u ->%10u (x%5.*f), %6.*f MB/s \r",
+ marks[markNb],
+ displayName,
+ (unsigned)srcSize,
+ (unsigned)cSize,
+ ratioAccuracy,
+ ratio,
+ benchResult.cSpeed < (10 * MB_UNIT) ? 2 : 1,
+ (double)benchResult.cSpeed / MB_UNIT);
+ }
+ compressionCompleted =
+ BMK_isCompleted_TimedFn(timeStateCompress);
}
- if(!decompressionCompleted) {
- BMK_runOutcome_t const dOutcome = BMK_benchTimedFn(timeStateDecompress, dbp);
+ if (!decompressionCompleted) {
+ BMK_runOutcome_t const dOutcome =
+ BMK_benchTimedFn(timeStateDecompress, dbp);
- if(!BMK_isSuccessful_runOutcome(dOutcome)) {
+ if (!BMK_isSuccessful_runOutcome(dOutcome)) {
RETURN_ERROR(30, BMK_benchOutcome_t, "decompression error");
}
- { BMK_runTime_t const dResult = BMK_extract_runTime(dOutcome);
- U64 const newDSpeed = (U64)((double)srcSize * TIMELOOP_NANOSEC / dResult.nanoSecPerRun);
+ {
+ BMK_runTime_t const dResult = BMK_extract_runTime(dOutcome);
+ U64 const newDSpeed =
+ (U64)((double)srcSize * TIMELOOP_NANOSEC
+ / dResult.nanoSecPerRun);
if (newDSpeed > benchResult.dSpeed)
benchResult.dSpeed = newDSpeed;
}
- { int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
- OUTPUTLEVEL(2, "%2s-%-17.17s :%10u ->%10u (x%5.*f), %6.*f MB/s, %6.1f MB/s\r",
- marks[markNb], displayName,
- (unsigned)srcSize, (unsigned)cSize,
- ratioAccuracy, ratio,
- benchResult.cSpeed < (10 * MB_UNIT) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT,
+ {
+ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
+ OUTPUTLEVEL(
+ 2,
+ "%2s-%-17.17s :%10u ->%10u (x%5.*f), %6.*f MB/s, %6.1f MB/s\r",
+ marks[markNb],
+ displayName,
+ (unsigned)srcSize,
+ (unsigned)cSize,
+ ratioAccuracy,
+ ratio,
+ benchResult.cSpeed < (10 * MB_UNIT) ? 2 : 1,
+ (double)benchResult.cSpeed / MB_UNIT,
(double)benchResult.dSpeed / MB_UNIT);
}
- decompressionCompleted = BMK_isCompleted_TimedFn(timeStateDecompress);
+ decompressionCompleted =
+ BMK_isCompleted_TimedFn(timeStateDecompress);
}
- markNb = (markNb+1) % NB_MARKS;
- } /* while (!(compressionCompleted && decompressionCompleted)) */
+ markNb = (markNb + 1) % NB_MARKS;
+ } /* while (!(compressionCompleted && decompressionCompleted)) */
/* CRC Checking */
- { const BYTE* resultBuffer = (const BYTE*)(*resultBufferPtr);
- U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
- if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) {
+ {
+ const BYTE* resultBuffer = (const BYTE*)(*resultBufferPtr);
+ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
+ if ((adv->mode == BMK_both) && (crcOrig != crcCheck)) {
size_t u;
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n",
- displayName, (unsigned)crcOrig, (unsigned)crcCheck);
- for (u=0; u u) break;
+ if (bacc + srcSizes[segNb] > u)
+ break;
bacc += srcSizes[segNb];
}
pos = (U32)(u - bacc);
bNb = pos / (128 KB);
- DISPLAY("(sample %u, block %u, pos %u) \n", segNb, bNb, pos);
- { size_t const lowest = (u>5) ? 5 : u;
+ DISPLAY("(sample %u, block %u, pos %u) \n",
+ segNb,
+ bNb,
+ pos);
+ {
+ size_t const lowest = (u > 5) ? 5 : u;
size_t n;
DISPLAY("origin: ");
- for (n=lowest; n>0; n--)
- DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u-n]);
+ for (n = lowest; n > 0; n--)
+ DISPLAY("%02X ",
+ ((const BYTE*)srcBuffer)[u - n]);
DISPLAY(" :%02X: ", ((const BYTE*)srcBuffer)[u]);
- for (n=1; n<3; n++)
- DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
+ for (n = 1; n < 3; n++)
+ DISPLAY("%02X ",
+ ((const BYTE*)srcBuffer)[u + n]);
DISPLAY(" \n");
DISPLAY("decode: ");
- for (n=lowest; n>0; n--)
- DISPLAY("%02X ", resultBuffer[u-n]);
+ for (n = lowest; n > 0; n--)
+ DISPLAY("%02X ", resultBuffer[u - n]);
DISPLAY(" :%02X: ", resultBuffer[u]);
- for (n=1; n<3; n++)
- DISPLAY("%02X ", resultBuffer[u+n]);
+ for (n = 1; n < 3; n++)
+ DISPLAY("%02X ", resultBuffer[u + n]);
DISPLAY(" \n");
}
break;
}
- if (u==srcSize-1) { /* should never happen */
+ if (u == srcSize - 1) { /* should never happen */
DISPLAY("no difference detected\n");
}
- } /* for (u=0; umode == BMK_both) && (crcOrig!=crcCheck)) */
- } /* CRC Checking */
+ } /* for (u=0; umode == BMK_both) && (crcOrig!=crcCheck)) */
+ } /* CRC Checking */
- if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */
+ if (displayLevel
+ == 1) { /* hidden display mode -q, used by python speed benchmark */
double const cSpeed = (double)benchResult.cSpeed / MB_UNIT;
double const dSpeed = (double)benchResult.dSpeed / MB_UNIT;
if (adv->additionalParam) {
- OUTPUT("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam);
+ OUTPUT("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n",
+ cLevel,
+ (int)cSize,
+ ratio,
+ cSpeed,
+ dSpeed,
+ displayName,
+ adv->additionalParam);
} else {
- OUTPUT("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
+ OUTPUT("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n",
+ cLevel,
+ (int)cSize,
+ ratio,
+ cSpeed,
+ dSpeed,
+ displayName);
}
}
OUTPUTLEVEL(2, "%2i#\n", cLevel);
- } /* Bench */
+ } /* Bench */
- benchResult.cMem = (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx);
+ benchResult.cMem =
+ (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx);
return BMK_benchOutcome_setValidResult(benchResult);
}
-BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
- void* dstBuffer, size_t dstCapacity,
- const size_t* fileSizes, unsigned nbFiles,
- int cLevel, const ZSTD_compressionParameters* comprParams,
- const void* dictBuffer, size_t dictBufferSize,
- int displayLevel, const char* displayName, const BMK_advancedParams_t* adv)
+BMK_benchOutcome_t BMK_benchMemAdvanced(
+ const void* srcBuffer,
+ size_t srcSize,
+ void* dstBuffer,
+ size_t dstCapacity,
+ const size_t* fileSizes,
+ unsigned nbFiles,
+ int cLevel,
+ const ZSTD_compressionParameters* comprParams,
+ const void* dictBuffer,
+ size_t dictBufferSize,
+ int displayLevel,
+ const char* displayName,
+ const BMK_advancedParams_t* adv)
{
- int const dstParamsError = !dstBuffer ^ !dstCapacity; /* must be both NULL or none */
+ int const dstParamsError =
+ !dstBuffer ^ !dstCapacity; /* must be both NULL or none */
- size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
- U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
+ size_t const blockSize =
+ ((adv->blockSize >= 32 && (adv->mode != BMK_decodeOnly))
+ ? adv->blockSize
+ : srcSize)
+ + (!srcSize) /* avoid div by 0 */;
+ U32 const maxNbBlocks =
+ (U32)((srcSize + (blockSize - 1)) / blockSize) + nbFiles;
/* these are the blockTable parameters, just split up */
- const void ** const srcPtrs = (const void**)malloc(maxNbBlocks * sizeof(void*));
+ const void** const srcPtrs =
+ (const void**)malloc(maxNbBlocks * sizeof(void*));
size_t* const srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
-
- void ** const cPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
- size_t* const cSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+ void** const cPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
+ size_t* const cSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
size_t* const cCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
- void ** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
+ void** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
- BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState(adv->nbSeconds * 1000, BMK_RUNTEST_DEFAULT_MS);
- BMK_timedFnState_t* timeStateDecompress = BMK_createTimedFnState(adv->nbSeconds * 1000, BMK_RUNTEST_DEFAULT_MS);
+ BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState(
+ adv->nbSeconds * 1000, BMK_RUNTEST_DEFAULT_MS);
+ BMK_timedFnState_t* timeStateDecompress = BMK_createTimedFnState(
+ adv->nbSeconds * 1000, BMK_RUNTEST_DEFAULT_MS);
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
- const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);
+ const size_t maxCompressedSize = dstCapacity
+ ? dstCapacity
+ : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);
- void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize);
+ void* const internalDstBuffer =
+ dstBuffer ? NULL : malloc(maxCompressedSize);
void* const compressedBuffer = dstBuffer ? dstBuffer : internalDstBuffer;
- BMK_benchOutcome_t outcome = BMK_benchOutcome_error(); /* error by default */
+ BMK_benchOutcome_t outcome =
+ BMK_benchOutcome_error(); /* error by default */
void* resultBuffer = srcSize ? malloc(srcSize) : NULL;
- int const allocationincomplete = !srcPtrs || !srcSizes || !cPtrs ||
- !cSizes || !cCapacities || !resPtrs || !resSizes ||
- !timeStateCompress || !timeStateDecompress ||
- !cctx || !dctx ||
- !compressedBuffer || !resultBuffer;
-
+ int const allocationincomplete = !srcPtrs || !srcSizes || !cPtrs || !cSizes
+ || !cCapacities || !resPtrs || !resSizes || !timeStateCompress
+ || !timeStateDecompress || !cctx || !dctx || !compressedBuffer
+ || !resultBuffer;
if (!allocationincomplete && !dstParamsError) {
- outcome = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes,
- cPtrs, cCapacities, cSizes,
- resPtrs, resSizes,
- &resultBuffer,
- compressedBuffer, maxCompressedSize,
- timeStateCompress, timeStateDecompress,
- srcBuffer, srcSize,
- fileSizes, nbFiles,
- cLevel, comprParams,
- dictBuffer, dictBufferSize,
- cctx, dctx,
- displayLevel, displayName, adv);
+ outcome = BMK_benchMemAdvancedNoAlloc(
+ srcPtrs,
+ srcSizes,
+ cPtrs,
+ cCapacities,
+ cSizes,
+ resPtrs,
+ resSizes,
+ &resultBuffer,
+ compressedBuffer,
+ maxCompressedSize,
+ timeStateCompress,
+ timeStateDecompress,
+ srcBuffer,
+ srcSize,
+ fileSizes,
+ nbFiles,
+ cLevel,
+ comprParams,
+ dictBuffer,
+ dictBufferSize,
+ cctx,
+ dctx,
+ displayLevel,
+ displayName,
+ adv);
}
/* clean up */
@@ -644,67 +876,104 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
free(resPtrs);
free(resSizes);
- if(allocationincomplete) {
- RETURN_ERROR(31, BMK_benchOutcome_t, "allocation error : not enough memory");
+ if (allocationincomplete) {
+ RETURN_ERROR(
+ 31, BMK_benchOutcome_t, "allocation error : not enough memory");
}
- if(dstParamsError) {
+ if (dstParamsError) {
RETURN_ERROR(32, BMK_benchOutcome_t, "Dst parameters not coherent");
}
return outcome;
}
-BMK_benchOutcome_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
- const size_t* fileSizes, unsigned nbFiles,
- int cLevel, const ZSTD_compressionParameters* comprParams,
- const void* dictBuffer, size_t dictBufferSize,
- int displayLevel, const char* displayName) {
-
+BMK_benchOutcome_t BMK_benchMem(
+ const void* srcBuffer,
+ size_t srcSize,
+ const size_t* fileSizes,
+ unsigned nbFiles,
+ int cLevel,
+ const ZSTD_compressionParameters* comprParams,
+ const void* dictBuffer,
+ size_t dictBufferSize,
+ int displayLevel,
+ const char* displayName)
+{
BMK_advancedParams_t const adv = BMK_initAdvancedParams();
- return BMK_benchMemAdvanced(srcBuffer, srcSize,
- NULL, 0,
- fileSizes, nbFiles,
- cLevel, comprParams,
- dictBuffer, dictBufferSize,
- displayLevel, displayName, &adv);
+ return BMK_benchMemAdvanced(
+ srcBuffer,
+ srcSize,
+ NULL,
+ 0,
+ fileSizes,
+ nbFiles,
+ cLevel,
+ comprParams,
+ dictBuffer,
+ dictBufferSize,
+ displayLevel,
+ displayName,
+ &adv);
}
-static BMK_benchOutcome_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
- const size_t* fileSizes, unsigned nbFiles,
- int cLevel, const ZSTD_compressionParameters* comprParams,
- const void* dictBuffer, size_t dictBufferSize,
- int displayLevel, const char* displayName,
- BMK_advancedParams_t const * const adv)
+static BMK_benchOutcome_t BMK_benchCLevel(
+ const void* srcBuffer,
+ size_t benchedSize,
+ const size_t* fileSizes,
+ unsigned nbFiles,
+ int cLevel,
+ const ZSTD_compressionParameters* comprParams,
+ const void* dictBuffer,
+ size_t dictBufferSize,
+ int displayLevel,
+ const char* displayName,
+ BMK_advancedParams_t const* const adv)
{
const char* pch = strrchr(displayName, '\\'); /* Windows */
- if (!pch) pch = strrchr(displayName, '/'); /* Linux */
- if (pch) displayName = pch+1;
+ if (!pch)
+ pch = strrchr(displayName, '/'); /* Linux */
+ if (pch)
+ displayName = pch + 1;
if (adv->realTime) {
DISPLAYLEVEL(2, "Note : switching to real-time priority \n");
SET_REALTIME_PRIORITY;
}
- if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */
+ if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */
OUTPUT("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n",
- ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING,
- (unsigned)benchedSize, adv->nbSeconds, (unsigned)(adv->blockSize>>10));
+ ZSTD_VERSION_STRING,
+ ZSTD_GIT_COMMIT_STRING,
+ (unsigned)benchedSize,
+ adv->nbSeconds,
+ (unsigned)(adv->blockSize >> 10));
- return BMK_benchMemAdvanced(srcBuffer, benchedSize,
- NULL, 0,
- fileSizes, nbFiles,
- cLevel, comprParams,
- dictBuffer, dictBufferSize,
- displayLevel, displayName, adv);
+ return BMK_benchMemAdvanced(
+ srcBuffer,
+ benchedSize,
+ NULL,
+ 0,
+ fileSizes,
+ nbFiles,
+ cLevel,
+ comprParams,
+ dictBuffer,
+ dictBufferSize,
+ displayLevel,
+ displayName,
+ adv);
}
-int BMK_syntheticTest(int cLevel, double compressibility,
- const ZSTD_compressionParameters* compressionParams,
- int displayLevel, const BMK_advancedParams_t* adv)
+int BMK_syntheticTest(
+ int cLevel,
+ double compressibility,
+ const ZSTD_compressionParameters* compressionParams,
+ int displayLevel,
+ const BMK_advancedParams_t* adv)
{
- char nameBuff[20] = {0};
- const char* name = nameBuff;
- size_t const benchedSize = 10000000;
+ char nameBuff[20] = { 0 };
+ const char* name = nameBuff;
+ size_t const benchedSize = adv->blockSize ? adv->blockSize : 10000000;
void* srcBuffer;
BMK_benchOutcome_t res;
@@ -726,15 +995,26 @@ int BMK_syntheticTest(int cLevel, double compressibility,
name = "Lorem ipsum";
} else {
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
- snprintf (nameBuff, sizeof(nameBuff), "Synthetic %2u%%", (unsigned)(compressibility*100));
+ formatString_u(
+ nameBuff,
+ sizeof(nameBuff),
+ "Synthetic %u%%",
+ (unsigned)(compressibility * 100));
}
/* Bench */
- res = BMK_benchCLevel(srcBuffer, benchedSize,
- &benchedSize /* ? */, 1 /* ? */,
- cLevel, compressionParams,
- NULL, 0, /* dictionary */
- displayLevel, name, adv);
+ res = BMK_benchCLevel(
+ srcBuffer,
+ benchedSize,
+ &benchedSize /* ? */,
+ 1 /* ? */,
+ cLevel,
+ compressionParams,
+ NULL,
+ 0, /* dictionary */
+ displayLevel,
+ name,
+ adv);
/* clean up */
free(srcBuffer);
@@ -742,16 +1022,15 @@ int BMK_syntheticTest(int cLevel, double compressibility,
return !BMK_isSuccessful_benchOutcome(res);
}
-
-
static size_t BMK_findMaxMem(U64 requiredMem)
{
size_t const step = 64 MB;
- BYTE* testmem = NULL;
+ BYTE* testmem = NULL;
requiredMem = (((requiredMem >> 26) + 1) << 26);
requiredMem += step;
- if (requiredMem > maxMemory) requiredMem = maxMemory;
+ if (requiredMem > maxMemory)
+ requiredMem = maxMemory;
do {
testmem = (BYTE*)malloc((size_t)requiredMem);
@@ -765,53 +1044,75 @@ static size_t BMK_findMaxMem(U64 requiredMem)
/*! BMK_loadFiles() :
* Loads `buffer` with content of files listed within `fileNamesTable`.
* At most, fills `buffer` entirely. */
-static int BMK_loadFiles(void* buffer, size_t bufferSize,
- size_t* fileSizes,
- const char* const * fileNamesTable, unsigned nbFiles,
- int displayLevel)
+static int BMK_loadFiles(
+ void* buffer,
+ size_t bufferSize,
+ size_t* fileSizes,
+ const char* const* fileNamesTable,
+ unsigned nbFiles,
+ int displayLevel)
{
size_t pos = 0, totalSize = 0;
unsigned n;
- for (n=0; n bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
- { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
- if (readSize != (size_t)fileSize) RETURN_ERROR_INT(11, "could not read %s", fileNamesTable[n]);
+ if (fileSize > bufferSize - pos)
+ fileSize = bufferSize - pos,
+ nbFiles = n; /* buffer too small - stop after this file */
+ {
+ size_t const readSize =
+ fread(((char*)buffer) + pos, 1, (size_t)fileSize, f);
+ if (readSize != (size_t)fileSize)
+ RETURN_ERROR_INT(
+ 11, "could not read %s", fileNamesTable[n]);
pos += readSize;
}
fileSizes[n] = (size_t)fileSize;
totalSize += (size_t)fileSize;
fclose(f);
- } }
+ }
+ }
- if (totalSize == 0) RETURN_ERROR_INT(12, "no data to bench");
+ if (totalSize == 0)
+ RETURN_ERROR_INT(12, "no data to bench");
return 0;
}
int BMK_benchFilesAdvanced(
- const char* const * fileNamesTable, unsigned nbFiles,
- const char* dictFileName, int cLevel,
- const ZSTD_compressionParameters* compressionParams,
- int displayLevel, const BMK_advancedParams_t* adv)
+ const char* const* fileNamesTable,
+ unsigned nbFiles,
+ const char* dictFileName,
+ int cLevel,
+ const ZSTD_compressionParameters* compressionParams,
+ int displayLevel,
+ const BMK_advancedParams_t* adv)
{
void* srcBuffer = NULL;
size_t benchedSize;
- void* dictBuffer = NULL;
+ void* dictBuffer = NULL;
size_t dictBufferSize = 0;
- size_t* fileSizes = NULL;
+ size_t* fileSizes = NULL;
BMK_benchOutcome_t res;
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
@@ -840,7 +1141,11 @@ int BMK_benchFilesAdvanced(
if (dictFileName != NULL) {
U64 const dictFileSize = UTIL_getFileSize(dictFileName);
if (dictFileSize == UTIL_FILESIZE_UNKNOWN) {
- DISPLAYLEVEL(1, "error loading %s : %s \n", dictFileName, strerror(errno));
+ DISPLAYLEVEL(
+ 1,
+ "error loading %s : %s \n",
+ dictFileName,
+ strerror(errno));
free(fileSizes);
DISPLAYLEVEL(1, "benchmark aborted");
return 17;
@@ -851,28 +1156,38 @@ int BMK_benchFilesAdvanced(
return 18;
}
dictBufferSize = (size_t)dictFileSize;
- dictBuffer = malloc(dictBufferSize);
- if (dictBuffer==NULL) {
+ dictBuffer = malloc(dictBufferSize);
+ if (dictBuffer == NULL) {
free(fileSizes);
- DISPLAYLEVEL(1, "not enough memory for dictionary (%u bytes)",
- (unsigned)dictBufferSize);
+ DISPLAYLEVEL(
+ 1,
+ "not enough memory for dictionary (%u bytes)",
+ (unsigned)dictBufferSize);
return 19;
}
- { int const errorCode = BMK_loadFiles(dictBuffer, dictBufferSize,
- fileSizes, &dictFileName /*?*/,
- 1 /*?*/, displayLevel);
+ {
+ int const errorCode = BMK_loadFiles(
+ dictBuffer,
+ dictBufferSize,
+ fileSizes,
+ &dictFileName /*?*/,
+ 1 /*?*/,
+ displayLevel);
if (errorCode) {
res = BMK_benchOutcome_error();
goto _cleanUp;
- } }
+ }
+ }
}
/* Memory allocation & restrictions */
benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
- if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
+ if ((U64)benchedSize > totalSizeToLoad)
+ benchedSize = (size_t)totalSizeToLoad;
if (benchedSize < totalSizeToLoad)
- DISPLAY("Not enough memory; testing %u MB only...\n", (unsigned)(benchedSize >> 20));
+ DISPLAY("Not enough memory; testing %u MB only...\n",
+ (unsigned)(benchedSize >> 20));
srcBuffer = benchedSize ? malloc(benchedSize) : NULL;
if (!srcBuffer) {
@@ -883,25 +1198,41 @@ int BMK_benchFilesAdvanced(
}
/* Load input buffer */
- { int const errorCode = BMK_loadFiles(srcBuffer, benchedSize,
- fileSizes, fileNamesTable, nbFiles,
- displayLevel);
+ {
+ int const errorCode = BMK_loadFiles(
+ srcBuffer,
+ benchedSize,
+ fileSizes,
+ fileNamesTable,
+ nbFiles,
+ displayLevel);
if (errorCode) {
res = BMK_benchOutcome_error();
goto _cleanUp;
- } }
+ }
+ }
/* Bench */
- { char mfName[20] = {0};
- snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
- { const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
- res = BMK_benchCLevel(srcBuffer, benchedSize,
- fileSizes, nbFiles,
- cLevel, compressionParams,
- dictBuffer, dictBufferSize,
- displayLevel, displayName,
- adv);
- } }
+ {
+ char mfName[20] = { 0 };
+ formatString_u(mfName, sizeof(mfName), " %u files", nbFiles);
+ {
+ const char* const displayName =
+ (nbFiles > 1) ? mfName : fileNamesTable[0];
+ res = BMK_benchCLevel(
+ srcBuffer,
+ benchedSize,
+ fileSizes,
+ nbFiles,
+ cLevel,
+ compressionParams,
+ dictBuffer,
+ dictBufferSize,
+ displayLevel,
+ displayName,
+ adv);
+ }
+ }
_cleanUp:
free(srcBuffer);
@@ -910,12 +1241,21 @@ _cleanUp:
return !BMK_isSuccessful_benchOutcome(res);
}
-
-int BMK_benchFiles(const char* const * fileNamesTable, unsigned nbFiles,
- const char* dictFileName,
- int cLevel, const ZSTD_compressionParameters* compressionParams,
- int displayLevel)
+int BMK_benchFiles(
+ const char* const* fileNamesTable,
+ unsigned nbFiles,
+ const char* dictFileName,
+ int cLevel,
+ const ZSTD_compressionParameters* compressionParams,
+ int displayLevel)
{
BMK_advancedParams_t const adv = BMK_initAdvancedParams();
- return BMK_benchFilesAdvanced(fileNamesTable, nbFiles, dictFileName, cLevel, compressionParams, displayLevel, &adv);
+ return BMK_benchFilesAdvanced(
+ fileNamesTable,
+ nbFiles,
+ dictFileName,
+ cLevel,
+ compressionParams,
+ displayLevel,
+ &adv);
}
diff --git a/programs/lorem.c b/programs/lorem.c
index 59dd6da62..79030c92f 100644
--- a/programs/lorem.c
+++ b/programs/lorem.c
@@ -8,7 +8,6 @@
* You may select, at your option, one of the above-listed licenses.
*/
-
/* Implementation notes:
*
* This is a very simple lorem ipsum generator
@@ -24,91 +23,162 @@
* and lacks a regularity more representative of text.
*
* The compression ratio achievable on the generated lorem ipsum
- * is still a bit too good, presumably because the dictionary is too small.
- * It would be possible to create some more complex scheme,
- * notably by enlarging the dictionary with a word generator,
- * and adding grammatical rules (composition) and syntax rules.
- * But that's probably overkill for the intended goal.
+ * is still a bit too good, presumably because the dictionary is a bit too
+ * small. It would be possible to create some more complex scheme, notably by
+ * enlarging the dictionary with a word generator, and adding grammatical rules
+ * (composition) and syntax rules. But that's probably overkill for the intended
+ * goal.
*/
#include "lorem.h"
-#include /* memcpy */
-#include /* INT_MAX */
#include
+#include /* INT_MAX */
+#include /* memcpy */
#define WORD_MAX_SIZE 20
/* Define the word pool */
-static const char *words[] = {
- "lorem", "ipsum", "dolor", "sit", "amet",
- "consectetur", "adipiscing", "elit", "sed", "do",
- "eiusmod", "tempor", "incididunt", "ut", "labore",
- "et", "dolore", "magna", "aliqua", "dis",
- "lectus", "vestibulum", "mattis", "ullamcorper", "velit",
- "commodo", "a", "lacus", "arcu", "magnis",
- "parturient", "montes", "nascetur", "ridiculus", "mus",
- "mauris", "nulla", "malesuada", "pellentesque", "eget",
- "gravida", "in", "dictum", "non", "erat",
- "nam", "voluptat", "maecenas", "blandit", "aliquam",
- "etiam", "enim", "lobortis", "scelerisque", "fermentum",
- "dui", "faucibus", "ornare", "at", "elementum",
- "eu", "facilisis", "odio", "morbi", "quis",
- "eros", "donec", "ac", "orci", "purus",
- "turpis", "cursus", "leo", "vel", "porta"};
+static const char* kWords[] = {
+ "lorem", "ipsum", "dolor", "sit", "amet",
+ "consectetur", "adipiscing", "elit", "sed", "do",
+ "eiusmod", "tempor", "incididunt", "ut", "labore",
+ "et", "dolore", "magna", "aliqua", "dis",
+ "lectus", "vestibulum", "mattis", "ullamcorper", "velit",
+ "commodo", "a", "lacus", "arcu", "magnis",
+ "parturient", "montes", "nascetur", "ridiculus", "mus",
+ "mauris", "nulla", "malesuada", "pellentesque", "eget",
+ "gravida", "in", "dictum", "non", "erat",
+ "nam", "voluptat", "maecenas", "blandit", "aliquam",
+ "etiam", "enim", "lobortis", "scelerisque", "fermentum",
+ "dui", "faucibus", "ornare", "at", "elementum",
+ "eu", "facilisis", "odio", "morbi", "quis",
+ "eros", "donec", "ac", "orci", "purus",
+ "turpis", "cursus", "leo", "vel", "porta",
+ "consequat", "interdum", "varius", "vulputate", "aliquet",
+ "pharetra", "nunc", "auctor", "urna", "id",
+ "metus", "viverra", "nibh", "cras", "mi",
+ "unde", "omnis", "iste", "natus", "error",
+ "perspiciatis", "voluptatem", "accusantium", "doloremque", "laudantium",
+ "totam", "rem", "aperiam", "eaque", "ipsa",
+ "quae", "ab", "illo", "inventore", "veritatis",
+ "quasi", "architecto", "beatae", "vitae", "dicta",
+ "sunt", "explicabo", "nemo", "ipsam", "quia",
+ "voluptas", "aspernatur", "aut", "odit", "fugit",
+ "consequuntur", "magni", "dolores", "eos", "qui",
+ "ratione", "sequi", "nesciunt", "neque", "porro",
+ "quisquam", "est", "dolorem", "adipisci", "numquam",
+ "eius", "modi", "tempora", "incidunt", "magnam",
+ "quaerat", "ad", "minima", "veniam", "nostrum",
+ "ullam", "corporis", "suscipit", "laboriosam", "nisi",
+ "aliquid", "ex", "ea", "commodi", "consequatur",
+ "autem", "eum", "iure", "voluptate", "esse",
+ "quam", "nihil", "molestiae", "illum", "fugiat",
+ "quo", "pariatur", "vero", "accusamus", "iusto",
+ "dignissimos", "ducimus", "blanditiis", "praesentium", "voluptatum",
+ "deleniti", "atque", "corrupti", "quos", "quas",
+ "molestias", "excepturi", "sint", "occaecati", "cupiditate",
+ "provident", "similique", "culpa", "officia", "deserunt",
+ "mollitia", "animi", "laborum", "dolorum", "fuga",
+ "harum", "quidem", "rerum", "facilis", "expedita",
+ "distinctio", "libero", "tempore", "cum", "soluta",
+ "nobis", "eligendi", "optio", "cumque", "impedit",
+ "minus", "quod", "maxime", "placeat", "facere",
+ "possimus", "assumenda", "repellendus", "temporibus", "quibusdam",
+ "officiis", "debitis", "saepe", "eveniet", "voluptates",
+ "repudiandae", "recusandae", "itaque", "earum", "hic",
+ "tenetur", "sapiente", "delectus", "reiciendis", "cillum",
+ "maiores", "alias", "perferendis", "doloribus", "asperiores",
+ "repellat", "minim", "nostrud", "exercitation", "ullamco",
+ "laboris", "aliquip", "duis", "aute", "irure",
+};
+static const unsigned kNbWords = sizeof(kWords) / sizeof(kWords[0]);
-/* simple distribution that favors small words :
- * 1 letter : weight 3
- * 2-3 letters : weight 2
- * 4+ letters : weight 1
- * This is expected to be a bit more difficult to compress */
-static const int distrib[] = {
- 0, 1, 2, 3, 3, 4, 5, 6, 7, 8,
- 8,9, 9, 10, 11, 12, 13, 13, 14, 15,
- 15, 16, 17, 18, 19, 19, 20, 21, 22, 23,
- 24, 25, 26, 26, 26, 27, 28, 29, 30, 31,
- 32, 33, 34, 34, 35, 36, 37, 38, 39, 40,
- 41, 41, 42, 43, 43, 44, 45, 45, 46, 47,
- 48, 49, 50, 51, 52, 53, 54, 55, 55, 56,
- 57, 58, 58, 59, 60, 60, 61, 62, 63, 64,
- 65, 66, 67, 67, 68, 69, 70, 71, 72, 72,
- 73, 73, 74 };
-static const unsigned distribCount = sizeof(distrib) / sizeof(distrib[0]);
+/* simple 1-dimension distribution, based on word's length, favors small words
+ */
+static const int kWeights[] = { 0, 8, 6, 4, 3, 2 };
+static const size_t kNbWeights = sizeof(kWeights) / sizeof(kWeights[0]);
+
+#define DISTRIB_SIZE_MAX 650
+static int g_distrib[DISTRIB_SIZE_MAX] = { 0 };
+static unsigned g_distribCount = 0;
+
+static void countFreqs(
+ const char* words[],
+ size_t nbWords,
+ const int* weights,
+ size_t nbWeights)
+{
+ unsigned total = 0;
+ size_t w;
+ for (w = 0; w < nbWords; w++) {
+ size_t len = strlen(words[w]);
+ int lmax;
+ if (len >= nbWeights)
+ len = nbWeights - 1;
+ lmax = weights[len];
+ total += (unsigned)lmax;
+ }
+ g_distribCount = total;
+ assert(g_distribCount <= DISTRIB_SIZE_MAX);
+}
+
+static void init_word_distrib(
+ const char* words[],
+ size_t nbWords,
+ const int* weights,
+ size_t nbWeights)
+{
+ size_t w, d = 0;
+ countFreqs(words, nbWords, weights, nbWeights);
+ for (w = 0; w < nbWords; w++) {
+ size_t len = strlen(words[w]);
+ int l, lmax;
+ if (len >= nbWeights)
+ len = nbWeights - 1;
+ lmax = weights[len];
+ for (l = 0; l < lmax; l++) {
+ g_distrib[d++] = (int)w;
+ }
+ }
+}
/* Note: this unit only works when invoked sequentially.
* No concurrent access is allowed */
-static char *g_ptr = NULL;
-static size_t g_nbChars = 0;
-static size_t g_maxChars = 10000000;
+static char* g_ptr = NULL;
+static size_t g_nbChars = 0;
+static size_t g_maxChars = 10000000;
static unsigned g_randRoot = 0;
#define RDG_rotl32(x, r) ((x << r) | (x >> (32 - r)))
-static unsigned LOREM_rand(unsigned range) {
- static const unsigned prime1 = 2654435761U;
- static const unsigned prime2 = 2246822519U;
- unsigned rand32 = g_randRoot;
- rand32 *= prime1;
- rand32 ^= prime2;
- rand32 = RDG_rotl32(rand32, 13);
- g_randRoot = rand32;
- return (unsigned)(((unsigned long long)rand32 * range) >> 32);
+static unsigned LOREM_rand(unsigned range)
+{
+ static const unsigned prime1 = 2654435761U;
+ static const unsigned prime2 = 2246822519U;
+ unsigned rand32 = g_randRoot;
+ rand32 *= prime1;
+ rand32 ^= prime2;
+ rand32 = RDG_rotl32(rand32, 13);
+ g_randRoot = rand32;
+ return (unsigned)(((unsigned long long)rand32 * range) >> 32);
}
-static void writeLastCharacters(void) {
- size_t lastChars = g_maxChars - g_nbChars;
- assert(g_maxChars >= g_nbChars);
- if (lastChars == 0)
- return;
- g_ptr[g_nbChars++] = '.';
- if (lastChars > 2) {
- memset(g_ptr + g_nbChars, ' ', lastChars - 2);
- }
- if (lastChars > 1) {
- g_ptr[g_maxChars-1] = '\n';
- }
- g_nbChars = g_maxChars;
+static void writeLastCharacters(void)
+{
+ size_t lastChars = g_maxChars - g_nbChars;
+ assert(g_maxChars >= g_nbChars);
+ if (lastChars == 0)
+ return;
+ g_ptr[g_nbChars++] = '.';
+ if (lastChars > 2) {
+ memset(g_ptr + g_nbChars, ' ', lastChars - 2);
+ }
+ if (lastChars > 1) {
+ g_ptr[g_maxChars - 1] = '\n';
+ }
+ g_nbChars = g_maxChars;
}
-static void generateWord(const char *word, const char *separator, int upCase)
+static void generateWord(const char* word, const char* separator, int upCase)
{
size_t const len = strlen(word) + strlen(separator);
if (g_nbChars + len > g_maxChars) {
@@ -118,90 +188,98 @@ static void generateWord(const char *word, const char *separator, int upCase)
memcpy(g_ptr + g_nbChars, word, strlen(word));
if (upCase) {
static const char toUp = 'A' - 'a';
- g_ptr[g_nbChars] = (char)(g_ptr[g_nbChars] + toUp);
+ g_ptr[g_nbChars] = (char)(g_ptr[g_nbChars] + toUp);
}
g_nbChars += strlen(word);
memcpy(g_ptr + g_nbChars, separator, strlen(separator));
g_nbChars += strlen(separator);
}
-static int about(unsigned target) {
- return (int)(LOREM_rand(target) + LOREM_rand(target) + 1);
+static int about(unsigned target)
+{
+ return (int)(LOREM_rand(target) + LOREM_rand(target) + 1);
}
/* Function to generate a random sentence */
-static void generateSentence(int nbWords) {
- int commaPos = about(9);
- int comma2 = commaPos + about(7);
- int i;
- for (i = 0; i < nbWords; i++) {
- int const wordID = distrib[LOREM_rand(distribCount)];
- const char *const word = words[wordID];
- const char* sep = " ";
- if (i == commaPos)
- sep = ", ";
- if (i == comma2)
- sep = ", ";
- if (i == nbWords - 1)
- sep = ". ";
- generateWord(word, sep, i==0);
- }
+static void generateSentence(int nbWords)
+{
+ int commaPos = about(9);
+ int comma2 = commaPos + about(7);
+ int qmark = (LOREM_rand(11) == 7);
+ const char* endSep = qmark ? "? " : ". ";
+ int i;
+ for (i = 0; i < nbWords; i++) {
+ int const wordID = g_distrib[LOREM_rand(g_distribCount)];
+ const char* const word = kWords[wordID];
+ const char* sep = " ";
+ if (i == commaPos)
+ sep = ", ";
+ if (i == comma2)
+ sep = ", ";
+ if (i == nbWords - 1)
+ sep = endSep;
+ generateWord(word, sep, i == 0);
+ }
}
-static void generateParagraph(int nbSentences) {
- int i;
- for (i = 0; i < nbSentences; i++) {
- int wordsPerSentence = about(8);
- generateSentence(wordsPerSentence);
- }
- if (g_nbChars < g_maxChars) {
- g_ptr[g_nbChars++] = '\n';
- }
- if (g_nbChars < g_maxChars) {
- g_ptr[g_nbChars++] = '\n';
- }
+static void generateParagraph(int nbSentences)
+{
+ int i;
+ for (i = 0; i < nbSentences; i++) {
+ int wordsPerSentence = about(11);
+ generateSentence(wordsPerSentence);
+ }
+ if (g_nbChars < g_maxChars) {
+ g_ptr[g_nbChars++] = '\n';
+ }
+ if (g_nbChars < g_maxChars) {
+ g_ptr[g_nbChars++] = '\n';
+ }
}
/* It's "common" for lorem ipsum generators to start with the same first
* pre-defined sentence */
-static void generateFirstSentence(void) {
- int i;
- for (i = 0; i < 18; i++) {
- const char *word = words[i];
- const char *separator = " ";
- if (i == 4)
- separator = ", ";
- if (i == 7)
- separator = ", ";
- generateWord(word, separator, i==0);
- }
- generateWord(words[18], ". ", 0);
+static void generateFirstSentence(void)
+{
+ int i;
+ for (i = 0; i < 18; i++) {
+ const char* word = kWords[i];
+ const char* separator = " ";
+ if (i == 4)
+ separator = ", ";
+ if (i == 7)
+ separator = ", ";
+ generateWord(word, separator, i == 0);
+ }
+ generateWord(kWords[18], ". ", 0);
}
-size_t LOREM_genBlock(void* buffer, size_t size,
- unsigned seed,
- int first, int fill)
+size_t
+LOREM_genBlock(void* buffer, size_t size, unsigned seed, int first, int fill)
{
- g_ptr = (char*)buffer;
- assert(size < INT_MAX);
- g_maxChars = size;
- g_nbChars = 0;
- g_randRoot = seed;
- if (first) {
- generateFirstSentence();
- }
- while (g_nbChars < g_maxChars) {
- int sentencePerParagraph = about(7);
- generateParagraph(sentencePerParagraph);
- if (!fill)
- break; /* only generate one paragraph in not-fill mode */
- }
- g_ptr = NULL;
- return g_nbChars;
+ g_ptr = (char*)buffer;
+ assert(size < INT_MAX);
+ g_maxChars = size;
+ g_nbChars = 0;
+ g_randRoot = seed;
+ if (g_distribCount == 0) {
+ init_word_distrib(kWords, kNbWords, kWeights, kNbWeights);
+ }
+
+ if (first) {
+ generateFirstSentence();
+ }
+ while (g_nbChars < g_maxChars) {
+ int sentencePerParagraph = about(7);
+ generateParagraph(sentencePerParagraph);
+ if (!fill)
+ break; /* only generate one paragraph in not-fill mode */
+ }
+ g_ptr = NULL;
+ return g_nbChars;
}
void LOREM_genBuffer(void* buffer, size_t size, unsigned seed)
{
- LOREM_genBlock(buffer, size, seed, 1, 1);
+ LOREM_genBlock(buffer, size, seed, 1, 1);
}
-
diff --git a/tests/Makefile b/tests/Makefile
index 35be1039b..ed7638b74 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -206,7 +206,7 @@ paramgrill : LDLIBS += -lm
paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c $(PRGDIR)/lorem.c paramgrill.c
CLEAN += datagen
-datagen : $(PRGDIR)/datagen.c datagencli.c
+datagen : $(PRGDIR)/datagen.c $(PRGDIR)/lorem.c loremOut.c datagencli.c
$(LINK.c) $^ -o $@$(EXT)
CLEAN += roundTripCrash
diff --git a/tests/datagencli.c b/tests/datagencli.c
index 09ec5e9ae..56616bef4 100644
--- a/tests/datagencli.c
+++ b/tests/datagencli.c
@@ -8,122 +8,141 @@
* You may select, at your option, one of the above-listed licenses.
*/
+/*-************************************
+ * Dependencies
+ **************************************/
+#include /* fprintf, stderr */
+#include "datagen.h" /* RDG_generate */
+#include "loremOut.h" /* LOREM_genOut */
+#include "util.h" /* Compiler options */
/*-************************************
-* Dependencies
-**************************************/
-#include "util.h" /* Compiler options */
-#include /* fprintf, stderr */
-#include "datagen.h" /* RDG_generate */
-
-
-/*-************************************
-* Constants
-**************************************/
-#define KB *(1 <<10)
-#define MB *(1 <<20)
-#define GB *(1U<<30)
+ * Constants
+ **************************************/
+#define KB *(1 << 10)
+#define MB *(1 << 20)
+#define GB *(1U << 30)
#define SIZE_DEFAULT ((64 KB) + 1)
#define SEED_DEFAULT 0
-#define COMPRESSIBILITY_DEFAULT 50
-
+#define COMPRESSIBILITY_DEFAULT 9999
/*-************************************
-* Macros
-**************************************/
-#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+ * Macros
+ **************************************/
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) \
+ if (displayLevel >= l) { \
+ DISPLAY(__VA_ARGS__); \
+ }
static unsigned displayLevel = 2;
-
/*-*******************************************************
-* Command line
-*********************************************************/
+ * Command line
+ *********************************************************/
static int usage(const char* programName)
{
- DISPLAY( "Compressible data generator\n");
- DISPLAY( "Usage :\n");
- DISPLAY( " %s [args]\n", programName);
- DISPLAY( "\n");
- DISPLAY( "Arguments :\n");
- DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT);
- DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT);
- DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n",
- COMPRESSIBILITY_DEFAULT);
- DISPLAY( " -h : display help and exit\n");
+ DISPLAY("Compressible data generator\n");
+ DISPLAY("Usage :\n");
+ DISPLAY(" %s [args]\n", programName);
+ DISPLAY("\n");
+ DISPLAY("Arguments :\n");
+ DISPLAY(" -g# : generate # data (default:%i)\n", SIZE_DEFAULT);
+ DISPLAY(" -s# : Select seed (default:%i)\n", SEED_DEFAULT);
+ DISPLAY(" -P# : Select compressibility in %% (range [0-100])\n");
+ DISPLAY(" -h : display help and exit\n");
return 0;
}
-
int main(int argc, const char** argv)
{
- unsigned probaU32 = COMPRESSIBILITY_DEFAULT;
- double litProba = 0.0;
- U64 size = SIZE_DEFAULT;
- U32 seed = SEED_DEFAULT;
+ unsigned probaU32 = COMPRESSIBILITY_DEFAULT;
+ double litProba = 0.0;
+ U64 size = SIZE_DEFAULT;
+ U32 seed = SEED_DEFAULT;
const char* const programName = argv[0];
int argNb;
- for(argNb=1; argNb='0') && (*argument<='9'))
- size *= 10, size += *argument++ - '0';
- if (*argument=='K') { size <<= 10; argument++; }
- if (*argument=='M') { size <<= 20; argument++; }
- if (*argument=='G') { size <<= 30; argument++; }
- if (*argument=='B') { argument++; }
- break;
- case 's':
- argument++;
- seed=0;
- while ((*argument>='0') && (*argument<='9'))
- seed *= 10, seed += *argument++ - '0';
- break;
- case 'P':
- argument++;
- probaU32 = 0;
- while ((*argument>='0') && (*argument<='9'))
- probaU32 *= 10, probaU32 += *argument++ - '0';
- if (probaU32>100) probaU32 = 100;
- break;
- case 'L': /* hidden argument : Literal distribution probability */
- argument++;
- litProba=0.;
- while ((*argument>='0') && (*argument<='9'))
- litProba *= 10, litProba += *argument++ - '0';
- if (litProba>100.) litProba=100.;
- litProba /= 100.;
- break;
- case 'v':
- displayLevel = 4;
- argument++;
- break;
- default:
- return usage(programName);
+ while (*argument != 0) {
+ switch (*argument) {
+ case 'h':
+ return usage(programName);
+ case 'g':
+ argument++;
+ size = 0;
+ while ((*argument >= '0') && (*argument <= '9'))
+ size *= 10, size += (U64)(*argument++ - '0');
+ if (*argument == 'K') {
+ size <<= 10;
+ argument++;
+ }
+ if (*argument == 'M') {
+ size <<= 20;
+ argument++;
+ }
+ if (*argument == 'G') {
+ size <<= 30;
+ argument++;
+ }
+ if (*argument == 'B') {
+ argument++;
+ }
+ break;
+ case 's':
+ argument++;
+ seed = 0;
+ while ((*argument >= '0') && (*argument <= '9'))
+ seed *= 10, seed += (U32)(*argument++ - '0');
+ break;
+ case 'P':
+ argument++;
+ probaU32 = 0;
+ while ((*argument >= '0') && (*argument <= '9'))
+ probaU32 *= 10,
+ probaU32 += (U32)(*argument++ - '0');
+ if (probaU32 > 100)
+ probaU32 = 100;
+ break;
+ case 'L': /* hidden argument : Literal distribution
+ probability */
+ argument++;
+ litProba = 0.;
+ while ((*argument >= '0') && (*argument <= '9'))
+ litProba *= 10, litProba += *argument++ - '0';
+ if (litProba > 100.)
+ litProba = 100.;
+ litProba /= 100.;
+ break;
+ case 'v':
+ displayLevel = 4;
+ argument++;
+ break;
+ default:
+ return usage(programName);
}
- } } } /* for(argNb=1; argNb 4 GB).
+ * Note that, beyond 1 paragraph, this generator produces
+ * a different content than LOREM_genBuffer (even when using same seed).
+ */
+
+#include "loremOut.h"
+#include
+#include
+#include "lorem.h" /* LOREM_genBlock */
+#include "platform.h" /* Compiler options, SET_BINARY_MODE */
+
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+#define LOREM_BLOCKSIZE (1 << 10)
+void LOREM_genOut(unsigned long long size, unsigned seed)
+{
+ char buff[LOREM_BLOCKSIZE] = { 0 };
+ unsigned long long total = 0;
+ size_t genBlockSize = (size_t)MIN(size, LOREM_BLOCKSIZE);
+
+ /* init */
+ SET_BINARY_MODE(stdout);
+
+ /* Generate Ipsum text, one paragraph at a time */
+ while (total < size) {
+ size_t generated =
+ LOREM_genBlock(buff, genBlockSize, seed++, total == 0, 0);
+ assert(generated <= genBlockSize);
+ total += generated;
+ assert(total <= size);
+ fwrite(buff,
+ 1,
+ generated,
+ stdout); /* note: should check potential write error */
+ if (size - total < genBlockSize)
+ genBlockSize = (size_t)(size - total);
+ }
+ assert(total == size);
+}
diff --git a/tests/loremOut.h b/tests/loremOut.h
new file mode 100644
index 000000000..3a32e1161
--- /dev/null
+++ b/tests/loremOut.h
@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* LOREM_genOut():
+ * Generate @size bytes of compressible data using lorem ipsum generator into
+ * stdout.
+ */
+void LOREM_genOut(unsigned long long size, unsigned seed);