diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml index 2a97c32d0..da35ec675 100644 --- a/.github/workflows/dev-short-tests.yml +++ b/.github/workflows/dev-short-tests.yml @@ -693,9 +693,9 @@ jobs: - name: Install musl-tools run: | sudo apt install -y musl-tools - - name: Compile the project with musl-gcc + - name: Compile with musl-gcc and test-zstd run: | - CC=musl-gcc CPPFLAGS=-DZSTD_USE_C90_QSORT make -j V=1 zstd + CC=musl-gcc CFLAGS="-Werror -O3" CPPFLAGS=-DZDICT_QSORT=ZDICT_QSORT_C90 make -j -C tests test-zstd V=1 intel-cet-compatibility: runs-on: ubuntu-latest diff --git a/lib/README.md b/lib/README.md index 9df516c5a..89e53e6c8 100644 --- a/lib/README.md +++ b/lib/README.md @@ -193,9 +193,10 @@ The file structure is designed to make this selection manually achievable for an and assembly decoding loops. You may want to use this macro if these loops are slower on your platform. -- The macro `ZSTD_USE_C90_QSORT` forces usage of C90's `qsort()`, - for situations where the code cannot determine that `qsort_r()` is not supported, - such as, for example, older versions of `musl`. +- The macro `ZDICT_QSORT` can enforce selection of a specific sorting variant, + which is useful when autodetection fails, for example with older versions of `musl`. + For this scenario, it can be set as `ZDICT_QSORT=ZDICT_QSORT_C90`. + Other selectable suffixes are `_GNU`, `_APPLE`, `_MSVC` and `_C11`. #### Windows : using MinGW+MSYS to create DLL diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index cfb756ff2..92c22a96c 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -29,6 +29,8 @@ # endif #endif +#define __STDC_WANT_LIB_EXT1__ 1 /* request C11 Annex K, which includes qsort_s() */ + #include /* fprintf */ #include /* malloc, free, qsort_r */ @@ -39,6 +41,7 @@ # define ZDICT_STATIC_LINKING_ONLY #endif +#include "../common/debug.h" /* DEBUG_STATIC_ASSERT */ #include "../common/mem.h" /* read */ #include "../common/pool.h" /* POOL_ctx */ #include "../common/threading.h" /* ZSTD_pthread_mutex_t */ @@ -60,6 +63,32 @@ #define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB)) #define COVER_DEFAULT_SPLITPOINT 1.0 +/** + * Select the qsort() variant used by cover + */ +#define ZDICT_QSORT_MIN 0 +#define ZDICT_QSORT_C90 ZDICT_QSORT_MIN +#define ZDICT_QSORT_GNU 1 +#define ZDICT_QSORT_APPLE 2 +#define ZDICT_QSORT_MSVC 3 +#define ZDICT_QSORT_C11 ZDICT_QSORT_MAX +#define ZDICT_QSORT_MAX 4 + +#ifndef ZDICT_QSORT +# if defined(__APPLE__) +# define ZDICT_QSORT ZDICT_QSORT_APPLE /* uses qsort_r() with a different order for parameters */ +# elif defined(_GNU_SOURCE) +# define ZDICT_QSORT ZDICT_QSORT_GNU /* uses qsort_r() */ +# elif defined(_WIN32) && defined(_MSC_VER) +# define ZDICT_QSORT ZDICT_QSORT_MSVC /* uses qsort_s() with a different order for parameters */ +# elif defined(STDC_LIB_EXT1) && (STDC_LIB_EXT1 > 0) /* C11 Annex K */ +# define ZDICT_QSORT ZDICT_QSORT_C11 /* uses qsort_s() */ +# else +# define ZDICT_QSORT ZDICT_QSORT_C90 /* uses standard qsort() which is not re-entrant (requires global variable) */ +# endif +#endif + + /*-************************************* * Console display * @@ -180,7 +209,7 @@ static U32 *COVER_map_at(COVER_map_t *map, U32 key) { */ static void COVER_map_remove(COVER_map_t *map, U32 key) { U32 i = COVER_map_index(map, key); - COVER_map_pair_t *del = &map->data[i]; + COVER_map_pair_t* del = &map->data[i]; U32 shift = 1; if (del->value == MAP_EMPTY_VALUE) { return; @@ -234,8 +263,7 @@ typedef struct { int displayLevel; } COVER_ctx_t; -#if defined(ZSTD_USE_C90_QSORT) \ - || (!defined(_GNU_SOURCE) && !defined(__APPLE__) && !defined(_MSC_VER)) +#if ZDICT_QSORT == ZDICT_QSORT_C90 /* Use global context for non-reentrant sort functions */ static COVER_ctx_t *g_coverCtx = NULL; #endif @@ -282,9 +310,9 @@ static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) { /** * Same as COVER_cmp() except ties are broken by pointer value */ -#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__) +#if (ZDICT_QSORT == ZDICT_QSORT_MSVC) || (ZDICT_QSORT == ZDICT_QSORT_APPLE) static int WIN_CDECL COVER_strict_cmp(void* g_coverCtx, const void* lp, const void* rp) { -#elif defined(_GNU_SOURCE) +#elif (ZDICT_QSORT == ZDICT_QSORT_GNU) || (ZDICT_QSORT == ZDICT_QSORT_C11) static int COVER_strict_cmp(const void *lp, const void *rp, void *g_coverCtx) { #else /* C90 fallback.*/ static int COVER_strict_cmp(const void *lp, const void *rp) { @@ -298,9 +326,9 @@ static int COVER_strict_cmp(const void *lp, const void *rp) { /** * Faster version for d <= 8. */ -#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__) +#if (ZDICT_QSORT == ZDICT_QSORT_MSVC) || (ZDICT_QSORT == ZDICT_QSORT_APPLE) static int WIN_CDECL COVER_strict_cmp8(void* g_coverCtx, const void* lp, const void* rp) { -#elif defined(_GNU_SOURCE) +#elif (ZDICT_QSORT == ZDICT_QSORT_GNU) || (ZDICT_QSORT == ZDICT_QSORT_C11) static int COVER_strict_cmp8(const void *lp, const void *rp, void *g_coverCtx) { #else /* C90 fallback.*/ static int COVER_strict_cmp8(const void *lp, const void *rp) { @@ -317,23 +345,25 @@ static int COVER_strict_cmp8(const void *lp, const void *rp) { * Hopefully when C11 become the norm, we will be able * to clean it up. */ -static void stableSort(COVER_ctx_t *ctx) { -#if defined(__APPLE__) +static void stableSort(COVER_ctx_t *ctx) +{ + DEBUG_STATIC_ASSERT(ZDICT_QSORT_MIN <= ZDICT_QSORT && ZDICT_QSORT <= ZDICT_QSORT_MAX); +#if (ZDICT_QSORT == ZDICT_QSORT_APPLE) qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32), ctx, (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); -#elif defined(_GNU_SOURCE) && !defined(ZSTD_USE_C90_QSORT) +#elif (ZDICT_QSORT == ZDICT_QSORT_GNU) qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32), (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp), ctx); -#elif defined(_WIN32) && defined(_MSC_VER) +#elif (ZDICT_QSORT == ZDICT_QSORT_MSVC) + qsort_s(ctx->suffix, ctx->suffixSize, sizeof(U32), + (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp), + ctx); +#elif (ZDICT_QSORT == ZDICT_QSORT_C11) qsort_s(ctx->suffix, ctx->suffixSize, sizeof(U32), (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp), ctx); -#elif defined(__OpenBSD__) - g_coverCtx = ctx; - mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32), - (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); #else /* C90 fallback.*/ g_coverCtx = ctx; /* TODO(cavalcanti): implement a reentrant qsort() when _r is not available. */ @@ -782,7 +812,7 @@ ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover( BYTE* const dict = (BYTE*)dictBuffer; COVER_ctx_t ctx; COVER_map_t activeDmers; - const int displayLevel = parameters.zParams.notificationLevel; + const int displayLevel = (int)parameters.zParams.notificationLevel; parameters.splitPoint = 1.0; /* Checks */ if (!COVER_checkParameters(parameters, dictBufferCapacity)) { diff --git a/tests/playTests.sh b/tests/playTests.sh index 72a47d634..15f6e7761 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -43,7 +43,7 @@ roundTripTest() { rm -f tmp1 tmp2 println "roundTripTest: datagen $1 $proba | zstd -v$cLevel | zstd -d$dLevel" datagen $1 $proba | $MD5SUM > tmp1 - datagen $1 $proba | zstd --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2 + datagen $1 $proba | zstd -T0 --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2 $DIFF -q tmp1 tmp2 } @@ -65,7 +65,7 @@ fileRoundTripTest() { println "fileRoundTripTest: datagen $1 $local_p > tmp && zstd -v$local_c -c tmp | zstd -d$local_d" datagen $1 $local_p > tmp < tmp $MD5SUM > tmp.md5.1 - zstd --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2 + zstd -T0 --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2 $DIFF -q tmp.md5.1 tmp.md5.2 } @@ -1885,36 +1885,36 @@ println "\n===> cover dictionary builder : advanced options " TESTFILE="$PRGDIR"/zstdcli.c datagen > tmpDict println "- Create first dictionary" -zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict +zstd -T0 --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict cp "$TESTFILE" tmp zstd -f tmp -D tmpDict zstd -f tmp -D tmpDict --patch-from=tmpDict && die "error: can't use -D and --patch-from=#at the same time" zstd -d tmp.zst -D tmpDict -fo result $DIFF "$TESTFILE" result -zstd --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)" +zstd -T0 --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)" println "- Create second (different) dictionary" -zstd --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC +zstd -T0 --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" println "- Create dictionary using shrink-dict flag" -zstd --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict -zstd --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1 -zstd --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2 -zstd --train-cover=shrink=5,steps=256 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict3 +zstd -T0 --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict +zstd -T0 --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1 +zstd -T0 --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2 +zstd -T0 --train-cover=shrink=5,steps=256 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict3 println "- Create dictionary with short dictID" -zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 +zstd -T0 --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" println "- Create dictionary with size limit" -zstd --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K +zstd -T0 --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K println "- Compare size of dictionary from 90% training samples with 80% training samples" -zstd --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c -zstd --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +zstd -T0 --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +zstd -T0 --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c println "- Create dictionary using all samples for both training and testing" -zstd --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +zstd -T0 --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c println "- Test -o before --train-cover" rm -f tmpDict dictionary zstd -o tmpDict --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c test -f tmpDict -zstd --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c +zstd -T0 --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c test -f dictionary rm -f tmp* dictionary