Merge pull request #4336 from Cyan4973/qsort_v2
fix #4312 - musl compilation compatibility
This commit is contained in:
Yann Collet
@@ -693,9 +693,9 @@ jobs:
|
||||
- name: Install musl-tools
|
||||
run: |
|
||||
sudo apt install -y musl-tools
|
||||
- name: Compile the project with musl-gcc
|
||||
- name: Compile with musl-gcc and test-zstd
|
||||
run: |
|
||||
CC=musl-gcc CPPFLAGS=-DZSTD_USE_C90_QSORT make -j V=1 zstd
|
||||
CC=musl-gcc CFLAGS="-Werror -O3" CPPFLAGS=-DZDICT_QSORT=ZDICT_QSORT_C90 make -j -C tests test-zstd V=1
|
||||
|
||||
intel-cet-compatibility:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
+4
-3
@@ -193,9 +193,10 @@ The file structure is designed to make this selection manually achievable for an
|
||||
and assembly decoding loops. You may want to use this macro if these loops are
|
||||
slower on your platform.
|
||||
|
||||
- The macro `ZSTD_USE_C90_QSORT` forces usage of C90's `qsort()`,
|
||||
for situations where the code cannot determine that `qsort_r()` is not supported,
|
||||
such as, for example, older versions of `musl`.
|
||||
- The macro `ZDICT_QSORT` can enforce selection of a specific sorting variant,
|
||||
which is useful when autodetection fails, for example with older versions of `musl`.
|
||||
For this scenario, it can be set as `ZDICT_QSORT=ZDICT_QSORT_C90`.
|
||||
Other selectable suffixes are `_GNU`, `_APPLE`, `_MSVC` and `_C11`.
|
||||
|
||||
#### Windows : using MinGW+MSYS to create DLL
|
||||
|
||||
|
||||
+46
-16
@@ -29,6 +29,8 @@
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#define __STDC_WANT_LIB_EXT1__ 1 /* request C11 Annex K, which includes qsort_s() */
|
||||
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <stdlib.h> /* malloc, free, qsort_r */
|
||||
|
||||
@@ -39,6 +41,7 @@
|
||||
# define ZDICT_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
|
||||
#include "../common/debug.h" /* DEBUG_STATIC_ASSERT */
|
||||
#include "../common/mem.h" /* read */
|
||||
#include "../common/pool.h" /* POOL_ctx */
|
||||
#include "../common/threading.h" /* ZSTD_pthread_mutex_t */
|
||||
@@ -60,6 +63,32 @@
|
||||
#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
|
||||
#define COVER_DEFAULT_SPLITPOINT 1.0
|
||||
|
||||
/**
|
||||
* Select the qsort() variant used by cover
|
||||
*/
|
||||
#define ZDICT_QSORT_MIN 0
|
||||
#define ZDICT_QSORT_C90 ZDICT_QSORT_MIN
|
||||
#define ZDICT_QSORT_GNU 1
|
||||
#define ZDICT_QSORT_APPLE 2
|
||||
#define ZDICT_QSORT_MSVC 3
|
||||
#define ZDICT_QSORT_C11 ZDICT_QSORT_MAX
|
||||
#define ZDICT_QSORT_MAX 4
|
||||
|
||||
#ifndef ZDICT_QSORT
|
||||
# if defined(__APPLE__)
|
||||
# define ZDICT_QSORT ZDICT_QSORT_APPLE /* uses qsort_r() with a different order for parameters */
|
||||
# elif defined(_GNU_SOURCE)
|
||||
# define ZDICT_QSORT ZDICT_QSORT_GNU /* uses qsort_r() */
|
||||
# elif defined(_WIN32) && defined(_MSC_VER)
|
||||
# define ZDICT_QSORT ZDICT_QSORT_MSVC /* uses qsort_s() with a different order for parameters */
|
||||
# elif defined(STDC_LIB_EXT1) && (STDC_LIB_EXT1 > 0) /* C11 Annex K */
|
||||
# define ZDICT_QSORT ZDICT_QSORT_C11 /* uses qsort_s() */
|
||||
# else
|
||||
# define ZDICT_QSORT ZDICT_QSORT_C90 /* uses standard qsort() which is not re-entrant (requires global variable) */
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Console display
|
||||
*
|
||||
@@ -180,7 +209,7 @@ static U32 *COVER_map_at(COVER_map_t *map, U32 key) {
|
||||
*/
|
||||
static void COVER_map_remove(COVER_map_t *map, U32 key) {
|
||||
U32 i = COVER_map_index(map, key);
|
||||
COVER_map_pair_t *del = &map->data[i];
|
||||
COVER_map_pair_t* del = &map->data[i];
|
||||
U32 shift = 1;
|
||||
if (del->value == MAP_EMPTY_VALUE) {
|
||||
return;
|
||||
@@ -234,8 +263,7 @@ typedef struct {
|
||||
int displayLevel;
|
||||
} COVER_ctx_t;
|
||||
|
||||
#if defined(ZSTD_USE_C90_QSORT) \
|
||||
|| (!defined(_GNU_SOURCE) && !defined(__APPLE__) && !defined(_MSC_VER))
|
||||
#if ZDICT_QSORT == ZDICT_QSORT_C90
|
||||
/* Use global context for non-reentrant sort functions */
|
||||
static COVER_ctx_t *g_coverCtx = NULL;
|
||||
#endif
|
||||
@@ -282,9 +310,9 @@ static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) {
|
||||
/**
|
||||
* Same as COVER_cmp() except ties are broken by pointer value
|
||||
*/
|
||||
#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__)
|
||||
#if (ZDICT_QSORT == ZDICT_QSORT_MSVC) || (ZDICT_QSORT == ZDICT_QSORT_APPLE)
|
||||
static int WIN_CDECL COVER_strict_cmp(void* g_coverCtx, const void* lp, const void* rp) {
|
||||
#elif defined(_GNU_SOURCE)
|
||||
#elif (ZDICT_QSORT == ZDICT_QSORT_GNU) || (ZDICT_QSORT == ZDICT_QSORT_C11)
|
||||
static int COVER_strict_cmp(const void *lp, const void *rp, void *g_coverCtx) {
|
||||
#else /* C90 fallback.*/
|
||||
static int COVER_strict_cmp(const void *lp, const void *rp) {
|
||||
@@ -298,9 +326,9 @@ static int COVER_strict_cmp(const void *lp, const void *rp) {
|
||||
/**
|
||||
* Faster version for d <= 8.
|
||||
*/
|
||||
#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__)
|
||||
#if (ZDICT_QSORT == ZDICT_QSORT_MSVC) || (ZDICT_QSORT == ZDICT_QSORT_APPLE)
|
||||
static int WIN_CDECL COVER_strict_cmp8(void* g_coverCtx, const void* lp, const void* rp) {
|
||||
#elif defined(_GNU_SOURCE)
|
||||
#elif (ZDICT_QSORT == ZDICT_QSORT_GNU) || (ZDICT_QSORT == ZDICT_QSORT_C11)
|
||||
static int COVER_strict_cmp8(const void *lp, const void *rp, void *g_coverCtx) {
|
||||
#else /* C90 fallback.*/
|
||||
static int COVER_strict_cmp8(const void *lp, const void *rp) {
|
||||
@@ -317,23 +345,25 @@ static int COVER_strict_cmp8(const void *lp, const void *rp) {
|
||||
* Hopefully when C11 become the norm, we will be able
|
||||
* to clean it up.
|
||||
*/
|
||||
static void stableSort(COVER_ctx_t *ctx) {
|
||||
#if defined(__APPLE__)
|
||||
static void stableSort(COVER_ctx_t *ctx)
|
||||
{
|
||||
DEBUG_STATIC_ASSERT(ZDICT_QSORT_MIN <= ZDICT_QSORT && ZDICT_QSORT <= ZDICT_QSORT_MAX);
|
||||
#if (ZDICT_QSORT == ZDICT_QSORT_APPLE)
|
||||
qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32),
|
||||
ctx,
|
||||
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
|
||||
#elif defined(_GNU_SOURCE) && !defined(ZSTD_USE_C90_QSORT)
|
||||
#elif (ZDICT_QSORT == ZDICT_QSORT_GNU)
|
||||
qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32),
|
||||
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
|
||||
ctx);
|
||||
#elif defined(_WIN32) && defined(_MSC_VER)
|
||||
#elif (ZDICT_QSORT == ZDICT_QSORT_MSVC)
|
||||
qsort_s(ctx->suffix, ctx->suffixSize, sizeof(U32),
|
||||
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
|
||||
ctx);
|
||||
#elif (ZDICT_QSORT == ZDICT_QSORT_C11)
|
||||
qsort_s(ctx->suffix, ctx->suffixSize, sizeof(U32),
|
||||
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
|
||||
ctx);
|
||||
#elif defined(__OpenBSD__)
|
||||
g_coverCtx = ctx;
|
||||
mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32),
|
||||
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
|
||||
#else /* C90 fallback.*/
|
||||
g_coverCtx = ctx;
|
||||
/* TODO(cavalcanti): implement a reentrant qsort() when _r is not available. */
|
||||
@@ -782,7 +812,7 @@ ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
|
||||
BYTE* const dict = (BYTE*)dictBuffer;
|
||||
COVER_ctx_t ctx;
|
||||
COVER_map_t activeDmers;
|
||||
const int displayLevel = parameters.zParams.notificationLevel;
|
||||
const int displayLevel = (int)parameters.zParams.notificationLevel;
|
||||
parameters.splitPoint = 1.0;
|
||||
/* Checks */
|
||||
if (!COVER_checkParameters(parameters, dictBufferCapacity)) {
|
||||
|
||||
+15
-15
@@ -43,7 +43,7 @@ roundTripTest() {
|
||||
rm -f tmp1 tmp2
|
||||
println "roundTripTest: datagen $1 $proba | zstd -v$cLevel | zstd -d$dLevel"
|
||||
datagen $1 $proba | $MD5SUM > tmp1
|
||||
datagen $1 $proba | zstd --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2
|
||||
datagen $1 $proba | zstd -T0 --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2
|
||||
$DIFF -q tmp1 tmp2
|
||||
}
|
||||
|
||||
@@ -65,7 +65,7 @@ fileRoundTripTest() {
|
||||
println "fileRoundTripTest: datagen $1 $local_p > tmp && zstd -v$local_c -c tmp | zstd -d$local_d"
|
||||
datagen $1 $local_p > tmp
|
||||
< tmp $MD5SUM > tmp.md5.1
|
||||
zstd --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2
|
||||
zstd -T0 --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2
|
||||
$DIFF -q tmp.md5.1 tmp.md5.2
|
||||
}
|
||||
|
||||
@@ -1885,36 +1885,36 @@ println "\n===> cover dictionary builder : advanced options "
|
||||
TESTFILE="$PRGDIR"/zstdcli.c
|
||||
datagen > tmpDict
|
||||
println "- Create first dictionary"
|
||||
zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
|
||||
zstd -T0 --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
|
||||
cp "$TESTFILE" tmp
|
||||
zstd -f tmp -D tmpDict
|
||||
zstd -f tmp -D tmpDict --patch-from=tmpDict && die "error: can't use -D and --patch-from=#at the same time"
|
||||
zstd -d tmp.zst -D tmpDict -fo result
|
||||
$DIFF "$TESTFILE" result
|
||||
zstd --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)"
|
||||
zstd -T0 --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)"
|
||||
println "- Create second (different) dictionary"
|
||||
zstd --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
|
||||
zstd -T0 --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
|
||||
zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
|
||||
println "- Create dictionary using shrink-dict flag"
|
||||
zstd --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict
|
||||
zstd --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1
|
||||
zstd --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2
|
||||
zstd --train-cover=shrink=5,steps=256 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict3
|
||||
zstd -T0 --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict
|
||||
zstd -T0 --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1
|
||||
zstd -T0 --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2
|
||||
zstd -T0 --train-cover=shrink=5,steps=256 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict3
|
||||
println "- Create dictionary with short dictID"
|
||||
zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
|
||||
zstd -T0 --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
|
||||
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
|
||||
println "- Create dictionary with size limit"
|
||||
zstd --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
|
||||
zstd -T0 --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
|
||||
println "- Compare size of dictionary from 90% training samples with 80% training samples"
|
||||
zstd --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
zstd --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
zstd -T0 --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
zstd -T0 --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
println "- Create dictionary using all samples for both training and testing"
|
||||
zstd --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
zstd -T0 --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
println "- Test -o before --train-cover"
|
||||
rm -f tmpDict dictionary
|
||||
zstd -o tmpDict --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
test -f tmpDict
|
||||
zstd --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
zstd -T0 --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
test -f dictionary
|
||||
rm -f tmp* dictionary
|
||||
|
||||
|
||||
Reference in New Issue
Block a user