ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4336 from Cyan4973/qsort_v2

Browse Source 
fix #4312 - musl compilation compatibility
This commit is contained in:
Yann Collet
2025-03-12 10:31:54 -07:00
committed by GitHub
parent 68dfd14a8c 4d53e27144
commit eca205fc78
4 changed files with 67 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/dev-short-tests.yml
+2 -2
View File
@@ -693,9 +693,9 @@ jobs:
- name: Install musl-tools
run: |
sudo apt install -y musl-tools
- name: Compile the project with musl-gcc
- name: Compile with musl-gcc and test-zstd
run: |
CC=musl-gcc CPPFLAGS=-DZSTD_USE_C90_QSORT make -j V=1 zstd
CC=musl-gcc CFLAGS="-Werror -O3" CPPFLAGS=-DZDICT_QSORT=ZDICT_QSORT_C90 make -j -C tests test-zstd V=1
intel-cet-compatibility:
runs-on: ubuntu-latest
lib/README.md
+4 -3
View File
@@ -193,9 +193,10 @@ The file structure is designed to make this selection manually achievable for an
and assembly decoding loops. You may want to use this macro if these loops are
slower on your platform.
- The macro `ZSTD_USE_C90_QSORT` forces usage of C90's `qsort()`,
for situations where the code cannot determine that `qsort_r()` is not supported,
such as, for example, older versions of `musl`.
- The macro `ZDICT_QSORT` can enforce selection of a specific sorting variant,
which is useful when autodetection fails, for example with older versions of `musl`.
For this scenario, it can be set as `ZDICT_QSORT=ZDICT_QSORT_C90`.
Other selectable suffixes are `_GNU`, `_APPLE`, `_MSVC` and `_C11`.
#### Windows : using MinGW+MSYS to create DLL
lib/dictBuilder/cover.c
+45 -15
View File
@@ -29,6 +29,8 @@
# endif
#endif
#define __STDC_WANT_LIB_EXT1__ 1 /* request C11 Annex K, which includes qsort_s() */
#include <stdio.h> /* fprintf */
#include <stdlib.h> /* malloc, free, qsort_r */
@@ -39,6 +41,7 @@
# define ZDICT_STATIC_LINKING_ONLY
#endif
#include "../common/debug.h" /* DEBUG_STATIC_ASSERT */
#include "../common/mem.h" /* read */
#include "../common/pool.h" /* POOL_ctx */
#include "../common/threading.h" /* ZSTD_pthread_mutex_t */
@@ -60,6 +63,32 @@
#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
#define COVER_DEFAULT_SPLITPOINT 1.0
/**
* Select the qsort() variant used by cover
*/
#define ZDICT_QSORT_MIN 0
#define ZDICT_QSORT_C90 ZDICT_QSORT_MIN
#define ZDICT_QSORT_GNU 1
#define ZDICT_QSORT_APPLE 2
#define ZDICT_QSORT_MSVC 3
#define ZDICT_QSORT_C11 ZDICT_QSORT_MAX
#define ZDICT_QSORT_MAX 4
#ifndef ZDICT_QSORT
# if defined(__APPLE__)
# define ZDICT_QSORT ZDICT_QSORT_APPLE /* uses qsort_r() with a different order for parameters */
# elif defined(_GNU_SOURCE)
# define ZDICT_QSORT ZDICT_QSORT_GNU /* uses qsort_r() */
# elif defined(_WIN32) && defined(_MSC_VER)
# define ZDICT_QSORT ZDICT_QSORT_MSVC /* uses qsort_s() with a different order for parameters */
# elif defined(STDC_LIB_EXT1) && (STDC_LIB_EXT1 > 0) /* C11 Annex K */
# define ZDICT_QSORT ZDICT_QSORT_C11 /* uses qsort_s() */
# else
# define ZDICT_QSORT ZDICT_QSORT_C90 /* uses standard qsort() which is not re-entrant (requires global variable) */
# endif
#endif
/*-*************************************
* Console display
*
@@ -234,8 +263,7 @@ typedef struct {
int displayLevel;
} COVER_ctx_t;
#if defined(ZSTD_USE_C90_QSORT) \
|| (!defined(_GNU_SOURCE) && !defined(__APPLE__) && !defined(_MSC_VER))
#if ZDICT_QSORT == ZDICT_QSORT_C90
/* Use global context for non-reentrant sort functions */
static COVER_ctx_t *g_coverCtx = NULL;
#endif
@@ -282,9 +310,9 @@ static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) {
/**
* Same as COVER_cmp() except ties are broken by pointer value
*/
#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__)
#if (ZDICT_QSORT == ZDICT_QSORT_MSVC) || (ZDICT_QSORT == ZDICT_QSORT_APPLE)
static int WIN_CDECL COVER_strict_cmp(void* g_coverCtx, const void* lp, const void* rp) {
#elif defined(_GNU_SOURCE)
#elif (ZDICT_QSORT == ZDICT_QSORT_GNU) || (ZDICT_QSORT == ZDICT_QSORT_C11)
static int COVER_strict_cmp(const void *lp, const void *rp, void *g_coverCtx) {
#else /* C90 fallback.*/
static int COVER_strict_cmp(const void *lp, const void *rp) {
@@ -298,9 +326,9 @@ static int COVER_strict_cmp(const void *lp, const void *rp) {
/**
* Faster version for d <= 8.
*/
#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__)
#if (ZDICT_QSORT == ZDICT_QSORT_MSVC) || (ZDICT_QSORT == ZDICT_QSORT_APPLE)
static int WIN_CDECL COVER_strict_cmp8(void* g_coverCtx, const void* lp, const void* rp) {
#elif defined(_GNU_SOURCE)
#elif (ZDICT_QSORT == ZDICT_QSORT_GNU) || (ZDICT_QSORT == ZDICT_QSORT_C11)
static int COVER_strict_cmp8(const void *lp, const void *rp, void *g_coverCtx) {
#else /* C90 fallback.*/
static int COVER_strict_cmp8(const void *lp, const void *rp) {
@@ -317,23 +345,25 @@ static int COVER_strict_cmp8(const void *lp, const void *rp) {
* Hopefully when C11 become the norm, we will be able
* to clean it up.
*/
static void stableSort(COVER_ctx_t *ctx) {
#if defined(__APPLE__)
static void stableSort(COVER_ctx_t *ctx)
{
DEBUG_STATIC_ASSERT(ZDICT_QSORT_MIN <= ZDICT_QSORT && ZDICT_QSORT <= ZDICT_QSORT_MAX);
#if (ZDICT_QSORT == ZDICT_QSORT_APPLE)
qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32),
ctx,
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
#elif defined(_GNU_SOURCE) && !defined(ZSTD_USE_C90_QSORT)
#elif (ZDICT_QSORT == ZDICT_QSORT_GNU)
qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
ctx);
#elif defined(_WIN32) && defined(_MSC_VER)
#elif (ZDICT_QSORT == ZDICT_QSORT_MSVC)
qsort_s(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
ctx);
#elif (ZDICT_QSORT == ZDICT_QSORT_C11)
qsort_s(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
ctx);
#elif defined(__OpenBSD__)
g_coverCtx = ctx;
mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
#else /* C90 fallback.*/
g_coverCtx = ctx;
/* TODO(cavalcanti): implement a reentrant qsort() when _r is not available. */
@@ -782,7 +812,7 @@ ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
BYTE* const dict = (BYTE*)dictBuffer;
COVER_ctx_t ctx;
COVER_map_t activeDmers;
const int displayLevel = parameters.zParams.notificationLevel;
const int displayLevel = (int)parameters.zParams.notificationLevel;
parameters.splitPoint = 1.0;
/* Checks */
if (!COVER_checkParameters(parameters, dictBufferCapacity)) {
tests/playTests.sh
+15 -15
View File
@@ -43,7 +43,7 @@ roundTripTest() {
rm -f tmp1 tmp2
println "roundTripTest: datagen $1 $proba | zstd -v$cLevel | zstd -d$dLevel"
datagen $1 $proba | $MD5SUM > tmp1
datagen $1 $proba | zstd --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2
datagen $1 $proba | zstd -T0 --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2
$DIFF -q tmp1 tmp2
}
@@ -65,7 +65,7 @@ fileRoundTripTest() {
println "fileRoundTripTest: datagen $1 $local_p > tmp && zstd -v$local_c -c tmp | zstd -d$local_d"
datagen $1 $local_p > tmp
< tmp $MD5SUM > tmp.md5.1
zstd --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2
zstd -T0 --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2
$DIFF -q tmp.md5.1 tmp.md5.2
}
@@ -1885,36 +1885,36 @@ println "\n===> cover dictionary builder : advanced options "
TESTFILE="$PRGDIR"/zstdcli.c
datagen > tmpDict
println "- Create first dictionary"
zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
zstd -T0 --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
cp "$TESTFILE" tmp
zstd -f tmp -D tmpDict
zstd -f tmp -D tmpDict --patch-from=tmpDict && die "error: can't use -D and --patch-from=#at the same time"
zstd -d tmp.zst -D tmpDict -fo result
$DIFF "$TESTFILE" result
zstd --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)"
zstd -T0 --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)"
println "- Create second (different) dictionary"
zstd --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
zstd -T0 --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
println "- Create dictionary using shrink-dict flag"
zstd --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict
zstd --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1
zstd --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2
zstd --train-cover=shrink=5,steps=256 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict3
zstd -T0 --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict
zstd -T0 --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1
zstd -T0 --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2
zstd -T0 --train-cover=shrink=5,steps=256 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict3
println "- Create dictionary with short dictID"
zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
zstd -T0 --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
println "- Create dictionary with size limit"
zstd --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
zstd -T0 --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
println "- Compare size of dictionary from 90% training samples with 80% training samples"
zstd --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
zstd --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
zstd -T0 --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
zstd -T0 --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary using all samples for both training and testing"
zstd --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
zstd -T0 --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Test -o before --train-cover"
rm -f tmpDict dictionary
zstd -o tmpDict --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f tmpDict
zstd --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c
zstd -T0 --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f dictionary
rm -f tmp* dictionary