ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2758 from facebook/qemu

Browse Source 
added qemu tests
This commit is contained in:
Yann Collet
2021-08-31 09:56:50 -07:00
committed by GitHub
parent 6933ac67d3 c1de65535f
commit 6715096611
6 changed files with 101 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/dev-long-tests.yml
+7 -1
View File
@@ -1,7 +1,7 @@
name: dev-long-tests name: dev-long-tests
# Tests longer than 10mn # Tests longer than 10mn
concurrency: concurrency:
group: long-${{ github.ref }} group: long-${{ github.ref }}
cancel-in-progress: true cancel-in-progress: true
@@ -10,6 +10,7 @@ on:
branches: [ dev, release, actionsTest ] branches: [ dev, release, actionsTest ]
jobs: jobs:
# lasts ~24mn
make-test: make-test:
runs-on: ubuntu-latest runs-on: ubuntu-latest
env: env:
@@ -20,6 +21,7 @@ jobs:
- name: make test - name: make test
run: make test run: make test
# lasts ~26mn
make-test-osx: make-test-osx:
runs-on: macos-latest runs-on: macos-latest
steps: steps:
@@ -41,6 +43,7 @@ jobs:
- name: thread sanitizer zstreamtest - name: thread sanitizer zstreamtest
run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream
# lasts ~15mn
tsan-fuzztest: tsan-fuzztest:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
@@ -48,6 +51,7 @@ jobs:
- name: thread sanitizer fuzztest - name: thread sanitizer fuzztest
run: CC=clang make tsan-fuzztest run: CC=clang make tsan-fuzztest
# lasts ~23mn
gcc-8-asan-ubsan-testzstd: gcc-8-asan-ubsan-testzstd:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
@@ -113,6 +117,7 @@ jobs:
sudo apt-get install clang sudo apt-get install clang
CC=clang FUZZER_FLAGS="--long-tests" make clean msan-fuzztest CC=clang FUZZER_FLAGS="--long-tests" make clean msan-fuzztest
# lasts ~24mn
clang-msan-testzstd: clang-msan-testzstd:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
@@ -145,6 +150,7 @@ jobs:
make clean make clean
make -C tests test-fuzzer-stackmode make -C tests test-fuzzer-stackmode
# lasts ~20mn
oss-fuzz: oss-fuzz:
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
.github/workflows/dev-short-tests.yml
+66 -1
View File
@@ -200,8 +200,73 @@ jobs:
make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
qemu-consistency:
name: QEMU ${{ matrix.name }}
runs-on: ubuntu-latest
strategy:
fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed.
matrix:
include: [
{ name: ARM, xcc_pkg: gcc-arm-linux-gnueabi, xcc: arm-linux-gnueabi-gcc, xemu_pkg: qemu-system-arm, xemu: qemu-arm-static },
{ name: ARM64, xcc_pkg: gcc-aarch64-linux-gnu, xcc: aarch64-linux-gnu-gcc, xemu_pkg: qemu-system-arm, xemu: qemu-aarch64-static },
{ name: PPC, xcc_pkg: gcc-powerpc-linux-gnu, xcc: powerpc-linux-gnu-gcc, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc-static },
{ name: PPC64LE, xcc_pkg: gcc-powerpc64le-linux-gnu, xcc: powerpc64le-linux-gnu-gcc, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc64le-static },
{ name: S390X, xcc_pkg: gcc-s390x-linux-gnu, xcc: s390x-linux-gnu-gcc, xemu_pkg: qemu-system-s390x, xemu: qemu-s390x-static },
{ name: MIPS, xcc_pkg: gcc-mips-linux-gnu, xcc: mips-linux-gnu-gcc, xemu_pkg: qemu-system-mips, xemu: qemu-mips-static },
{ name: M68K, xcc_pkg: gcc-m68k-linux-gnu, xcc: m68k-linux-gnu-gcc, xemu_pkg: qemu-system-m68k, xemu: qemu-m68k-static },
]
env: # Set environment variables
XCC: ${{ matrix.xcc }}
XEMU: ${{ matrix.xemu }}
steps:
- uses: actions/checkout@v2 # https://github.com/actions/checkout
- name: apt update & install
run: |
sudo apt-get update
sudo apt-get install gcc-multilib g++-multilib qemu-utils qemu-user-static
sudo apt-get install ${{ matrix.xcc_pkg }} ${{ matrix.xemu_pkg }}
- name: Environment info
run: |
echo && which $XCC
echo && $XCC --version
echo && $XCC -v # Show built-in specs
echo && which $XEMU
echo && $XEMU --version
- name: ARM
if: ${{ matrix.name == 'ARM' }}
run: |
LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
- name: ARM64
if: ${{ matrix.name == 'ARM64' }}
run: |
LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
- name: PPC
if: ${{ matrix.name == 'PPC' }}
run: |
LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
- name: PPC64LE
if: ${{ matrix.name == 'PPC64LE' }}
run: |
LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
- name: S390X
if: ${{ matrix.name == 'S390X' }}
run: |
LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
- name: MIPS
if: ${{ matrix.name == 'MIPS' }}
run: |
LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
- name: M68K
if: ${{ matrix.name == 'M68K' }}
continue-on-error: true # disable reporting errors (alignment issues)
run: |
LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
# This test currently fails on Github Actions specifically. # This test currently fails on Github Actions specifically.
# No clear reason, as the same test works fine locally and on travisCI. # Possible reason : TTY emulation.
# Note that the same test works fine locally and on travisCI.
# This will have to be fixed before transfering the test to GA. # This will have to be fixed before transfering the test to GA.
# versions-compatibility: # versions-compatibility:
# runs-on: ubuntu-latest # runs-on: ubuntu-latest
lib/compress/fse_compress.c
+19 -28
View File
@@ -78,12 +78,11 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
U32 const maxSV1 = maxSymbolValue+1; U32 const maxSV1 = maxSymbolValue+1;
U16* cumul = (U16*)workSpace; /* size = maxSV1 */ U16* cumul = (U16*)workSpace; /* size = maxSV1 */
FSE_FUNCTION_TYPE* tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSV1+1)); /* size = tableSize */ FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSV1+1)); /* size = tableSize */
BYTE* spread = tableSymbol + tableSize; /* size = tableSize */
U32 highThreshold = tableSize-1; U32 highThreshold = tableSize-1;
if ((size_t)workSpace & 3) return ERROR(GENERIC); /* Must be 4 byte aligned */ assert(((size_t)workSpace & 1) == 0); /* Must be 2 bytes-aligned */
if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge); if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge);
/* CTable header */ /* CTable header */
tableU16[-2] = (U16) tableLog; tableU16[-2] = (U16) tableLog;
@@ -105,7 +104,9 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
cumul[u] = cumul[u-1] + 1; cumul[u] = cumul[u-1] + 1;
tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1);
} else { } else {
cumul[u] = cumul[u-1] + normalizedCounter[u-1]; assert(normalizedCounter[u-1] >= 0);
cumul[u] = cumul[u-1] + (U16)normalizedCounter[u-1];
assert(cumul[u] >= cumul[u-1]); /* no overflow */
} } } }
cumul[maxSV1] = (U16)(tableSize+1); cumul[maxSV1] = (U16)(tableSize+1);
} }
@@ -115,8 +116,8 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
/* Case for no low prob count symbols. Lay down 8 bytes at a time /* Case for no low prob count symbols. Lay down 8 bytes at a time
* to reduce branch misses since we are operating on a small block * to reduce branch misses since we are operating on a small block
*/ */
{ BYTE* const spread = tableSymbol + tableSize; /* size = tableSize */
U64 const add = 0x0101010101010101ull; { U64 const add = 0x0101010101010101ull;
size_t pos = 0; size_t pos = 0;
U64 sv = 0; U64 sv = 0;
U32 s; U32 s;
@@ -127,15 +128,15 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
for (i = 8; i < n; i += 8) { for (i = 8; i < n; i += 8) {
MEM_write64(spread + pos + i, sv); MEM_write64(spread + pos + i, sv);
} }
pos += n; assert(n>=0);
pos += (size_t)n;
} }
} }
/* Spread symbols across the table. Lack of lowprob symbols means that /* Spread symbols across the table. Lack of lowprob symbols means that
* we don't need variable sized inner loop, so we can unroll the loop and * we don't need variable sized inner loop, so we can unroll the loop and
* reduce branch misses. * reduce branch misses.
*/ */
{ { size_t position = 0;
size_t position = 0;
size_t s; size_t s;
size_t const unroll = 2; /* Experimentally determined optimal unroll */ size_t const unroll = 2; /* Experimentally determined optimal unroll */
assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
@@ -147,7 +148,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
} }
position = (position + (unroll * step)) & tableMask; position = (position + (unroll * step)) & tableMask;
} }
assert(position == 0); assert(position == 0); /* Must have initialized all positions */
} }
} else { } else {
U32 position = 0; U32 position = 0;
@@ -161,7 +162,6 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
while (position > highThreshold) while (position > highThreshold)
position = (position + step) & tableMask; /* Low proba area */ position = (position + step) & tableMask; /* Low proba area */
} } } }
assert(position==0); /* Must have initialized all positions */ assert(position==0); /* Must have initialized all positions */
} }
@@ -185,16 +185,17 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
case -1: case -1:
case 1: case 1:
symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog); symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog);
symbolTT[s].deltaFindState = total - 1; assert(total <= INT_MAX);
symbolTT[s].deltaFindState = (int)(total - 1);
total ++; total ++;
break; break;
default : default :
{ assert(normalizedCounter[s] > 1);
U32 const maxBitsOut = tableLog - BIT_highbit32 (normalizedCounter[s]-1); { U32 const maxBitsOut = tableLog - BIT_highbit32 ((U32)normalizedCounter[s]-1);
U32 const minStatePlus = normalizedCounter[s] << maxBitsOut; U32 const minStatePlus = (U32)normalizedCounter[s] << maxBitsOut;
symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
symbolTT[s].deltaFindState = total - normalizedCounter[s]; symbolTT[s].deltaFindState = (int)(total - (unsigned)normalizedCounter[s]);
total += normalizedCounter[s]; total += (unsigned)normalizedCounter[s];
} } } } } } } }
#if 0 /* debug : symbol costs */ #if 0 /* debug : symbol costs */
@@ -205,26 +206,16 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
symbol, normalizedCounter[symbol], symbol, normalizedCounter[symbol],
FSE_getMaxNbBits(symbolTT, symbol), FSE_getMaxNbBits(symbolTT, symbol),
(double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256); (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256);
} } }
}
#endif #endif
return 0; return 0;
} }
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */
return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, tableSymbol, sizeof(tableSymbol));
}
#endif
#ifndef FSE_COMMONDEFS_ONLY #ifndef FSE_COMMONDEFS_ONLY
/*-************************************************************** /*-**************************************************************
* FSE NCount encoding * FSE NCount encoding
****************************************************************/ ****************************************************************/
lib/compress/zstd_compress_sequences.c
+4 -3
View File
@@ -275,10 +275,11 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
assert(nbSeq_1 > 1); assert(nbSeq_1 > 1);
assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp)); assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp));
(void)entropyWorkspaceSize; (void)entropyWorkspaceSize;
FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), ""); FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), "FSE_normalizeCount failed");
{ size_t const NCountSize = FSE_writeNCount(op, oend - op, wksp->norm, max, tableLog); /* overflow protected */ assert(oend >= op);
{ size_t const NCountSize = FSE_writeNCount(op, (size_t)(oend - op), wksp->norm, max, tableLog); /* overflow protected */
FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed"); FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed");
FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), ""); FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), "FSE_buildCTable_wksp failed");
return NCountSize; return NCountSize;
} }
} }
programs/Makefile
+3 -3
View File
@@ -156,7 +156,7 @@ endif
# zlib detection # zlib detection
NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
HAVE_ZLIB := $(shell printf '$(NUM_SYMBOL)include <zlib.h>\nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c) HAVE_ZLIB ?= $(shell printf '$(NUM_SYMBOL)include <zlib.h>\nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c)
ifeq ($(HAVE_ZLIB), 1) ifeq ($(HAVE_ZLIB), 1)
ZLIB_MSG := ==> building zstd with .gz compression support ZLIB_MSG := ==> building zstd with .gz compression support
ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
@@ -167,7 +167,7 @@ endif
# lzma detection # lzma detection
NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support
HAVE_LZMA := $(shell printf '$(NUM_SYMBOL)include <lzma.h>\nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c) HAVE_LZMA ?= $(shell printf '$(NUM_SYMBOL)include <lzma.h>\nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c)
ifeq ($(HAVE_LZMA), 1) ifeq ($(HAVE_LZMA), 1)
LZMA_MSG := ==> building zstd with .xz/.lzma compression support LZMA_MSG := ==> building zstd with .xz/.lzma compression support
LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
@@ -178,7 +178,7 @@ endif
# lz4 detection # lz4 detection
NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support
HAVE_LZ4 := $(shell printf '$(NUM_SYMBOL)include <lz4frame.h>\n$(NUM_SYMBOL)include <lz4.h>\nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c) HAVE_LZ4 ?= $(shell printf '$(NUM_SYMBOL)include <lz4frame.h>\n$(NUM_SYMBOL)include <lz4.h>\nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c)
ifeq ($(HAVE_LZ4), 1) ifeq ($(HAVE_LZ4), 1)
LZ4_MSG := ==> building zstd with .lz4 compression support LZ4_MSG := ==> building zstd with .lz4 compression support
LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS
tests/playTests.sh
+2 -1
View File
@@ -166,7 +166,8 @@ if [ -z "${DATAGEN_BIN}" ]; then
DATAGEN_BIN="$TESTDIR/datagen" DATAGEN_BIN="$TESTDIR/datagen"
fi fi
ZSTD_BIN="$EXE_PREFIX$ZSTD_BIN" # Why was this line here ? Generates a strange ZSTD_BIN when EXE_PREFIX is non empty
# ZSTD_BIN="$EXE_PREFIX$ZSTD_BIN"
# assertions # assertions
[ -n "$ZSTD_BIN" ] || die "zstd not found at $ZSTD_BIN! \n Please define ZSTD_BIN pointing to the zstd binary. You might also consider rebuilding zstd follwing the instructions in README.md" [ -n "$ZSTD_BIN" ] || die "zstd not found at $ZSTD_BIN! \n Please define ZSTD_BIN pointing to the zstd binary. You might also consider rebuilding zstd follwing the instructions in README.md"