From ef6953984954dbb5dd333d277a5700ff35df6781 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 29 Aug 2021 11:53:56 -0700 Subject: [PATCH 1/5] transferred inter-versions compatibility tests to GA --- .github/workflows/dev-short-tests.yml | 14 +++++++++++--- .travis.yml | 19 ++++--------------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml index 1f676c46b..bd3e16da9 100644 --- a/.github/workflows/dev-short-tests.yml +++ b/.github/workflows/dev-short-tests.yml @@ -2,7 +2,7 @@ name: dev-short-tests # Faster tests: mostly build tests, along with some other # misc tests -concurrency: +concurrency: group: fast-${{ github.ref }} cancel-in-progress: true @@ -140,7 +140,7 @@ jobs: sudo apt-get -qqq update make libc6install CFLAGS="-Werror -m32" make -j all32 - + gcc-8-make: runs-on: ubuntu-latest steps: @@ -184,7 +184,7 @@ jobs: run: > msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}} - + minimal-decompressor-macros: runs-on: ubuntu-latest steps: @@ -200,6 +200,14 @@ jobs: make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" + versions-compatibility: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Versions Compatibility Test + run: | + make -C tests versionsTest + # For reference : icc tests # icc tests are currently failing on Github Actions, likely to issues during installation stage # To be fixed later diff --git a/.travis.yml b/.travis.yml index 7b202cb88..85fbf78b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,31 +43,19 @@ matrix: arch: arm64 script: - make check - + - name: arm64fuzz os: linux arch: arm64 script: - make -C tests fuzztest - - # TODO: migrate to GH actions once warnings are fixed - - name: Minimal Decompressor Macros # ~5mn - script: - - make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror" - - make clean && make check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror" - - make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT" - - make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT" - - make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG" - - make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG" - - make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" - - make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" # TODO: migrate to GH actions once newest clang staticanalyze warnings are fixed - name: static analyzer scanbuild # ~26mn dist: trusty # note : it's important to pin down a version of static analyzer, since different versions report different false positives script: - make staticAnalyze - + # GH actions can't run this command on OS-X, non-tty issues - name: OS-X make all lib os: osx @@ -115,7 +103,8 @@ matrix: script: - make arminstall - make aarch64fuzz - + + # To be removed once confirmed transferred to GA - name: Versions Compatibility Test # 11.5mn script: - make -C tests versionsTest From 7f37b8a547e960e58dd3c247925a66dcb0412f77 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 29 Aug 2021 14:48:11 -0700 Subject: [PATCH 2/5] accelerate versionsCompatibilityTest by allowing parallel build of units, and reducing optimization levels. Parallel build is only effective on "recent" versions of `zstd`, as previously, the list of units was passed as a list of source files, which is something neither `make` nor `gcc` can parallelize. So its impact is mildly effective (-20%). Reducing optimization level to `-O1` makes compilation much faster. It also makes runtime slower, but in this test, compilation time dominates run time. The savings are very significant (-50%). On my test system, it reduces the length of this test from 13mn to 5mn. --- tests/test-zstd-versions.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index c86af7ddf..cc1693a26 100755 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -23,6 +23,7 @@ from subprocess import Popen, PIPE repo_url = 'https://github.com/facebook/zstd.git' tmp_dir_name = 'tests/versionsTest' make_cmd = 'make' +make_args = ['-j','CFLAGS=-O1'] git_cmd = 'git' test_dat_src = 'README.md' test_dat = 'test_dat' @@ -56,8 +57,11 @@ def proc(cmd_args, pipe=True, dummy=False): return subproc.communicate() -def make(args, pipe=True): - return proc([make_cmd] + args, pipe) +def make(targets, pipe=True): + cmd = [make_cmd] + make_args + targets + cmd_str = str(cmd) + print('compilation command : ' + cmd_str) + return proc(cmd, pipe) def git(args, pipe=True): @@ -223,20 +227,25 @@ if __name__ == '__main__': dst_zstd = '{}/zstd.{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/zstd. if not os.path.isfile(dst_zstd) or tag == head: if tag != head: + print('-----------------------------------------------') + print('compiling ' + tag) + print('-----------------------------------------------') r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/ os.makedirs(r_dir, exist_ok=True) os.chdir(clone_dir) git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False) if tag == 'v0.5.0': os.chdir(r_dir + '/dictBuilder') # /path/to/zstd/tests/versionsTest/v0.5.0/dictBuilder - make(['clean', 'dictBuilder'], False) + make(['clean'], False) # separate 'clean' target to allow parallel build + make(['dictBuilder'], False) shutil.copy2('dictBuilder', '{}/dictBuilder.{}'.format(tmp_dir, tag)) os.chdir(r_dir + '/programs') # /path/to/zstd/tests/versionsTest//programs - make(['clean', 'zstd'], False) + make(['clean'], False) # separate 'clean' target to allow parallel build + make(['zstd'], False) else: os.chdir(programs_dir) make(['zstd'], False) - shutil.copy2('zstd', dst_zstd) + shutil.copy2('zstd', dst_zstd) # remove any remaining *.zst and *.dec from previous test os.chdir(tmp_dir) From 72bd2a83a01c70938f4d55337ecc2c3211694a2a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 29 Aug 2021 15:26:31 -0700 Subject: [PATCH 3/5] reduce length of scanbuild static analyzer test This was ~30mn, by far the longest run on travisCI. That's because it re-analyzes multiple times the same files (library files notably). It also performs actions that make no sense for the static analyzer purpose, such as building the single-file library. Reduced time spent in this test by reducing its scope : just build the CLI, and obviously the library along it. These are the only ones that really deserve to be analyzed. Unfortunately, it still results in a number of false positives when using newer versions of scanbuild (each version of scanbuild generates a different list of false positives). These will have to be fixed before transfering to Github Actions. --- .travis.yml | 7 ++++--- Makefile | 2 +- tests/test-zstd-versions.py | 5 +++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 85fbf78b8..1acbfecf1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,7 +51,7 @@ matrix: - make -C tests fuzztest # TODO: migrate to GH actions once newest clang staticanalyze warnings are fixed - - name: static analyzer scanbuild # ~26mn + - name: static analyzer scanbuild # ~8mn dist: trusty # note : it's important to pin down a version of static analyzer, since different versions report different false positives script: - make staticAnalyze @@ -104,8 +104,9 @@ matrix: - make arminstall - make aarch64fuzz - # To be removed once confirmed transferred to GA - - name: Versions Compatibility Test # 11.5mn + # This test currently fails on GA specifically, for no obvious reason + # (it works fine on travisCI, and on local test platforms). + - name: Versions Compatibility Test # ~6mn script: - make -C tests versionsTest diff --git a/Makefile b/Makefile index c1908f0a1..b9265e7f8 100644 --- a/Makefile +++ b/Makefile @@ -416,5 +416,5 @@ bmi32build: clean staticAnalyze: SCANBUILD ?= scan-build staticAnalyze: $(CC) -v - CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) allzstd examples contrib + CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) zstd endif diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index cc1693a26..baca251f5 100755 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -244,6 +244,9 @@ if __name__ == '__main__': make(['zstd'], False) else: os.chdir(programs_dir) + print('-----------------------------------------------') + print('compiling head') + print('-----------------------------------------------') make(['zstd'], False) shutil.copy2('zstd', dst_zstd) @@ -260,7 +263,9 @@ if __name__ == '__main__': print('cp ' + dict_files + ' ' + dict_source_path) execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True) + print('-----------------------------------------------') print('Compress test.dat by all released zstd') + print('-----------------------------------------------') error_code = 0 for tag in tags: From b341aa2f95e0ec363a593452faa58b6e6ac1f587 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 29 Aug 2021 15:47:04 -0700 Subject: [PATCH 4/5] remove versions-compatibility test from GA since it fails on Github Actions specifically. The test is run on TravisCI for the time being. Its duration has been reduced to ~6mn anyway. --- .github/workflows/dev-short-tests.yml | 18 +++++++++++------- .travis.yml | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml index bd3e16da9..85c966d36 100644 --- a/.github/workflows/dev-short-tests.yml +++ b/.github/workflows/dev-short-tests.yml @@ -200,13 +200,17 @@ jobs: make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" - versions-compatibility: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Versions Compatibility Test - run: | - make -C tests versionsTest +# This test currently fails on Github Actions specifically. +# No clear reason, as the same test works fine locally and on travisCI. +# This will have to be fixed before transfering the test to GA. +# versions-compatibility: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v2 +# - name: Versions Compatibility Test +# run: | +# make -C tests versionsTest + # For reference : icc tests # icc tests are currently failing on Github Actions, likely to issues during installation stage diff --git a/.travis.yml b/.travis.yml index 1acbfecf1..1e963246c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,7 @@ matrix: script: - make -C tests fuzztest - # TODO: migrate to GH actions once newest clang staticanalyze warnings are fixed + # TODO: migrate to GH Actions once newest clang staticanalyze warnings are fixed - name: static analyzer scanbuild # ~8mn dist: trusty # note : it's important to pin down a version of static analyzer, since different versions report different false positives script: From 1e5c90cb5b9f562ed9d23abb793c1f13d6aacdd0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 29 Aug 2021 20:54:18 -0700 Subject: [PATCH 5/5] remove qemu tests that are being transfered to GA in #2758. This represents a saving of ~25mn of cpu time on TravisCI. --- .travis.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1e963246c..6a1295b4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,12 +69,6 @@ matrix: - make arminstall - make armbuild - - name: Qemu PPC + Fuzz Test # ~13mn - dist: trusty # it seems ppc cross-compilation fails on "current" - script: - - make ppcinstall - - make ppcfuzz - # check release number (release/new tag only) - name: Tag-Specific Test if: tag =~ ^v[0-9]\.[0-9] @@ -91,19 +85,6 @@ matrix: - cat /proc/cpuinfo - make -C tests fuzztest - - name: Qemu PPC64 + Fuzz test # ~13mn, presumed Big-Endian (?) - dist: trusty # note : PPC64 cross-compilation for Qemu tests seems broken on Xenial - script: - - make ppcinstall - - make ppc64fuzz - - # note : we already have aarch64 tests on hardware - - name: Qemu aarch64 + Fuzz Test (on Xenial) # ~14mn - dist: xenial - script: - - make arminstall - - make aarch64fuzz - # This test currently fails on GA specifically, for no obvious reason # (it works fine on travisCI, and on local test platforms). - name: Versions Compatibility Test # ~6mn