Merge pull request #2757 from facebook/transferGA

Reduce test time on TravisCI

.github/workflows/dev-short-tests.yml

@@ -200,6 +200,18 @@ jobs:
         make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
         make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
 
+# This test currently fails on Github Actions specifically.
+# No clear reason, as the same test works fine locally and on travisCI.
+# This will have to be fixed before transfering the test to GA.
+#  versions-compatibility:
+#    runs-on: ubuntu-latest
+#    steps:
+#    - uses: actions/checkout@v2
+#    - name: Versions Compatibility Test
+#      run: |
+#        make -C tests versionsTest
+
+
 # For reference : icc tests
 # icc tests are currently failing on Github Actions, likely to issues during installation stage
 # To be fixed later

.travis.yml

@@ -50,20 +50,8 @@ matrix:
       script:
         - make -C tests fuzztest
 
-    # TODO: migrate to GH actions once warnings are fixed
+    # TODO: migrate to GH Actions once newest clang staticanalyze warnings are fixed
-    - name: Minimal Decompressor Macros    # ~5mn
+    - name: static analyzer scanbuild    # ~8mn
-      script:
-        - make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
-        - make clean && make check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
-        - make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
-        - make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
-        - make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
-        - make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
-        - make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
-        - make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
-
-    # TODO: migrate to GH actions once newest clang staticanalyze warnings are fixed
-    - name: static analyzer scanbuild    # ~26mn
       dist: trusty  # note : it's important to pin down a version of static analyzer, since different versions report different false positives
       script:
         - make staticAnalyze
@@ -81,12 +69,6 @@ matrix:
         - make arminstall
         - make armbuild
 
-    - name: Qemu PPC + Fuzz Test    # ~13mn
-      dist: trusty   # it seems ppc cross-compilation fails on "current"
-      script:
-        - make ppcinstall
-        - make ppcfuzz
-
     # check release number (release/new tag only)
     - name: Tag-Specific Test
       if: tag =~ ^v[0-9]\.[0-9]
@@ -103,20 +85,9 @@ matrix:
         - cat /proc/cpuinfo
         - make -C tests fuzztest
 
-    - name: Qemu PPC64 + Fuzz test  # ~13mn, presumed Big-Endian (?)
+    # This test currently fails on GA specifically, for no obvious reason
-      dist: trusty  # note : PPC64 cross-compilation for Qemu tests seems broken on Xenial
+    # (it works fine on travisCI, and on local test platforms).
-      script:
+    - name: Versions Compatibility Test   # ~6mn
-        - make ppcinstall
-        - make ppc64fuzz
-
-    # note : we already have aarch64 tests on hardware
-    - name: Qemu aarch64 + Fuzz Test (on Xenial)    # ~14mn
-      dist: xenial
-      script:
-        - make arminstall
-        - make aarch64fuzz
-    
-    - name: Versions Compatibility Test   # 11.5mn
       script:
         - make -C tests versionsTest
 

Makefile

@@ -416,5 +416,5 @@ bmi32build: clean
 staticAnalyze: SCANBUILD ?= scan-build
 staticAnalyze:
 	$(CC) -v
-	CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) allzstd examples contrib
+	CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) zstd
 endif

tests/test-zstd-versions.py

@@ -23,6 +23,7 @@ from subprocess import Popen, PIPE
 repo_url = 'https://github.com/facebook/zstd.git'
 tmp_dir_name = 'tests/versionsTest'
 make_cmd = 'make'
+make_args = ['-j','CFLAGS=-O1']
 git_cmd = 'git'
 test_dat_src = 'README.md'
 test_dat = 'test_dat'
@@ -56,8 +57,11 @@ def proc(cmd_args, pipe=True, dummy=False):
     return subproc.communicate()
 
 
-def make(args, pipe=True):
+def make(targets, pipe=True):
-    return proc([make_cmd] + args, pipe)
+    cmd = [make_cmd] + make_args + targets
+    cmd_str = str(cmd)
+    print('compilation command : ' + cmd_str)
+    return proc(cmd, pipe)
 
 
 def git(args, pipe=True):
@@ -223,18 +227,26 @@ if __name__ == '__main__':
         dst_zstd = '{}/zstd.{}'.format(tmp_dir, tag)  # /path/to/zstd/tests/versionsTest/zstd.<TAG>
         if not os.path.isfile(dst_zstd) or tag == head:
             if tag != head:
+                print('-----------------------------------------------')
+                print('compiling ' + tag)
+                print('-----------------------------------------------')
                 r_dir = '{}/{}'.format(tmp_dir, tag)  # /path/to/zstd/tests/versionsTest/<TAG>
                 os.makedirs(r_dir, exist_ok=True)
                 os.chdir(clone_dir)
                 git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
                 if tag == 'v0.5.0':
                     os.chdir(r_dir + '/dictBuilder')  # /path/to/zstd/tests/versionsTest/v0.5.0/dictBuilder
-                    make(['clean', 'dictBuilder'], False)
+                    make(['clean'], False)   # separate 'clean' target to allow parallel build
+                    make(['dictBuilder'], False)
                     shutil.copy2('dictBuilder', '{}/dictBuilder.{}'.format(tmp_dir, tag))
                 os.chdir(r_dir + '/programs')  # /path/to/zstd/tests/versionsTest/<TAG>/programs
-                make(['clean', 'zstd'], False)
+                make(['clean'], False)  # separate 'clean' target to allow parallel build
+                make(['zstd'], False)
             else:
                 os.chdir(programs_dir)
+                print('-----------------------------------------------')
+                print('compiling head')
+                print('-----------------------------------------------')
                 make(['zstd'], False)
             shutil.copy2('zstd', dst_zstd)
 
@@ -251,7 +263,9 @@ if __name__ == '__main__':
         print('cp ' + dict_files + ' ' + dict_source_path)
         execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True)
 
+    print('-----------------------------------------------')
     print('Compress test.dat by all released zstd')
+    print('-----------------------------------------------')
 
     error_code = 0
     for tag in tags: