From 3a751edeaedf79f320aae63de4134e0fcf54786e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 15:57:03 +0100 Subject: [PATCH 01/16] uasan --- .travis.yml | 71 ++++++++++++++++++++--------------------------------- 1 file changed, 27 insertions(+), 44 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0ac8efb80..7379fc504 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,20 @@ matrix: os: linux sudo: false + - env: Ubu=12.04cont Cmd="make uasan" + os: linux + sudo: false + + - env: Ubu=14.04 Cmd='make test CC=clang-4.0 MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"' + os: linux + dist: trusty + sudo: required + addons: + apt: + sources: + - llvm-toolchain-trusty-4.0 + packages: + - clang-4.0 - env: Ubu=14.04 Cmd='make -C tests test32 CC=clang-4.0 MOREFLAGS="-g -fsanitize=address"' os: linux @@ -31,9 +45,13 @@ matrix: - clang-4.0 - # Standard Ubuntu 12.04 LTS Server Edition 64 bit - - env: Ubu=12.04 Cmd='cd contrib/pzstd && make googletest && make tsan && make check && make clean && make asan && make check && make clean && cd ../..' + # Ubuntu 14.04 LTS Server Edition 64 bit + - env: Ubu=14.04 Cmd='cd contrib/pzstd && make googletest pzstd tests check && make clean + && make googletest32 all32 check && make clean + && make googletest tsan check && make clean + && make asan check && make clean' os: linux + dist: trusty sudo: required install: - export CXX="g++-6" CC="gcc-6" @@ -43,30 +61,15 @@ matrix: apt: sources: - ubuntu-toolchain-r-test - packages: - - gcc-6 - - g++-6 - - - # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd="make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean - && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean" - os: linux - dist: trusty - sudo: required - install: - - export CXX="g++-4.8" CC="gcc-4.8" - addons: - apt: packages: - libc6-dev-i386 - g++-multilib - - gcc-4.8 - - gcc-4.8-multilib - - g++-4.8 - - g++-4.8-multilib + - gcc-6 + - gcc-6-multilib + - g++-6 + - g++-6-multilib - - env: Ubu=14.04 Cmd="make armtest" + - env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test" dist: trusty sudo: required addons: @@ -76,19 +79,10 @@ matrix: - qemu-user-static - gcc-arm-linux-gnueabi - libc6-dev-armel-cross - - - env: Ubu=14.04 Cmd="make aarch64test" - dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-arm - - qemu-user-static - gcc-aarch64-linux-gnu - libc6-dev-arm64-cross - - env: Ubu=14.04 Cmd='make ppctest' + - env: Ubu=14.04 Cmd='make ppctest && make clean && make ppc64test' dist: trusty sudo: required addons: @@ -98,17 +92,6 @@ matrix: - qemu-user-static - gcc-powerpc-linux-gnu - - env: Ubu=14.04 Cmd='make ppc64test' - dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-ppc - - qemu-user-static - - gcc-powerpc-linux-gnu - - # other feature branches => short tests - env: Ubu=14.04 Cmd='make lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' os: linux @@ -137,7 +120,7 @@ script: # cron & master => long tests, as this is the final step towards a Release # dev => normal tests # other feature branches => short tests (number > 10) - - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then + - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "asan" ]; then FUZZERTEST=-T10mn sh -c "$Cmd" || travis_terminate 1; else if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 10 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then From 684858e7b7924d7789395ec5950d4b29315a4516 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 18:17:24 +0100 Subject: [PATCH 02/16] fix memory leaks --- .travis.yml | 15 +-------------- contrib/pzstd/Makefile | 17 +++++++++++++++++ programs/zstdcli.c | 4 ++-- tests/zstreamtest.c | 2 ++ 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7379fc504..38ed23431 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,14 +8,6 @@ matrix: # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - env: Ubu=12.04cont Cmd="make usan" - os: linux - sudo: false - - - env: Ubu=12.04cont Cmd="make asan" - os: linux - sudo: false - - env: Ubu=12.04cont Cmd="make uasan" os: linux sudo: false @@ -46,17 +38,12 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='cd contrib/pzstd && make googletest pzstd tests check && make clean - && make googletest32 all32 check && make clean - && make googletest tsan check && make clean - && make asan check && make clean' + - env: Ubu=14.04 Cmd='cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && test-pzstd-asan' os: linux dist: trusty sudo: required install: - export CXX="g++-6" CC="gcc-6" - - export LDFLAGS="-fuse-ld=gold" - - export TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' addons: apt: sources: diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index f148bfd8e..10a133dd7 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -85,6 +85,23 @@ endif .PHONY: default default: all +.PHONY: test-pzstd +test-pzstd: TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' +test-pzstd: clean googletest pzstd tests check + +.PHONY: test-pzstd32 +test-pzstd32: clean googletest32 all32 check + +.PHONY: test-pzstd-tsan +test-pzstd-tsan: LDFLAGS="-fuse-ld=gold" +test-pzstd-tsan: TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' +test-pzstd-tsan: clean googletest tsan check + +.PHONY: test-pzstd-asan +test-pzstd-asan: LDFLAGS="-fuse-ld=gold" +test-pzstd-asan: TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' +test-pzstd-asan: clean asan check + .PHONY: check check: $(TESTPROG) ./utils/test/BufferTest$(EXT) $(TESTFLAGS) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 588111913..a7b4fddc8 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -399,7 +399,7 @@ int main(int argCount, const char* argv[]) while (argument[0]!=0) { if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); - return 1; + CLEAN_RETURN(1); } #ifndef ZSTD_NOCOMPRESS /* compression Level */ @@ -555,7 +555,7 @@ int main(int argCount, const char* argv[]) filenameTable[filenameIdx++] = argument; } - if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); return 1; } /* forgotten argument */ + if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); CLEAN_RETURN(1); } /* forgotten argument */ /* Welcome message (if verbose) */ DISPLAYLEVEL(3, WELCOME_MESSAGE); diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 9a9fed98d..c22a284c7 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -496,6 +496,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Bug will cause checksum to fail */ if (ZSTD_isError(r)) goto _output_error; } + + ZSTD_freeDStream(zds); } DISPLAYLEVEL(3, "OK \n"); From d8114e5802edfb236758d7a4d7ce24795e74afa2 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 18:59:56 +0100 Subject: [PATCH 03/16] zstd_compress.c: fix memory leaks --- contrib/pzstd/Makefile | 10 +++++----- lib/compress/zstd_compress.c | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index 10a133dd7..21ef935c6 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -86,20 +86,20 @@ endif default: all .PHONY: test-pzstd -test-pzstd: TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' +test-pzstd: TESTFLAGS=--gtest_filter=-*ExtremelyLarge* test-pzstd: clean googletest pzstd tests check .PHONY: test-pzstd32 test-pzstd32: clean googletest32 all32 check .PHONY: test-pzstd-tsan -test-pzstd-tsan: LDFLAGS="-fuse-ld=gold" -test-pzstd-tsan: TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' +test-pzstd-tsan: LDFLAGS=-fuse-ld=gold +test-pzstd-tsan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge* test-pzstd-tsan: clean googletest tsan check .PHONY: test-pzstd-asan -test-pzstd-asan: LDFLAGS="-fuse-ld=gold" -test-pzstd-asan: TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' +test-pzstd-asan: LDFLAGS=-fuse-ld=gold +test-pzstd-asan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge* test-pzstd-asan: clean asan check .PHONY: check diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 924189b0c..0e0f9d373 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2786,7 +2786,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u if (!cdict || !cctx) { ZSTD_free(cdict, customMem); - ZSTD_free(cctx, customMem); + ZSTD_freeCCtx(cctx); return NULL; } @@ -2804,8 +2804,8 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0); if (ZSTD_isError(errorCode)) { ZSTD_free(cdict->dictBuffer, customMem); - ZSTD_free(cctx, customMem); ZSTD_free(cdict, customMem); + ZSTD_freeCCtx(cctx); return NULL; } } From 346ce32adeb57c468d40f7c4e8ed75c0c84a4f4e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 20:10:21 +0100 Subject: [PATCH 04/16] legacy.c: fix memory leaks --- contrib/pzstd/Makefile | 2 +- tests/legacy.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index 21ef935c6..cec6959e6 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -134,7 +134,7 @@ debug: pzstd$(EXT) tests roundtrip .PHONY: tsan tsan: PZSTD_CCXXFLAGS += -fsanitize=thread -fPIC -tsan: PZSTD_LDFLAGS += -fsanitize=thread -pie +tsan: PZSTD_LDFLAGS += -fsanitize=thread tsan: debug .PHONY: asan diff --git a/tests/legacy.c b/tests/legacy.c index 5d93c68fa..e84e31273 100644 --- a/tests/legacy.c +++ b/tests/legacy.c @@ -65,6 +65,7 @@ int testSimpleAPI(void) return 1; } + free(output); DISPLAY("Simple API OK\n"); return 0; } @@ -118,6 +119,8 @@ int testStreamingAPI(void) } } + free(outBuff); + ZSTD_freeDStream(stream); DISPLAY("Streaming API OK\n"); return 0; } From 97cfec5e12b64bb0494f658143080b1735b8df0e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 20:44:35 +0100 Subject: [PATCH 05/16] travis.yml: reduce number of jobs --- .travis.yml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38ed23431..1020df8e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,38 +7,34 @@ matrix: os: osx - # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - env: Ubu=12.04cont Cmd="make uasan" - os: linux - sudo: false - - - env: Ubu=14.04 Cmd='make test CC=clang-4.0 MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"' + # Ubuntu 14.04 LTS Server Edition 64 bit + - env: Ubu=14.04 Cmd='make test CC=gcc-6 MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"' os: linux dist: trusty sudo: required addons: apt: sources: - - llvm-toolchain-trusty-4.0 + - ubuntu-toolchain-r-test packages: - - clang-4.0 + - gcc-6 + - gcc-6-multilib - - env: Ubu=14.04 Cmd='make -C tests test32 CC=clang-4.0 MOREFLAGS="-g -fsanitize=address"' + - env: Ubu=14.04 Cmd='make -C tests test32 CC=gcc-6 MOREFLAGS="-g -fsanitize=address"' os: linux dist: trusty sudo: required addons: apt: sources: - - llvm-toolchain-trusty-4.0 + - ubuntu-toolchain-r-test packages: - libc6-dev-i386 - gcc-multilib - - clang-4.0 + - gcc-6 + - gcc-6-multilib - - # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && test-pzstd-asan' + - env: Ubu=14.04 Cmd='cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' os: linux dist: trusty sudo: required From 4d7a24328b3312c4531b68cb75d3d7e8e411cb4c Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 21:12:09 +0100 Subject: [PATCH 06/16] travis.yml: added LDFLAGS=-fuse-ld=gold --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1020df8e8..6d5f22bb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='make test CC=gcc-6 MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"' + - env: Ubu=14.04 Cmd='make test CC=gcc-6 MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" LDFLAGS=-fuse-ld=gold' os: linux dist: trusty sudo: required @@ -18,9 +18,8 @@ matrix: - ubuntu-toolchain-r-test packages: - gcc-6 - - gcc-6-multilib - - env: Ubu=14.04 Cmd='make -C tests test32 CC=gcc-6 MOREFLAGS="-g -fsanitize=address"' + - env: Ubu=14.04 Cmd='make -C tests test32 CC=gcc-6 MOREFLAGS="-g -fsanitize=address" LDFLAGS=-fuse-ld=gold' os: linux dist: trusty sudo: required From 7704c3ca1ae07b76c4593418d22ebf89b64b9710 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 21:48:14 +0100 Subject: [PATCH 07/16] travis.yml: use CFLAGS=-Og with -fsanitize --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6d5f22bb1..cf3bbb5bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='make test CC=gcc-6 MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" LDFLAGS=-fuse-ld=gold' + - env: Ubu=14.04 Cmd='LDFLAGS=-fuse-ld=gold CFLAGS=-Og make test CC=gcc-6 MOREFLAGS="-fsanitize=address -fsanitize=undefined"' os: linux dist: trusty sudo: required @@ -19,7 +19,7 @@ matrix: packages: - gcc-6 - - env: Ubu=14.04 Cmd='make -C tests test32 CC=gcc-6 MOREFLAGS="-g -fsanitize=address" LDFLAGS=-fuse-ld=gold' + - env: Ubu=14.04 Cmd='LDFLAGS=-fuse-ld=gold CFLAGS=-Og make -C tests test32 CC=gcc-6 MOREFLAGS="-fsanitize=address -fsanitize=undefined"' os: linux dist: trusty sudo: required From 8a51c692184e524d1f3bb167750c0e378f7c8f35 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 22:48:04 +0100 Subject: [PATCH 08/16] travis.yml: added uasan-test and uasan-test32 --- .travis.yml | 46 ++++++++++++++-------------------------------- Makefile | 3 +++ 2 files changed, 17 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index cf3bbb5bf..dba15d7ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,33 +8,7 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='LDFLAGS=-fuse-ld=gold CFLAGS=-Og make test CC=gcc-6 MOREFLAGS="-fsanitize=address -fsanitize=undefined"' - os: linux - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-6 - - - env: Ubu=14.04 Cmd='LDFLAGS=-fuse-ld=gold CFLAGS=-Og make -C tests test32 CC=gcc-6 MOREFLAGS="-fsanitize=address -fsanitize=undefined"' - os: linux - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - libc6-dev-i386 - - gcc-multilib - - gcc-6 - - gcc-6-multilib - - - env: Ubu=14.04 Cmd='cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' - os: linux + - env: Ubu=14.04 Cmd='make uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' dist: trusty sudo: required install: @@ -47,9 +21,19 @@ matrix: - libc6-dev-i386 - g++-multilib - gcc-6 - - gcc-6-multilib - g++-6 - - g++-6-multilib + + - env: Ubu=14.04 Cmd='CC=gcc-6 make uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy versionsTest' + dist: trusty + sudo: required + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - libc6-dev-i386 + - gcc-multilib + - gcc-6 - env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test" dist: trusty @@ -76,7 +60,6 @@ matrix: # other feature branches => short tests - env: Ubu=14.04 Cmd='make lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' - os: linux dist: trusty sudo: required addons: @@ -84,8 +67,7 @@ matrix: packages: - valgrind - - env: Ubu=14.04 Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make -C tests test32 versionsTest" - os: linux + - env: Ubu=14.04 Cmd="make -C tests test32" dist: trusty sudo: required addons: diff --git a/Makefile b/Makefile index d86db7cb3..128c72bb0 100644 --- a/Makefile +++ b/Makefile @@ -143,6 +143,9 @@ asan32: clean uasan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" +uasan-%: clean + LDFLAGS=-fuse-ld=gold CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $* + endif From f58ac79f513cda1acd468de1853dde6ecfe793aa Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 23:40:21 +0100 Subject: [PATCH 09/16] fix uasan-test32 --- .travis.yml | 7 ++++--- Makefile | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index dba15d7ea..8688035e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='make uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' + - env: Ubu=14.04 Cmd='LDFLAGS=-fuse-ld=gold make uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' dist: trusty sudo: required install: @@ -22,6 +22,7 @@ matrix: - g++-multilib - gcc-6 - g++-6 + - g++-6-multilib - env: Ubu=14.04 Cmd='CC=gcc-6 make uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy versionsTest' dist: trusty @@ -83,11 +84,11 @@ script: # cron & master => long tests, as this is the final step towards a Release # dev => normal tests - # other feature branches => short tests (number > 10) + # other feature branches => short tests (number > 5) - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "asan" ]; then FUZZERTEST=-T10mn sh -c "$Cmd" || travis_terminate 1; else - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 10 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then + if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 5 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then sh -c "$Cmd" || travis_terminate 1; fi fi diff --git a/Makefile b/Makefile index 128c72bb0..ff624e907 100644 --- a/Makefile +++ b/Makefile @@ -144,7 +144,7 @@ uasan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" uasan-%: clean - LDFLAGS=-fuse-ld=gold CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $* + CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $* endif From 5dd18b314b482020be0014f6a0257d20a183b630 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 22 Feb 2017 08:15:17 +0100 Subject: [PATCH 10/16] travis.yml: reduce number of jobs to 7 --- .travis.yml | 3 ++- Makefile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8688035e2..41d90f380 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='LDFLAGS=-fuse-ld=gold make uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' + - env: Ubu=14.04 Cmd='make uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' dist: trusty sudo: required install: @@ -35,6 +35,7 @@ matrix: - libc6-dev-i386 - gcc-multilib - gcc-6 + - gcc-6-multilib - env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test" dist: trusty diff --git a/Makefile b/Makefile index ff624e907..128c72bb0 100644 --- a/Makefile +++ b/Makefile @@ -144,7 +144,7 @@ uasan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" uasan-%: clean - CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $* + LDFLAGS=-fuse-ld=gold CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $* endif From 21911ad6cbc76a4aeab23fa0ecc64fec6c4da2c9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 22 Feb 2017 08:54:56 +0100 Subject: [PATCH 11/16] move Ubuntu packages install to Makefile --- .travis.yml | 64 ++++++----------------------------------------------- Makefile | 25 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 57 deletions(-) diff --git a/.travis.yml b/.travis.yml index 41d90f380..6ea9d31dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,77 +6,27 @@ matrix: - env: Ubu=OS_X_Mavericks Cmd="make gnu90test && make clean && make test && make clean && make travis-install" os: osx - # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='make uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' + - env: Ubu=14.04 Cmd='make gpp6install uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' dist: trusty - sudo: required install: - export CXX="g++-6" CC="gcc-6" - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - libc6-dev-i386 - - g++-multilib - - gcc-6 - - g++-6 - - g++-6-multilib - - env: Ubu=14.04 Cmd='CC=gcc-6 make uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy versionsTest' + - env: Ubu=14.04 Cmd='CC=gcc-6 make gcc6install uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy versionsTest' dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - libc6-dev-i386 - - gcc-multilib - - gcc-6 - - gcc-6-multilib - - env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test" + - env: Ubu=14.04 Cmd="make arminstall armtest && make clean && make aarch64test" dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-arm - - qemu-user-static - - gcc-arm-linux-gnueabi - - libc6-dev-armel-cross - - gcc-aarch64-linux-gnu - - libc6-dev-arm64-cross - - env: Ubu=14.04 Cmd='make ppctest && make clean && make ppc64test' + - env: Ubu=14.04 Cmd='make ppcinstall ppctest && make clean && make ppc64test' dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-ppc - - qemu-user-static - - gcc-powerpc-linux-gnu # other feature branches => short tests - - env: Ubu=14.04 Cmd='make lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' + - env: Ubu=14.04 Cmd='make valgrindinstall lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' dist: trusty - sudo: required - addons: - apt: - packages: - - valgrind - - env: Ubu=14.04 Cmd="make -C tests test32" + - env: Ubu=14.04 Cmd="make libc6install && make -C tests test32" dist: trusty - sudo: required - addons: - apt: - packages: - - libc6-dev-i386 - - gcc-multilib @@ -87,7 +37,7 @@ script: # dev => normal tests # other feature branches => short tests (number > 5) - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "asan" ]; then - FUZZERTEST=-T10mn sh -c "$Cmd" || travis_terminate 1; + FUZZERTEST=-T5mn sh -c "$Cmd" || travis_terminate 1; else if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 5 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then sh -c "$Cmd" || travis_terminate 1; diff --git a/Makefile b/Makefile index 128c72bb0..709d2f0ea 100644 --- a/Makefile +++ b/Makefile @@ -146,6 +146,31 @@ uasan: clean uasan-%: clean LDFLAGS=-fuse-ld=gold CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $* +apt-install: + sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES) + +apt-add-repo: + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update -y -qq + +ppcinstall: + APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install + +arminstall: + APT_PACKAGES="qemu-system-arm qemu-user-static gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install + +valgrindinstall: + APT_PACKAGES="valgrind" $(MAKE) apt-install + +libc6install: + APT_PACKAGES="libc6-dev-i386 gcc-multilib" $(MAKE) apt-install + +gcc6install: apt-add-repo + APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install + +gpp6install: apt-add-repo + APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install + endif From 2e8ae51f8cf32ce22ec3caebd6c153c6248dfab9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 22 Feb 2017 09:21:04 +0100 Subject: [PATCH 12/16] travis.yml: set "dist: trusty" as default --- .travis.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6ea9d31dc..e6b011003 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,6 @@ language: c +sudo: required +dist: trusty matrix: fast_finish: true include: @@ -8,27 +10,15 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - env: Ubu=14.04 Cmd='make gpp6install uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' - dist: trusty install: - export CXX="g++-6" CC="gcc-6" - - env: Ubu=14.04 Cmd='CC=gcc-6 make gcc6install uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy versionsTest' - dist: trusty - - env: Ubu=14.04 Cmd="make arminstall armtest && make clean && make aarch64test" - dist: trusty - - env: Ubu=14.04 Cmd='make ppcinstall ppctest && make clean && make ppc64test' - dist: trusty # other feature branches => short tests - env: Ubu=14.04 Cmd='make valgrindinstall lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' - dist: trusty - - env: Ubu=14.04 Cmd="make libc6install && make -C tests test32" - dist: trusty - - script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') From 3d836bfd18e67f406024e874e97aedddb3ea0355 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 22 Feb 2017 09:36:42 +0100 Subject: [PATCH 13/16] travis.yml: fix versionsTest target --- .travis.yml | 7 +++---- tests/Makefile | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index e6b011003..958633de7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,12 +22,11 @@ matrix: script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') - # cron & master => long tests, as this is the final step towards a Release - # dev => normal tests + # dev && pull requests => normal tests # other feature branches => short tests (number > 5) - - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "asan" ]; then - FUZZERTEST=-T5mn sh -c "$Cmd" || travis_terminate 1; + - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then + FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1; else if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 5 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then sh -c "$Cmd" || travis_terminate 1; diff --git a/tests/Makefile b/tests/Makefile index c5b8bdfa7..937ec96d3 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -170,7 +170,7 @@ namespaceTest: if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi $(RM) $@ -versionsTest: +versionsTest: clean $(PYTHON) test-zstd-versions.py clean: From 337ec875b61f56cc98bf297887da80029741b8d4 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 22 Feb 2017 10:31:30 +0100 Subject: [PATCH 14/16] minor tweaks --- .travis.yml | 2 +- programs/Makefile | 2 +- tests/Makefile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 958633de7..c1985d785 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ matrix: - env: Ubu=14.04 Cmd='make gpp6install uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' install: - export CXX="g++-6" CC="gcc-6" - - env: Ubu=14.04 Cmd='CC=gcc-6 make gcc6install uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy versionsTest' + - env: Ubu=14.04 Cmd='CC=gcc-6 make gcc6install uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy && make -C tests versionsTest' - env: Ubu=14.04 Cmd="make arminstall armtest && make clean && make aarch64test" - env: Ubu=14.04 Cmd='make ppcinstall ppctest && make clean && make ppc64test' diff --git a/programs/Makefile b/programs/Makefile index 0a9ab5a79..db718d14c 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -148,7 +148,7 @@ generate_res: windres/generate_res.bat clean: - $(MAKE) -C ../lib clean + $(MAKE) -C $(ZSTDDIR) clean @$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda @$(RM) core *.o tmp* result* *.gcda dictionary *.zst \ zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ diff --git a/tests/Makefile b/tests/Makefile index 937ec96d3..17286a022 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -174,7 +174,7 @@ versionsTest: clean $(PYTHON) test-zstd-versions.py clean: - $(MAKE) -C ../lib clean + $(MAKE) -C $(ZSTDDIR) clean @$(RM) -fR $(TESTARTEFACT) @$(RM) -f core *.o tmp* result* *.gcda dictionary *.zst \ $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \ From d41c048394fb4123d567560013e72424bea5a920 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 22 Feb 2017 11:07:28 +0100 Subject: [PATCH 15/16] added arm-ppc-compilation Makefile target --- .travis.yml | 2 +- Makefile | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c1985d785..a4d4a50ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ matrix: - env: Ubu=14.04 Cmd='make ppcinstall ppctest && make clean && make ppc64test' # other feature branches => short tests - - env: Ubu=14.04 Cmd='make valgrindinstall lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' + - env: Ubu=14.04 Cmd='make arminstall ppcinstall arm-ppc-compilation && make valgrindinstall lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' - env: Ubu=14.04 Cmd="make libc6install && make -C tests test32" script: diff --git a/Makefile b/Makefile index 709d2f0ea..ed8f16107 100644 --- a/Makefile +++ b/Makefile @@ -128,6 +128,12 @@ ppc64test: clean $(MAKE) -C $(TESTDIR) datagen # use native, faster $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" +arm-ppc-compilation: + $(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" + $(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" + $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" + $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" + usan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined" From bbbd43509950701d076cfdc3f0ef325a80f5d55a Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 22 Feb 2017 11:21:34 +0100 Subject: [PATCH 16/16] travis.yml: test arm-ppc-compilation target --- .travis.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index a4d4a50ab..b20c43329 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,20 +5,20 @@ matrix: fast_finish: true include: # OS X Mavericks - - env: Ubu=OS_X_Mavericks Cmd="make gnu90test && make clean && make test && make clean && make travis-install" + - env: Cmd="make gnu90test && make clean && make test && make clean && make travis-install" os: osx # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='make gpp6install uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' + - env: Cmd='make gpp6install uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' install: - export CXX="g++-6" CC="gcc-6" - - env: Ubu=14.04 Cmd='CC=gcc-6 make gcc6install uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy && make -C tests versionsTest' - - env: Ubu=14.04 Cmd="make arminstall armtest && make clean && make aarch64test" - - env: Ubu=14.04 Cmd='make ppcinstall ppctest && make clean && make ppc64test' + - env: Cmd='CC=gcc-6 make gcc6install uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy && make -C tests versionsTest' + - env: Cmd="make arminstall armtest && make clean && make aarch64test" + - env: Cmd='make ppcinstall ppctest && make clean && make ppc64test' # other feature branches => short tests - - env: Ubu=14.04 Cmd='make arminstall ppcinstall arm-ppc-compilation && make valgrindinstall lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' - - env: Ubu=14.04 Cmd="make libc6install && make -C tests test32" + - env: Cmd='make valgrindinstall arminstall ppcinstall arm-ppc-compilation && make clean lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' + - env: Cmd="make libc6install && make -C tests test32" script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')