From 29157320fb5dca130e13572066273acf75d5f5f0 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 24 Jan 2017 13:18:50 +0100 Subject: [PATCH 01/10] improved ZSTD_compressBlock_opt_extDict_generic --- lib/compress/zstd_opt.h | 4 ++-- zlibWrapper/.gitignore | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index 8393e7b4c..8862bbd6b 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -825,7 +825,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch); - if (match_num > 0 && matches[match_num-1].len > sufficient_len) { + if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) { best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; last_pos = cur + 1; @@ -835,7 +835,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, /* set prices using matches at position = cur */ for (u = 0; u < match_num; u++) { mlen = (u>0) ? matches[u-1].len+1 : best_mlen; - best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur; + best_mlen = matches[u].len; while (mlen <= best_mlen) { if (opt[cur].mlen == 1) { diff --git a/zlibWrapper/.gitignore b/zlibWrapper/.gitignore index 23d2f3a66..6167ca4da 100644 --- a/zlibWrapper/.gitignore +++ b/zlibWrapper/.gitignore @@ -22,4 +22,4 @@ zwrapbench *.txt # Directories -minizip/ \ No newline at end of file +minizip/ From a1cc1796690482e6d982a8faa99f1ebb6960e88d Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 24 Jan 2017 15:01:46 +0100 Subject: [PATCH 02/10] JOB_NUMBER -eq 9 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b0489bd63..e8b05a4e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -168,4 +168,4 @@ matrix: script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') # - if [ $JOB_NUMBER -eq 9 ] || [ $JOB_NUMBER -eq 10 ]; then sh -c "$Cmd"; fi - - sh -c "$Cmd" + - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ "$TRAVIS_BRANCH" = "master" ] || [ "$TRAVIS_BRANCH" = "dev" ] || [ $JOB_NUMBER -eq 9 ]; then sh -c "$Cmd"; fi From 89f74fc0a08f8d88e5fb2281339454d3f9fcdd35 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 24 Jan 2017 17:42:28 +0100 Subject: [PATCH 03/10] .travis.yml: test jobs 12-15 --- .travis.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index e8b05a4e4..7120284ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,10 +8,6 @@ matrix: # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install" - os: linux - sudo: false - - env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-symbols && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean" os: linux sudo: false @@ -165,7 +161,12 @@ matrix: - gcc-6 - gcc-6-multilib + # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) + - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install" + os: linux + sudo: false + script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') # - if [ $JOB_NUMBER -eq 9 ] || [ $JOB_NUMBER -eq 10 ]; then sh -c "$Cmd"; fi - - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ "$TRAVIS_BRANCH" = "master" ] || [ "$TRAVIS_BRANCH" = "dev" ] || [ $JOB_NUMBER -eq 9 ]; then sh -c "$Cmd"; fi + - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ "$TRAVIS_BRANCH" = "master" ] || [ "$TRAVIS_BRANCH" = "dev" ] || [ $JOB_NUMBER -gt 11 ]; then sh -c "$Cmd"; fi From e1ccaa79574a7cb3e71cebecab0ae5fb2aa697eb Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 25 Jan 2017 11:19:35 +0100 Subject: [PATCH 04/10] .travis.yml: optimized order of short tests --- .travis.yml | 39 +++++++++++++++++++++------------------ tests/Makefile | 4 ++-- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7120284ea..ef39ae7f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,7 +76,6 @@ matrix: - gcc-arm-linux-gnueabi - libc6-dev-armel-cross - # Ubuntu 14.04 LTS Server Edition 64 bit - env: Ubu=14.04 Cmd="make aarch64test" dist: trusty sudo: required @@ -117,6 +116,23 @@ matrix: packages: - valgrind + + + # other feature branches => short tests + - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install" + os: linux + sudo: false + + - env: Ubu=14.04 Cmd="make -C tests test32" + os: linux + dist: trusty + sudo: required + addons: + apt: + packages: + - libc6-dev-i386 + - gcc-multilib + - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd googletest32 @@ -136,16 +152,6 @@ matrix: - g++-4.8 - g++-4.8-multilib - - env: Ubu=14.04 Cmd="make -C tests test32" - os: linux - dist: trusty - sudo: required - addons: - apt: - packages: - - libc6-dev-i386 - - gcc-multilib - - env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test && make clean && make -C tests dll" os: linux dist: trusty @@ -161,12 +167,9 @@ matrix: - gcc-6 - gcc-6-multilib - # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install" - os: linux - sudo: false - script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') - # - if [ $JOB_NUMBER -eq 9 ] || [ $JOB_NUMBER -eq 10 ]; then sh -c "$Cmd"; fi - - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ "$TRAVIS_BRANCH" = "master" ] || [ "$TRAVIS_BRANCH" = "dev" ] || [ $JOB_NUMBER -gt 11 ]; then sh -c "$Cmd"; fi + # dev => normal tests; other feature branches => short tests (number > 11) + - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ "$TRAVIS_BRANCH" = "dev" ] || [ $JOB_NUMBER -gt 11 ]; then sh -c "$Cmd"; fi + # master => long tests, as this is the final step towards a Release + - if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T10mn sh -c "$Cmd"; fi diff --git a/tests/Makefile b/tests/Makefile index 937f3b41e..f49d23018 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -54,8 +54,8 @@ endif VOID = /dev/null ZSTREAM_TESTTIME = -T2mn -FUZZERTEST= -T5mn -ZSTDRTTEST= --test-large-data +FUZZERTEST ?= -T5mn +ZSTDRTTEST = --test-large-data .PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest From 4b66ddea7e70cee2d71c09b151b13cc973bd2db6 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 25 Jan 2017 11:57:28 +0100 Subject: [PATCH 05/10] .travis.yml: different tests for "master" branch --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ef39ae7f1..bbd775001 100644 --- a/.travis.yml +++ b/.travis.yml @@ -170,6 +170,6 @@ matrix: script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') # dev => normal tests; other feature branches => short tests (number > 11) - - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ "$TRAVIS_BRANCH" = "dev" ] || [ $JOB_NUMBER -gt 11 ]; then sh -c "$Cmd"; fi + - if [ "$TRAVIS_BRANCH" != "master" ] && [ "$TRAVIS_BRANCH" = "dev" ] || [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 11 ]; then sh -c "$Cmd"; fi # master => long tests, as this is the final step towards a Release - - if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T10mn sh -c "$Cmd"; fi + - if [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T10mn sh -c "$Cmd"; fi From 92a4dbf2e456525d7f834cf97c8f16702ec0592e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 25 Jan 2017 13:02:33 +0100 Subject: [PATCH 06/10] Fixed https://github.com/facebook/zstd/issues/232 --- .travis.yml | 2 +- programs/fileio.c | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bbd775001..ba9f9965d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -170,6 +170,6 @@ matrix: script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') # dev => normal tests; other feature branches => short tests (number > 11) - - if [ "$TRAVIS_BRANCH" != "master" ] && [ "$TRAVIS_BRANCH" = "dev" ] || [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 11 ]; then sh -c "$Cmd"; fi + - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 11 ] || [ "$TRAVIS_BRANCH" = "dev" ] && [ "$TRAVIS_BRANCH" != "master" ]; then sh -c "$Cmd"; fi # master => long tests, as this is the final step towards a Release - if [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T10mn sh -c "$Cmd"; fi diff --git a/programs/fileio.c b/programs/fileio.c index ac7dffb31..596c922f4 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -414,6 +414,11 @@ static int FIO_compressFilename_srcFile(cRess_t ress, DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); return 1; } + if (!UTIL_doesFileExists(srcFileName)) { + DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName); + return 1; + } + ress.srcFile = FIO_openSrcFile(srcFileName); if (!ress.srcFile) return 1; /* srcFile could not be opened */ @@ -772,6 +777,10 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); return 1; } + if (!UTIL_doesFileExists(srcFileName)) { + DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName); + return 1; + } srcFile = FIO_openSrcFile(srcFileName); if (srcFile==0) return 1; From eb2d23a90c888476dfed26b73b640a8f3e2cf075 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 25 Jan 2017 13:11:26 +0100 Subject: [PATCH 07/10] improved #232 fix --- programs/fileio.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 596c922f4..86da00131 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -158,6 +158,10 @@ static FILE* FIO_openSrcFile(const char* srcFileName) f = stdin; SET_BINARY_MODE(stdin); } else { + if (!UTIL_doesFileExists(srcFileName)) { + DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName); + return NULL; + } f = fopen(srcFileName, "rb"); if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); } @@ -414,10 +418,6 @@ static int FIO_compressFilename_srcFile(cRess_t ress, DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); return 1; } - if (!UTIL_doesFileExists(srcFileName)) { - DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName); - return 1; - } ress.srcFile = FIO_openSrcFile(srcFileName); if (!ress.srcFile) return 1; /* srcFile could not be opened */ @@ -777,10 +777,6 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); return 1; } - if (!UTIL_doesFileExists(srcFileName)) { - DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName); - return 1; - } srcFile = FIO_openSrcFile(srcFileName); if (srcFile==0) return 1; From 9c018cc1407d2e18da377744b3b6f26dc6effc8f Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 26 Jan 2017 15:56:34 -0800 Subject: [PATCH 08/10] Add BUCK files for Nuclide support --- .buckconfig | 9 ++ .buckversion | 1 + .gitignore | 2 + contrib/pzstd/BUCK | 72 +++++++++++++ contrib/pzstd/test/BUCK | 37 +++++++ contrib/pzstd/utils/BUCK | 75 ++++++++++++++ contrib/pzstd/utils/test/BUCK | 35 +++++++ lib/BUCK | 186 ++++++++++++++++++++++++++++++++++ lib/common/pool.c | 2 +- programs/BUCK | 63 ++++++++++++ programs/bench.c | 3 +- zlibWrapper/BUCK | 22 ++++ 12 files changed, 504 insertions(+), 3 deletions(-) create mode 100644 .buckconfig create mode 100644 .buckversion create mode 100644 contrib/pzstd/BUCK create mode 100644 contrib/pzstd/test/BUCK create mode 100644 contrib/pzstd/utils/BUCK create mode 100644 contrib/pzstd/utils/test/BUCK create mode 100644 lib/BUCK create mode 100644 programs/BUCK create mode 100644 zlibWrapper/BUCK diff --git a/.buckconfig b/.buckconfig new file mode 100644 index 000000000..b2b9c036f --- /dev/null +++ b/.buckconfig @@ -0,0 +1,9 @@ +[cxx] + cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1 + cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith + cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1 + cxxflags = -std=c++11 -Wno-format-security -Wno-deprecated-declarations + gtest_dep = //contrib/pzstd:gtest + +[httpserver] + port = 0 diff --git a/.buckversion b/.buckversion new file mode 100644 index 000000000..892fad966 --- /dev/null +++ b/.buckversion @@ -0,0 +1 @@ +c8dec2e8da52d483f6dd7c6cd2ad694e8e6fed2b diff --git a/.gitignore b/.gitignore index dd7a74519..e02119883 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ googletest/ # Directories bin/ +.buckd/ +buck-out/ diff --git a/contrib/pzstd/BUCK b/contrib/pzstd/BUCK new file mode 100644 index 000000000..d04eeedd8 --- /dev/null +++ b/contrib/pzstd/BUCK @@ -0,0 +1,72 @@ +cxx_library( + name='libpzstd', + visibility=['PUBLIC'], + header_namespace='', + exported_headers=[ + 'ErrorHolder.h', + 'Logging.h', + 'Pzstd.h', + ], + headers=[ + 'SkippableFrame.h', + ], + srcs=[ + 'Pzstd.cpp', + 'SkippableFrame.cpp', + ], + deps=[ + ':options', + '//contrib/pzstd/utils:utils', + '//lib:mem', + '//lib:zstd', + ], +) + +cxx_library( + name='options', + visibility=['PUBLIC'], + header_namespace='', + exported_headers=['Options.h'], + srcs=['Options.cpp'], + deps=[ + '//contrib/pzstd/utils:scope_guard', + '//lib:zstd', + '//programs:util', + ], +) + +cxx_binary( + name='pzstd', + visibility=['PUBLIC'], + srcs=['main.cpp'], + deps=[ + ':libpzstd', + ':options', + ], +) + +# Must run "make googletest" first +cxx_library( + name='gtest', + srcs=glob([ + 'googletest/googletest/src/gtest-all.cc', + 'googletest/googlemock/src/gmock-all.cc', + 'googletest/googlemock/src/gmock_main.cc', + ]), + header_namespace='', + exported_headers=subdir_glob([ + ('googletest/googletest/include', '**/*.h'), + ('googletest/googlemock/include', '**/*.h'), + ]), + headers=subdir_glob([ + ('googletest/googletest', 'src/*.cc'), + ('googletest/googletest', 'src/*.h'), + ('googletest/googlemock', 'src/*.cc'), + ('googletest/googlemock', 'src/*.h'), + ]), + platform_linker_flags=[ + ('android', []), + ('', ['-lpthread']), + ], + visibility=['PUBLIC'], +) diff --git a/contrib/pzstd/test/BUCK b/contrib/pzstd/test/BUCK new file mode 100644 index 000000000..6d3fdd3c2 --- /dev/null +++ b/contrib/pzstd/test/BUCK @@ -0,0 +1,37 @@ +cxx_test( + name='options_test', + srcs=['OptionsTest.cpp'], + deps=['//contrib/pzstd:options'], +) + +cxx_test( + name='pzstd_test', + srcs=['PzstdTest.cpp'], + deps=[ + ':round_trip', + '//contrib/pzstd:libpzstd', + '//contrib/pzstd/utils:scope_guard', + '//programs:datagen', + ], +) + +cxx_binary( + name='round_trip_test', + srcs=['RoundTripTest.cpp'], + deps=[ + ':round_trip', + '//contrib/pzstd/utils:scope_guard', + '//programs:datagen', + ] +) + +cxx_library( + name='round_trip', + header_namespace='test', + exported_headers=['RoundTrip.h'], + deps=[ + '//contrib/pzstd:libpzstd', + '//contrib/pzstd:options', + '//contrib/pzstd/utils:scope_guard', + ] +) diff --git a/contrib/pzstd/utils/BUCK b/contrib/pzstd/utils/BUCK new file mode 100644 index 000000000..e757f4120 --- /dev/null +++ b/contrib/pzstd/utils/BUCK @@ -0,0 +1,75 @@ +cxx_library( + name='buffer', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['Buffer.h'], + deps=[':range'], +) + +cxx_library( + name='file_system', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['FileSystem.h'], + deps=[':range'], +) + +cxx_library( + name='likely', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['Likely.h'], +) + +cxx_library( + name='range', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['Range.h'], + deps=[':likely'], +) + +cxx_library( + name='resource_pool', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['ResourcePool.h'], +) + +cxx_library( + name='scope_guard', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['ScopeGuard.h'], +) + +cxx_library( + name='thread_pool', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['ThreadPool.h'], + deps=[':work_queue'], +) + +cxx_library( + name='work_queue', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['WorkQueue.h'], + deps=[':buffer'], +) + +cxx_library( + name='utils', + visibility=['PUBLIC'], + deps=[ + ':buffer', + ':file_system', + ':likely', + ':range', + ':resource_pool', + ':scope_guard', + ':thread_pool', + ':work_queue', + ], +) diff --git a/contrib/pzstd/utils/test/BUCK b/contrib/pzstd/utils/test/BUCK new file mode 100644 index 000000000..a5113cab6 --- /dev/null +++ b/contrib/pzstd/utils/test/BUCK @@ -0,0 +1,35 @@ +cxx_test( + name='buffer_test', + srcs=['BufferTest.cpp'], + deps=['//contrib/pzstd/utils:buffer'], +) + +cxx_test( + name='range_test', + srcs=['RangeTest.cpp'], + deps=['//contrib/pzstd/utils:range'], +) + +cxx_test( + name='resource_pool_test', + srcs=['ResourcePoolTest.cpp'], + deps=['//contrib/pzstd/utils:resource_pool'], +) + +cxx_test( + name='scope_guard_test', + srcs=['ScopeGuardTest.cpp'], + deps=['//contrib/pzstd/utils:scope_guard'], +) + +cxx_test( + name='thread_pool_test', + srcs=['ThreadPoolTest.cpp'], + deps=['//contrib/pzstd/utils:thread_pool'], +) + +cxx_test( + name='work_queue_test', + srcs=['RangeTest.cpp'], + deps=['//contrib/pzstd/utils:work_queue'], +) diff --git a/lib/BUCK b/lib/BUCK new file mode 100644 index 000000000..6812c1b1e --- /dev/null +++ b/lib/BUCK @@ -0,0 +1,186 @@ +cxx_library( + name='zstd', + header_namespace='', + visibility=['PUBLIC'], + deps=[ + ':common', + ':compress', + ':decompress', + ':deprecated', + ], +) + +cxx_library( + name='compress', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('compress', 'zstdmt_compress.h'), + ]), + headers=subdir_glob([ + ('compress', 'zstd_opt.h'), + ]), + srcs=[ + 'compress/zstd_compress.c', + 'compress/zstdmt_compress.c', + ], + deps=[':common'], +) + +cxx_library( + name='decompress', + header_namespace='', + visibility=['PUBLIC'], + srcs=['decompress/zstd_decompress.c'], + deps=[ + ':common', + ':legacy', + ], +) + +cxx_library( + name='deprecated', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('decprecated', '*.h'), + ]), + srcs=glob(['deprecated/*.c']), + deps=[':common'], +) + +cxx_library( + name='legacy', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('legacy', '*.h'), + ]), + srcs=glob(['legacy/*.c']), + deps=[':common'], +) + +cxx_library( + name='zdict', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('dictBuilder', 'zdict.h'), + ]), + headers=subdir_glob([ + ('dictBuilder', 'divsufsort.h'), + ]), + srcs=glob(['dictBuilder/*.c']), + deps=[':common'], +) + +cxx_library( + name='bitstream', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'bitstream.h'), + ]), +) + +cxx_library( + name='entropy', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'fse.h'), + ('common', 'huf.h'), + ]), + srcs=[ + 'common/entropy_common.c', + 'common/fse_decompress.c', + 'compress/fse_compress.c', + 'compress/huf_compress.c', + 'decompress/huf_decompress.c', + ], + deps=[ + ':bitstream', + ':errors', + ':mem', + ], +) + +cxx_library( + name='errors', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'error_private.h'), + ('common', 'zstd_errors.h'), + ]), + srcs=['common/error_private.c'], +) + +cxx_library( + name='mem', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'mem.h'), + ]), +) + +cxx_library( + name='pool', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'pool.h'), + ]), + srcs=['common/pool.c'], + deps=[':threading'], +) + +cxx_library( + name='threading', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'threading.h'), + ]), + srcs=['common/threading.c'], +) + +cxx_library( + name='xxhash', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'xxhash.h'), + ]), + srcs=['common/xxhash.c'], +) + +cxx_library( + name='zstd_common', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('', 'zstd.h'), + ('common', 'zstd_internal.h'), + ]), + srcs=['common/zstd_common.c'], + deps=[ + ':errors', + ':mem', + ], +) + +cxx_library( + name='common', + deps=[ + ':bitstream', + ':entropy', + ':errors', + ':mem', + ':pool', + ':threading', + ':xxhash', + ':zstd_common', + ] +) diff --git a/lib/common/pool.c b/lib/common/pool.c index 693217f24..e439fe1b0 100644 --- a/lib/common/pool.c +++ b/lib/common/pool.c @@ -21,7 +21,7 @@ #ifdef ZSTD_MULTITHREAD -#include /* pthread adaptation */ +#include "threading.h" /* pthread adaptation */ /* A job is a function and an opaque argument */ typedef struct POOL_job_s { diff --git a/programs/BUCK b/programs/BUCK new file mode 100644 index 000000000..069403042 --- /dev/null +++ b/programs/BUCK @@ -0,0 +1,63 @@ +cxx_binary( + name='zstd', + headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']), + srcs=glob(['*.c'], excludes=['datagen.c']), + deps=[ + ':datagen', + ':util', + '//lib:zstd', + '//lib:zdict', + '//lib:mem', + '//lib:xxhash', + ], +) + +cxx_binary( + name='zstdmt', + headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']), + srcs=glob(['*.c'], excludes=['datagen.c']), + deps=[ + ':datagen', + ':util', + '//lib:zstd', + '//lib:zdict', + '//lib:mem', + '//lib:xxhash', + ], + preprocessor_flags=['-DZSTD_MULTITHREAD'], + linker_flags=['-lpthread'], +) + +cxx_binary( + name='gzstd', + headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']), + srcs=glob(['*.c'], excludes=['datagen.c']), + deps=[ + ':datagen', + ':util', + '//lib:zstd', + '//lib:zdict', + '//lib:mem', + '//lib:xxhash', + ], + preprocessor_flags=['-DZSTD_GZDECOMPRESS'], + linker_flags=['-lz'], +) + +cxx_library( + name='datagen', + visibility=['PUBLIC'], + header_namespace='', + exported_headers=['datagen.h'], + srcs=['datagen.c'], + deps=['//lib:mem'], +) + + +cxx_library( + name='util', + visibility=['PUBLIC'], + header_namespace='', + exported_headers=['util.h', 'platform.h'], + deps=['//lib:mem'], +) diff --git a/programs/bench.c b/programs/bench.c index 1ca40d6b9..dcb23b1f2 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -40,6 +40,7 @@ #include "zstd.h" #include "datagen.h" /* RDG_genBuffer */ #include "xxhash.h" +#include "zstdmt_compress.h" /* ************************************* @@ -148,8 +149,6 @@ typedef struct { #define MIN(a,b) ((a)<(b) ? (a) : (b)) #define MAX(a,b) ((a)>(b) ? (a) : (b)) -#include "compress/zstdmt_compress.h" - static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const char* displayName, int cLevel, const size_t* fileSizes, U32 nbFiles, diff --git a/zlibWrapper/BUCK b/zlibWrapper/BUCK new file mode 100644 index 000000000..a3b74ac3f --- /dev/null +++ b/zlibWrapper/BUCK @@ -0,0 +1,22 @@ +cxx_library( + name='zlib_wrapper', + visibility=['PUBLIC'], + exported_linker_flags=['-lz'], + header_namespace='', + exported_headers=['zstd_zlibwrapper.h'], + headers=[ + 'gzcompatibility.h', + 'gzguts.h', + ], + srcs=glob(['*.c']), + deps=[ + '//lib:zstd', + '//lib:zstd_common', + ] +) + +cxx_binary( + name='minigzip', + srcs=['examples/minigzip.c'], + deps=[':zlib_wrapper'], +) From 58f499c41e49875359e555a8bcba31709a88a523 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 26 Jan 2017 20:47:59 -0800 Subject: [PATCH 09/10] Clean imports and shorten tests --- contrib/pzstd/Logging.h | 4 +- contrib/pzstd/Options.cpp | 2 +- contrib/pzstd/main.cpp | 5 --- contrib/pzstd/test/PzstdTest.cpp | 64 +++++++++++++++----------------- 4 files changed, 32 insertions(+), 43 deletions(-) diff --git a/contrib/pzstd/Logging.h b/contrib/pzstd/Logging.h index 76c982ab2..6d5582a00 100644 --- a/contrib/pzstd/Logging.h +++ b/contrib/pzstd/Logging.h @@ -37,8 +37,8 @@ class Logger { return level <= level_; } - template - void operator()(int level, const char *fmt, Args... args) { + template + void operator()(int level, String fmt, Args... args) { if (level > level_) { return; } diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp index 0b1403354..a0d969393 100644 --- a/contrib/pzstd/Options.cpp +++ b/contrib/pzstd/Options.cpp @@ -7,6 +7,7 @@ * of patent rights can be found in the PATENTS file in the same directory. */ #include "Options.h" +#include "util.h" #include "utils/ScopeGuard.h" #include @@ -15,7 +16,6 @@ #include #include #include -#include #include #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || \ diff --git a/contrib/pzstd/main.cpp b/contrib/pzstd/main.cpp index 279cbfb5e..7d8dbfbcf 100644 --- a/contrib/pzstd/main.cpp +++ b/contrib/pzstd/main.cpp @@ -9,11 +9,6 @@ #include "ErrorHolder.h" #include "Options.h" #include "Pzstd.h" -#include "utils/FileSystem.h" -#include "utils/Range.h" -#include "utils/ScopeGuard.h" -#include "utils/ThreadPool.h" -#include "utils/WorkQueue.h" using namespace pzstd; diff --git a/contrib/pzstd/test/PzstdTest.cpp b/contrib/pzstd/test/PzstdTest.cpp index c85f73a39..cadfa83f7 100644 --- a/contrib/pzstd/test/PzstdTest.cpp +++ b/contrib/pzstd/test/PzstdTest.cpp @@ -41,23 +41,20 @@ TEST(Pzstd, SmallSizes) { std::fclose(fd); ASSERT_EQ(written, len); } - for (unsigned headers = 0; headers <= 1; ++headers) { - for (unsigned numThreads = 1; numThreads <= 2; ++numThreads) { - for (unsigned level = 1; level <= 4; level *= 4) { - auto errorGuard = makeScopeGuard([&] { - std::fprintf(stderr, "pzstd headers: %u\n", headers); - std::fprintf(stderr, "# threads: %u\n", numThreads); - std::fprintf(stderr, "compression level: %u\n", level); - }); - Options options; - options.overwrite = true; - options.inputFiles = {inputFile}; - options.numThreads = numThreads; - options.compressionLevel = level; - options.verbosity = 1; - ASSERT_TRUE(roundTrip(options)); - errorGuard.dismiss(); - } + for (unsigned numThreads = 1; numThreads <= 2; ++numThreads) { + for (unsigned level = 1; level <= 4; level *= 4) { + auto errorGuard = makeScopeGuard([&] { + std::fprintf(stderr, "# threads: %u\n", numThreads); + std::fprintf(stderr, "compression level: %u\n", level); + }); + Options options; + options.overwrite = true; + options.inputFiles = {inputFile}; + options.numThreads = numThreads; + options.compressionLevel = level; + options.verbosity = 1; + ASSERT_TRUE(roundTrip(options)); + errorGuard.dismiss(); } } } @@ -79,29 +76,26 @@ TEST(Pzstd, LargeSizes) { std::fclose(fd); ASSERT_EQ(written, len); } - for (unsigned headers = 0; headers <= 1; ++headers) { - for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) { - for (unsigned level = 1; level <= 4; level *= 2) { - auto errorGuard = makeScopeGuard([&] { - std::fprintf(stderr, "pzstd headers: %u\n", headers); - std::fprintf(stderr, "# threads: %u\n", numThreads); - std::fprintf(stderr, "compression level: %u\n", level); - }); - Options options; - options.overwrite = true; - options.inputFiles = {inputFile}; - options.numThreads = std::min(numThreads, options.numThreads); - options.compressionLevel = level; - options.verbosity = 1; - ASSERT_TRUE(roundTrip(options)); - errorGuard.dismiss(); - } + for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) { + for (unsigned level = 1; level <= 4; level *= 4) { + auto errorGuard = makeScopeGuard([&] { + std::fprintf(stderr, "# threads: %u\n", numThreads); + std::fprintf(stderr, "compression level: %u\n", level); + }); + Options options; + options.overwrite = true; + options.inputFiles = {inputFile}; + options.numThreads = std::min(numThreads, options.numThreads); + options.compressionLevel = level; + options.verbosity = 1; + ASSERT_TRUE(roundTrip(options)); + errorGuard.dismiss(); } } } } -TEST(Pzstd, ExtremelyLargeSize) { +TEST(Pzstd, DISABLED_ExtremelyLargeSize) { unsigned seed = std::random_device{}(); std::fprintf(stderr, "Pzstd.ExtremelyLargeSize seed: %u\n", seed); std::mt19937 gen(seed); From 5cf84a05e734dbd7872865bcbc67d70c596aa541 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 27 Jan 2017 13:26:44 -0800 Subject: [PATCH 10/10] Revert unnecessary change to Logging.h --- contrib/pzstd/Logging.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/pzstd/Logging.h b/contrib/pzstd/Logging.h index 6d5582a00..76c982ab2 100644 --- a/contrib/pzstd/Logging.h +++ b/contrib/pzstd/Logging.h @@ -37,8 +37,8 @@ class Logger { return level <= level_; } - template - void operator()(int level, String fmt, Args... args) { + template + void operator()(int level, const char *fmt, Args... args) { if (level > level_) { return; }