diff --git a/.buckconfig b/.buckconfig new file mode 100644 index 000000000..b2b9c036f --- /dev/null +++ b/.buckconfig @@ -0,0 +1,9 @@ +[cxx] + cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1 + cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith + cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1 + cxxflags = -std=c++11 -Wno-format-security -Wno-deprecated-declarations + gtest_dep = //contrib/pzstd:gtest + +[httpserver] + port = 0 diff --git a/.buckversion b/.buckversion new file mode 100644 index 000000000..892fad966 --- /dev/null +++ b/.buckversion @@ -0,0 +1 @@ +c8dec2e8da52d483f6dd7c6cd2ad694e8e6fed2b diff --git a/.gitignore b/.gitignore index dd7a74519..e02119883 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ googletest/ # Directories bin/ +.buckd/ +buck-out/ diff --git a/contrib/pzstd/BUCK b/contrib/pzstd/BUCK new file mode 100644 index 000000000..d04eeedd8 --- /dev/null +++ b/contrib/pzstd/BUCK @@ -0,0 +1,72 @@ +cxx_library( + name='libpzstd', + visibility=['PUBLIC'], + header_namespace='', + exported_headers=[ + 'ErrorHolder.h', + 'Logging.h', + 'Pzstd.h', + ], + headers=[ + 'SkippableFrame.h', + ], + srcs=[ + 'Pzstd.cpp', + 'SkippableFrame.cpp', + ], + deps=[ + ':options', + '//contrib/pzstd/utils:utils', + '//lib:mem', + '//lib:zstd', + ], +) + +cxx_library( + name='options', + visibility=['PUBLIC'], + header_namespace='', + exported_headers=['Options.h'], + srcs=['Options.cpp'], + deps=[ + '//contrib/pzstd/utils:scope_guard', + '//lib:zstd', + '//programs:util', + ], +) + +cxx_binary( + name='pzstd', + visibility=['PUBLIC'], + srcs=['main.cpp'], + deps=[ + ':libpzstd', + ':options', + ], +) + +# Must run "make googletest" first +cxx_library( + name='gtest', + srcs=glob([ + 'googletest/googletest/src/gtest-all.cc', + 'googletest/googlemock/src/gmock-all.cc', + 'googletest/googlemock/src/gmock_main.cc', + ]), + header_namespace='', + exported_headers=subdir_glob([ + ('googletest/googletest/include', '**/*.h'), + ('googletest/googlemock/include', '**/*.h'), + ]), + headers=subdir_glob([ + ('googletest/googletest', 'src/*.cc'), + ('googletest/googletest', 'src/*.h'), + ('googletest/googlemock', 'src/*.cc'), + ('googletest/googlemock', 'src/*.h'), + ]), + platform_linker_flags=[ + ('android', []), + ('', ['-lpthread']), + ], + visibility=['PUBLIC'], +) diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp index 0b1403354..a0d969393 100644 --- a/contrib/pzstd/Options.cpp +++ b/contrib/pzstd/Options.cpp @@ -7,6 +7,7 @@ * of patent rights can be found in the PATENTS file in the same directory. */ #include "Options.h" +#include "util.h" #include "utils/ScopeGuard.h" #include @@ -15,7 +16,6 @@ #include #include #include -#include #include #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || \ diff --git a/contrib/pzstd/main.cpp b/contrib/pzstd/main.cpp index 279cbfb5e..7d8dbfbcf 100644 --- a/contrib/pzstd/main.cpp +++ b/contrib/pzstd/main.cpp @@ -9,11 +9,6 @@ #include "ErrorHolder.h" #include "Options.h" #include "Pzstd.h" -#include "utils/FileSystem.h" -#include "utils/Range.h" -#include "utils/ScopeGuard.h" -#include "utils/ThreadPool.h" -#include "utils/WorkQueue.h" using namespace pzstd; diff --git a/contrib/pzstd/test/BUCK b/contrib/pzstd/test/BUCK new file mode 100644 index 000000000..6d3fdd3c2 --- /dev/null +++ b/contrib/pzstd/test/BUCK @@ -0,0 +1,37 @@ +cxx_test( + name='options_test', + srcs=['OptionsTest.cpp'], + deps=['//contrib/pzstd:options'], +) + +cxx_test( + name='pzstd_test', + srcs=['PzstdTest.cpp'], + deps=[ + ':round_trip', + '//contrib/pzstd:libpzstd', + '//contrib/pzstd/utils:scope_guard', + '//programs:datagen', + ], +) + +cxx_binary( + name='round_trip_test', + srcs=['RoundTripTest.cpp'], + deps=[ + ':round_trip', + '//contrib/pzstd/utils:scope_guard', + '//programs:datagen', + ] +) + +cxx_library( + name='round_trip', + header_namespace='test', + exported_headers=['RoundTrip.h'], + deps=[ + '//contrib/pzstd:libpzstd', + '//contrib/pzstd:options', + '//contrib/pzstd/utils:scope_guard', + ] +) diff --git a/contrib/pzstd/test/PzstdTest.cpp b/contrib/pzstd/test/PzstdTest.cpp index c85f73a39..cadfa83f7 100644 --- a/contrib/pzstd/test/PzstdTest.cpp +++ b/contrib/pzstd/test/PzstdTest.cpp @@ -41,23 +41,20 @@ TEST(Pzstd, SmallSizes) { std::fclose(fd); ASSERT_EQ(written, len); } - for (unsigned headers = 0; headers <= 1; ++headers) { - for (unsigned numThreads = 1; numThreads <= 2; ++numThreads) { - for (unsigned level = 1; level <= 4; level *= 4) { - auto errorGuard = makeScopeGuard([&] { - std::fprintf(stderr, "pzstd headers: %u\n", headers); - std::fprintf(stderr, "# threads: %u\n", numThreads); - std::fprintf(stderr, "compression level: %u\n", level); - }); - Options options; - options.overwrite = true; - options.inputFiles = {inputFile}; - options.numThreads = numThreads; - options.compressionLevel = level; - options.verbosity = 1; - ASSERT_TRUE(roundTrip(options)); - errorGuard.dismiss(); - } + for (unsigned numThreads = 1; numThreads <= 2; ++numThreads) { + for (unsigned level = 1; level <= 4; level *= 4) { + auto errorGuard = makeScopeGuard([&] { + std::fprintf(stderr, "# threads: %u\n", numThreads); + std::fprintf(stderr, "compression level: %u\n", level); + }); + Options options; + options.overwrite = true; + options.inputFiles = {inputFile}; + options.numThreads = numThreads; + options.compressionLevel = level; + options.verbosity = 1; + ASSERT_TRUE(roundTrip(options)); + errorGuard.dismiss(); } } } @@ -79,29 +76,26 @@ TEST(Pzstd, LargeSizes) { std::fclose(fd); ASSERT_EQ(written, len); } - for (unsigned headers = 0; headers <= 1; ++headers) { - for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) { - for (unsigned level = 1; level <= 4; level *= 2) { - auto errorGuard = makeScopeGuard([&] { - std::fprintf(stderr, "pzstd headers: %u\n", headers); - std::fprintf(stderr, "# threads: %u\n", numThreads); - std::fprintf(stderr, "compression level: %u\n", level); - }); - Options options; - options.overwrite = true; - options.inputFiles = {inputFile}; - options.numThreads = std::min(numThreads, options.numThreads); - options.compressionLevel = level; - options.verbosity = 1; - ASSERT_TRUE(roundTrip(options)); - errorGuard.dismiss(); - } + for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) { + for (unsigned level = 1; level <= 4; level *= 4) { + auto errorGuard = makeScopeGuard([&] { + std::fprintf(stderr, "# threads: %u\n", numThreads); + std::fprintf(stderr, "compression level: %u\n", level); + }); + Options options; + options.overwrite = true; + options.inputFiles = {inputFile}; + options.numThreads = std::min(numThreads, options.numThreads); + options.compressionLevel = level; + options.verbosity = 1; + ASSERT_TRUE(roundTrip(options)); + errorGuard.dismiss(); } } } } -TEST(Pzstd, ExtremelyLargeSize) { +TEST(Pzstd, DISABLED_ExtremelyLargeSize) { unsigned seed = std::random_device{}(); std::fprintf(stderr, "Pzstd.ExtremelyLargeSize seed: %u\n", seed); std::mt19937 gen(seed); diff --git a/contrib/pzstd/utils/BUCK b/contrib/pzstd/utils/BUCK new file mode 100644 index 000000000..e757f4120 --- /dev/null +++ b/contrib/pzstd/utils/BUCK @@ -0,0 +1,75 @@ +cxx_library( + name='buffer', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['Buffer.h'], + deps=[':range'], +) + +cxx_library( + name='file_system', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['FileSystem.h'], + deps=[':range'], +) + +cxx_library( + name='likely', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['Likely.h'], +) + +cxx_library( + name='range', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['Range.h'], + deps=[':likely'], +) + +cxx_library( + name='resource_pool', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['ResourcePool.h'], +) + +cxx_library( + name='scope_guard', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['ScopeGuard.h'], +) + +cxx_library( + name='thread_pool', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['ThreadPool.h'], + deps=[':work_queue'], +) + +cxx_library( + name='work_queue', + visibility=['PUBLIC'], + header_namespace='utils', + exported_headers=['WorkQueue.h'], + deps=[':buffer'], +) + +cxx_library( + name='utils', + visibility=['PUBLIC'], + deps=[ + ':buffer', + ':file_system', + ':likely', + ':range', + ':resource_pool', + ':scope_guard', + ':thread_pool', + ':work_queue', + ], +) diff --git a/contrib/pzstd/utils/test/BUCK b/contrib/pzstd/utils/test/BUCK new file mode 100644 index 000000000..a5113cab6 --- /dev/null +++ b/contrib/pzstd/utils/test/BUCK @@ -0,0 +1,35 @@ +cxx_test( + name='buffer_test', + srcs=['BufferTest.cpp'], + deps=['//contrib/pzstd/utils:buffer'], +) + +cxx_test( + name='range_test', + srcs=['RangeTest.cpp'], + deps=['//contrib/pzstd/utils:range'], +) + +cxx_test( + name='resource_pool_test', + srcs=['ResourcePoolTest.cpp'], + deps=['//contrib/pzstd/utils:resource_pool'], +) + +cxx_test( + name='scope_guard_test', + srcs=['ScopeGuardTest.cpp'], + deps=['//contrib/pzstd/utils:scope_guard'], +) + +cxx_test( + name='thread_pool_test', + srcs=['ThreadPoolTest.cpp'], + deps=['//contrib/pzstd/utils:thread_pool'], +) + +cxx_test( + name='work_queue_test', + srcs=['RangeTest.cpp'], + deps=['//contrib/pzstd/utils:work_queue'], +) diff --git a/lib/BUCK b/lib/BUCK new file mode 100644 index 000000000..6812c1b1e --- /dev/null +++ b/lib/BUCK @@ -0,0 +1,186 @@ +cxx_library( + name='zstd', + header_namespace='', + visibility=['PUBLIC'], + deps=[ + ':common', + ':compress', + ':decompress', + ':deprecated', + ], +) + +cxx_library( + name='compress', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('compress', 'zstdmt_compress.h'), + ]), + headers=subdir_glob([ + ('compress', 'zstd_opt.h'), + ]), + srcs=[ + 'compress/zstd_compress.c', + 'compress/zstdmt_compress.c', + ], + deps=[':common'], +) + +cxx_library( + name='decompress', + header_namespace='', + visibility=['PUBLIC'], + srcs=['decompress/zstd_decompress.c'], + deps=[ + ':common', + ':legacy', + ], +) + +cxx_library( + name='deprecated', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('decprecated', '*.h'), + ]), + srcs=glob(['deprecated/*.c']), + deps=[':common'], +) + +cxx_library( + name='legacy', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('legacy', '*.h'), + ]), + srcs=glob(['legacy/*.c']), + deps=[':common'], +) + +cxx_library( + name='zdict', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('dictBuilder', 'zdict.h'), + ]), + headers=subdir_glob([ + ('dictBuilder', 'divsufsort.h'), + ]), + srcs=glob(['dictBuilder/*.c']), + deps=[':common'], +) + +cxx_library( + name='bitstream', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'bitstream.h'), + ]), +) + +cxx_library( + name='entropy', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'fse.h'), + ('common', 'huf.h'), + ]), + srcs=[ + 'common/entropy_common.c', + 'common/fse_decompress.c', + 'compress/fse_compress.c', + 'compress/huf_compress.c', + 'decompress/huf_decompress.c', + ], + deps=[ + ':bitstream', + ':errors', + ':mem', + ], +) + +cxx_library( + name='errors', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'error_private.h'), + ('common', 'zstd_errors.h'), + ]), + srcs=['common/error_private.c'], +) + +cxx_library( + name='mem', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'mem.h'), + ]), +) + +cxx_library( + name='pool', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'pool.h'), + ]), + srcs=['common/pool.c'], + deps=[':threading'], +) + +cxx_library( + name='threading', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'threading.h'), + ]), + srcs=['common/threading.c'], +) + +cxx_library( + name='xxhash', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'xxhash.h'), + ]), + srcs=['common/xxhash.c'], +) + +cxx_library( + name='zstd_common', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('', 'zstd.h'), + ('common', 'zstd_internal.h'), + ]), + srcs=['common/zstd_common.c'], + deps=[ + ':errors', + ':mem', + ], +) + +cxx_library( + name='common', + deps=[ + ':bitstream', + ':entropy', + ':errors', + ':mem', + ':pool', + ':threading', + ':xxhash', + ':zstd_common', + ] +) diff --git a/programs/BUCK b/programs/BUCK new file mode 100644 index 000000000..069403042 --- /dev/null +++ b/programs/BUCK @@ -0,0 +1,63 @@ +cxx_binary( + name='zstd', + headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']), + srcs=glob(['*.c'], excludes=['datagen.c']), + deps=[ + ':datagen', + ':util', + '//lib:zstd', + '//lib:zdict', + '//lib:mem', + '//lib:xxhash', + ], +) + +cxx_binary( + name='zstdmt', + headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']), + srcs=glob(['*.c'], excludes=['datagen.c']), + deps=[ + ':datagen', + ':util', + '//lib:zstd', + '//lib:zdict', + '//lib:mem', + '//lib:xxhash', + ], + preprocessor_flags=['-DZSTD_MULTITHREAD'], + linker_flags=['-lpthread'], +) + +cxx_binary( + name='gzstd', + headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']), + srcs=glob(['*.c'], excludes=['datagen.c']), + deps=[ + ':datagen', + ':util', + '//lib:zstd', + '//lib:zdict', + '//lib:mem', + '//lib:xxhash', + ], + preprocessor_flags=['-DZSTD_GZDECOMPRESS'], + linker_flags=['-lz'], +) + +cxx_library( + name='datagen', + visibility=['PUBLIC'], + header_namespace='', + exported_headers=['datagen.h'], + srcs=['datagen.c'], + deps=['//lib:mem'], +) + + +cxx_library( + name='util', + visibility=['PUBLIC'], + header_namespace='', + exported_headers=['util.h', 'platform.h'], + deps=['//lib:mem'], +) diff --git a/zlibWrapper/BUCK b/zlibWrapper/BUCK new file mode 100644 index 000000000..a3b74ac3f --- /dev/null +++ b/zlibWrapper/BUCK @@ -0,0 +1,22 @@ +cxx_library( + name='zlib_wrapper', + visibility=['PUBLIC'], + exported_linker_flags=['-lz'], + header_namespace='', + exported_headers=['zstd_zlibwrapper.h'], + headers=[ + 'gzcompatibility.h', + 'gzguts.h', + ], + srcs=glob(['*.c']), + deps=[ + '//lib:zstd', + '//lib:zstd_common', + ] +) + +cxx_binary( + name='minigzip', + srcs=['examples/minigzip.c'], + deps=[':zlib_wrapper'], +)