diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml index 87b250728..ae4ee7e9f 100644 --- a/.github/workflows/generic-dev.yml +++ b/.github/workflows/generic-dev.yml @@ -30,6 +30,9 @@ jobs: test: runs-on: ubuntu-latest + env: + DEVNULLRIGHTS: 1 + READFROMBLOCKDEVICE: 1 steps: - uses: actions/checkout@v2 - name: make test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 44f2393a2..5effa26eb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -399,7 +399,105 @@ disclosure of security bugs. In those cases, please go through the process outlined on that page and do not file a public issue. ## Coding Style +It's a pretty long topic, which is difficult to summarize in a single paragraph. +As a rule of thumbs, try to imitate the coding style of +similar lines of codes around your contribution. +The following is a non-exhaustive list of rules employed in zstd code base: + +### C90 +This code base is following strict C90 standard, +with 2 extensions : 64-bit `long long` types, and variadic macros. +This rule is applied strictly to code within `lib/` and `programs/`. +Sub-project in `contrib/` are allowed to use other conventions. + +### C++ direct compatibility : symbol mangling +All public symbol declarations must be wrapped in `extern “C” { … }`, +so that this project can be compiled as C++98 code, +and linked into C++ applications. + +### Minimal Frugal +This design requirement is fundamental to preserve the portability of the code base. +#### Dependencies +- Reduce dependencies to the minimum possible level. + Any dependency should be considered “bad” by default, + and only tolerated because it provides a service in a better way than can be achieved locally. + The only external dependencies this repository tolerates are + standard C libraries, and in rare cases, system level headers. +- Within `lib/`, this policy is even more drastic. + The only external dependencies allowed are ``, ``, ``, + and even then, not directly. + In particular, no function shall ever allocate on heap directly, + and must use instead `ZSTD_malloc()` and equivalent. + Other accepted non-symbol headers are `` and ``. +- Within the project, there is a strict hierarchy of dependencies that must be respected. + `programs/` is allowed to depend on `lib/`, but only its public API. + Within `lib/`, `lib/common` doesn't depend on any other directory. + `lib/compress` and `lib/decompress` shall not depend on each other. + `lib/dictBuilder` can depend on `lib/common` and `lib/compress`, but not `lib/decompress`. +#### Resources +- Functions in `lib/` must use very little stack space, + several dozens of bytes max. + Everything larger must use the heap allocator, + or require a scratch buffer to be emplaced manually. + +### Naming +* All public symbols are prefixed with `ZSTD_` + + private symbols, with a scope limited to their own unit, are free of this restriction. + However, since `libzstd` source code can be amalgamated, + each symbol name must attempt to be (and remain) unique. + Avoid too generic names that could become ground for future collisions. + This generally implies usage of some form of prefix. +* For symbols (functions and variables), naming convention is `PREFIX_camelCase`. + + In some advanced cases, one can also find : + - `PREFIX_prefix2_camelCase` + - `PREFIX_camelCase_extendedQualifier` +* Multi-words names generally consist of an action followed by object: + - for example : `ZSTD_createCCtx()` +* Prefer positive actions + - `goBackward` rather than `notGoForward` +* Type names (`struct`, etc.) follow similar convention, + except that they are allowed and even invited to start by an Uppercase letter. + Example : `ZSTD_CCtx`, `ZSTD_CDict` +* Macro names are all Capital letters. + The same composition rules (`PREFIX_NAME_QUALIFIER`) apply. +* File names are all lowercase letters. + The convention is `snake_case`. + File names **must** be unique across the entire code base, + even when they stand in clearly separated directories. + +### Qualifiers +* This code base is `const` friendly, if not `const` fanatical. + Any variable that can be `const` (aka. read-only) **must** be `const`. + Any pointer which content will not be modified must be `const`. + This property is then controlled at compiler level. + `const` variables are an important signal to readers that this variable isn’t modified. + Conversely, non-const variables are a signal to readers to watch out for modifications later on in the function. +* If a function must be inlined, mention it explicitly, + using project's own portable macros, such as `FORCE_INLINE_ATTR`, + defined in `lib/common/compiler.h`. + +### Debugging +* **Assertions** are welcome, and should be used very liberally, + to control any condition the code expects for its correct execution. + These assertion checks will be run in debug builds, and disabled in production. +* For traces, this project provides its own debug macros, + in particular `DEBUGLOG(level, ...)`, defined in `lib/common/debug.h`. + +### Code documentation +* Avoid code documentation that merely repeats what the code is already stating. + Whenever applicable, prefer employing the code as the primary way to convey explanations. + Example 1 : `int nbTokens = n;` instead of `int i = n; /* i is a nb of tokens *./`. + Example 2 : `assert(size > 0);` instead of `/* here, size should be positive */`. +* At declaration level, the documentation explains how to use the function or variable + and when applicable why it's needed, of the scenarios where it can be useful. +* At implementation level, the documentation explains the general outline of the algorithm employed, + and when applicable why this specific choice was preferred. + +### General layout * 4 spaces for indentation rather than tabs +* Code documentation shall directly precede function declaration or implementation +* Function implementations and its code documentation should be preceded and followed by an empty line + ## License By contributing to Zstandard, you agree that your contributions will be licensed diff --git a/Makefile b/Makefile index 8522d9d5e..55c24530b 100644 --- a/Makefile +++ b/Makefile @@ -122,8 +122,8 @@ contrib: lib $(MAKE) -C contrib/seekable_format/examples all $(MAKE) -C contrib/seekable_format/tests test $(MAKE) -C contrib/largeNbDicts all - cd contrib/single_file_libs/ ; ./build_decoder_test.sh - cd contrib/single_file_libs/ ; ./build_library_test.sh + cd build/single_file_libs/ ; ./build_decoder_test.sh + cd build/single_file_libs/ ; ./build_library_test.sh .PHONY: cleanTabs cleanTabs: diff --git a/TESTING.md b/TESTING.md index b851d1c8d..32b133b67 100644 --- a/TESTING.md +++ b/TESTING.md @@ -40,5 +40,4 @@ They consist of the following tests: - Versions test (ensuring `zstd` can decode files from all previous versions) - `pzstd` with asan and tsan, as well as in 32-bits mode - Testing `zstd` with legacy mode off -- Testing `zbuff` (old streaming API) - Entire test suite and make install on macOS diff --git a/appveyor.yml b/appveyor.yml index 172159315..24437be96 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -52,6 +52,15 @@ PLATFORM: "Win32" CONFIGURATION: "Release" + - COMPILER: "clang-cl" + HOST: "cmake-visual" + PLATFORM: "x64" + CONFIGURATION: "Release" + CMAKE_GENERATOR: "Visual Studio 15 2017" + CMAKE_GENERATOR_PLATFORM: "x64" + CMAKE_GENERATOR_TOOLSET: "LLVM" + APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017" + install: - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% - SET PATH_ORIGINAL=%PATH% @@ -154,6 +163,15 @@ COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\ ) + - if [%HOST%]==[cmake-visual] ( + ECHO *** && + ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% && + PUSHD build\cmake && + cmake -DBUILD_TESTING=ON . && + cmake --build . --config %CONFIGURATION% -j4 && + POPD && + ECHO *** + ) test_script: - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% @@ -223,6 +241,15 @@ PLATFORM: "Win32" CONFIGURATION: "Release" + - COMPILER: "clang-cl" + HOST: "cmake-visual" + PLATFORM: "x64" + CONFIGURATION: "Release" + CMAKE_GENERATOR: "Visual Studio 15 2017" + CMAKE_GENERATOR_PLATFORM: "x64" + CMAKE_GENERATOR_TOOLSET: "LLVM" + APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017" + install: - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% - SET PATH_ORIGINAL=%PATH% @@ -281,6 +308,15 @@ COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\ ) + - if [%HOST%]==[cmake-visual] ( + ECHO *** && + ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% && + PUSHD build\cmake && + cmake -DBUILD_TESTING=ON . && + cmake --build . --config %CONFIGURATION% -j4 && + POPD && + ECHO *** + ) test_script: diff --git a/build/VS2008/fullbench/fullbench.vcproj b/build/VS2008/fullbench/fullbench.vcproj index 5e349dce9..5a30c0b00 100644 --- a/build/VS2008/fullbench/fullbench.vcproj +++ b/build/VS2008/fullbench/fullbench.vcproj @@ -45,7 +45,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy" - PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -121,7 +121,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy" - PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" @@ -195,7 +195,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy" - PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -272,7 +272,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy" - PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" diff --git a/build/VS2008/fuzzer/fuzzer.vcproj b/build/VS2008/fuzzer/fuzzer.vcproj index 32f284684..664e60cbd 100644 --- a/build/VS2008/fuzzer/fuzzer.vcproj +++ b/build/VS2008/fuzzer/fuzzer.vcproj @@ -45,7 +45,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs" - PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -121,7 +121,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs" - PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" @@ -195,7 +195,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs" - PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -272,7 +272,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\programs" - PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" diff --git a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj index befdc0445..74336c2dc 100644 --- a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj +++ b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj @@ -90,7 +90,7 @@ Level4 Disabled - WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) true false @@ -108,7 +108,7 @@ Level4 Disabled - WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) true false @@ -127,7 +127,7 @@ MaxSpeed true true - WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) false false MultiThreaded @@ -150,7 +150,7 @@ MaxSpeed true true - WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) false false MultiThreaded diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj index 2e0a042b0..e2d7b0de3 100644 --- a/build/VS2010/fullbench/fullbench.vcxproj +++ b/build/VS2010/fullbench/fullbench.vcxproj @@ -90,7 +90,7 @@ Level4 Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true false @@ -105,7 +105,7 @@ Level4 Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true false @@ -122,7 +122,7 @@ MaxSpeed true true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false false MultiThreaded @@ -142,7 +142,7 @@ MaxSpeed true true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false false MultiThreaded diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj index 91974ec7f..c1d31cf5d 100644 --- a/build/VS2010/fuzzer/fuzzer.vcxproj +++ b/build/VS2010/fuzzer/fuzzer.vcxproj @@ -90,7 +90,7 @@ Level4 Disabled - ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true false @@ -105,7 +105,7 @@ Level4 Disabled - ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true false @@ -122,7 +122,7 @@ MaxSpeed true true - ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false false MultiThreaded @@ -142,7 +142,7 @@ MaxSpeed true true - ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false false MultiThreaded diff --git a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake index 623897170..e23b9d603 100644 --- a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake +++ b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake @@ -26,7 +26,12 @@ macro(ADD_ZSTD_COMPILATION_FLAGS) EnableCompilerFlag("-std=c++11" false true) #Set c99 by default EnableCompilerFlag("-std=c99" true false) - EnableCompilerFlag("-Wall" true true) + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND MSVC) + # clang-cl normally maps -Wall to -Weverything. + EnableCompilerFlag("/clang:-Wall" true true) + else () + EnableCompilerFlag("-Wall" true true) + endif () EnableCompilerFlag("-Wextra" true true) EnableCompilerFlag("-Wundef" true true) EnableCompilerFlag("-Wshadow" true true) diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index d58c652a1..5f756652e 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -24,28 +24,24 @@ file(GLOB CommonSources ${LIBRARY_DIR}/common/*.c) file(GLOB CompressSources ${LIBRARY_DIR}/compress/*.c) file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c) file(GLOB DictBuilderSources ${LIBRARY_DIR}/dictBuilder/*.c) -file(GLOB DeprecatedSources ${LIBRARY_DIR}/deprecated/*.c) set(Sources ${CommonSources} ${CompressSources} ${DecompressSources} - ${DictBuilderSources} - ${DeprecatedSources}) + ${DictBuilderSources}) file(GLOB CommonHeaders ${LIBRARY_DIR}/common/*.h) file(GLOB CompressHeaders ${LIBRARY_DIR}/compress/*.h) file(GLOB DecompressHeaders ${LIBRARY_DIR}/decompress/*.h) file(GLOB DictBuilderHeaders ${LIBRARY_DIR}/dictBuilder/*.h) -file(GLOB DeprecatedHeaders ${LIBRARY_DIR}/deprecated/*.h) set(Headers ${LIBRARY_DIR}/zstd.h ${CommonHeaders} ${CompressHeaders} ${DecompressHeaders} - ${DictBuilderHeaders} - ${DeprecatedHeaders}) + ${DictBuilderHeaders}) if (ZSTD_LEGACY_SUPPORT) set(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy) diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt index 34eca9109..f7b6e9087 100644 --- a/build/cmake/tests/CMakeLists.txt +++ b/build/cmake/tests/CMakeLists.txt @@ -57,6 +57,7 @@ target_link_libraries(datagen libzstd_static) # fullbench # add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c) +set_property(TARGET fullbench APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DISABLE_DEPRECATE_WARNINGS") target_link_libraries(fullbench libzstd_static) add_test(NAME fullbench COMMAND fullbench) @@ -64,6 +65,7 @@ add_test(NAME fullbench COMMAND fullbench) # fuzzer # add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c) +set_property(TARGET fuzzer APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DISABLE_DEPRECATE_WARNINGS") target_link_libraries(fuzzer libzstd_static) AddTestFlagsOption(ZSTD_FUZZER_FLAGS "$ENV{FUZZERTEST} $ENV{FUZZER_FLAGS}" "Semicolon-separated list of flags to pass to the fuzzer test (see `fuzzer -h` for usage)") @@ -76,6 +78,7 @@ add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS}) # zstreamtest # add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c) +set_property(TARGET zstreamtest APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DISABLE_DEPRECATE_WARNINGS") target_link_libraries(zstreamtest libzstd_static) AddTestFlagsOption(ZSTD_ZSTREAM_FLAGS "$ENV{ZSTREAM_TESTTIME} $ENV{FUZZER_FLAGS}" "Semicolon-separated list of flags to pass to the zstreamtest test (see `zstreamtest -h` for usage)") diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build index 2cbd39cf2..5cc9fee86 100644 --- a/build/meson/lib/meson.build +++ b/build/meson/lib/meson.build @@ -14,8 +14,7 @@ libzstd_includes = [include_directories(join_paths(zstd_rootdir,'lib'), join_paths(zstd_rootdir, 'lib/common'), join_paths(zstd_rootdir, 'lib/compress'), join_paths(zstd_rootdir, 'lib/decompress'), - join_paths(zstd_rootdir, 'lib/dictBuilder'), - join_paths(zstd_rootdir, 'lib/deprecated'))] + join_paths(zstd_rootdir, 'lib/dictBuilder'))] libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'), join_paths(zstd_rootdir, 'lib/common/fse_decompress.c'), diff --git a/contrib/single_file_libs/.gitignore b/build/single_file_libs/.gitignore similarity index 100% rename from contrib/single_file_libs/.gitignore rename to build/single_file_libs/.gitignore diff --git a/contrib/single_file_libs/README.md b/build/single_file_libs/README.md similarity index 96% rename from contrib/single_file_libs/README.md rename to build/single_file_libs/README.md index d88e8fc6b..1705b769b 100644 --- a/contrib/single_file_libs/README.md +++ b/build/single_file_libs/README.md @@ -11,7 +11,7 @@ This is the most common use case. The decompression library is small, adding, fo Create `zstddeclib.c` from the Zstd source using: ``` -cd zstd/contrib/single_file_libs +cd zstd/build/single_file_libs ./combine.sh -r ../../lib -o zstddeclib.c zstddeclib-in.c ``` Then add the resulting file to your project (see the [example files](examples)). @@ -25,7 +25,7 @@ The same tool can amalgamate the entire Zstd library for ease of adding both com Create `zstd.c` from the Zstd source using: ``` -cd zstd/contrib/single_file_libs +cd zstd/build/single_file_libs ./combine.sh -r ../../lib -o zstd.c zstd-in.c ``` It's possible to create a compressor-only library but since the decompressor is so small in comparison this doesn't bring much of a gain (but for the curious, simply remove the files in the _decompress_ section at the end of `zstd-in.c`). diff --git a/contrib/single_file_libs/build_decoder_test.sh b/build/single_file_libs/build_decoder_test.sh similarity index 100% rename from contrib/single_file_libs/build_decoder_test.sh rename to build/single_file_libs/build_decoder_test.sh diff --git a/contrib/single_file_libs/build_library_test.sh b/build/single_file_libs/build_library_test.sh similarity index 100% rename from contrib/single_file_libs/build_library_test.sh rename to build/single_file_libs/build_library_test.sh diff --git a/contrib/single_file_libs/combine.sh b/build/single_file_libs/combine.sh similarity index 100% rename from contrib/single_file_libs/combine.sh rename to build/single_file_libs/combine.sh diff --git a/contrib/single_file_libs/create_single_file_decoder.sh b/build/single_file_libs/create_single_file_decoder.sh similarity index 100% rename from contrib/single_file_libs/create_single_file_decoder.sh rename to build/single_file_libs/create_single_file_decoder.sh diff --git a/contrib/single_file_libs/create_single_file_library.sh b/build/single_file_libs/create_single_file_library.sh similarity index 100% rename from contrib/single_file_libs/create_single_file_library.sh rename to build/single_file_libs/create_single_file_library.sh diff --git a/contrib/single_file_libs/examples/README.md b/build/single_file_libs/examples/README.md similarity index 100% rename from contrib/single_file_libs/examples/README.md rename to build/single_file_libs/examples/README.md diff --git a/contrib/single_file_libs/examples/emscripten.c b/build/single_file_libs/examples/emscripten.c similarity index 100% rename from contrib/single_file_libs/examples/emscripten.c rename to build/single_file_libs/examples/emscripten.c diff --git a/contrib/single_file_libs/examples/roundtrip.c b/build/single_file_libs/examples/roundtrip.c similarity index 100% rename from contrib/single_file_libs/examples/roundtrip.c rename to build/single_file_libs/examples/roundtrip.c diff --git a/contrib/single_file_libs/examples/shell.html b/build/single_file_libs/examples/shell.html similarity index 100% rename from contrib/single_file_libs/examples/shell.html rename to build/single_file_libs/examples/shell.html diff --git a/contrib/single_file_libs/examples/simple.c b/build/single_file_libs/examples/simple.c similarity index 100% rename from contrib/single_file_libs/examples/simple.c rename to build/single_file_libs/examples/simple.c diff --git a/contrib/single_file_libs/examples/testcard-dxt1.inl b/build/single_file_libs/examples/testcard-dxt1.inl similarity index 100% rename from contrib/single_file_libs/examples/testcard-dxt1.inl rename to build/single_file_libs/examples/testcard-dxt1.inl diff --git a/contrib/single_file_libs/examples/testcard-zstd.inl b/build/single_file_libs/examples/testcard-zstd.inl similarity index 100% rename from contrib/single_file_libs/examples/testcard-zstd.inl rename to build/single_file_libs/examples/testcard-zstd.inl diff --git a/contrib/single_file_libs/examples/testcard.png b/build/single_file_libs/examples/testcard.png similarity index 100% rename from contrib/single_file_libs/examples/testcard.png rename to build/single_file_libs/examples/testcard.png diff --git a/contrib/single_file_libs/zstd-in.c b/build/single_file_libs/zstd-in.c similarity index 100% rename from contrib/single_file_libs/zstd-in.c rename to build/single_file_libs/zstd-in.c diff --git a/contrib/single_file_libs/zstddeclib-in.c b/build/single_file_libs/zstddeclib-in.c similarity index 100% rename from contrib/single_file_libs/zstddeclib-in.c rename to build/single_file_libs/zstddeclib-in.c diff --git a/contrib/VS2005/zstdlib/zstdlib.vcproj b/contrib/VS2005/zstdlib/zstdlib.vcproj index e028bd677..67ddd2dba 100644 --- a/contrib/VS2005/zstdlib/zstdlib.vcproj +++ b/contrib/VS2005/zstdlib/zstdlib.vcproj @@ -473,10 +473,6 @@ RelativePath="..\..\..\lib\common\xxhash.h" > - - diff --git a/contrib/linux-kernel/test/Makefile b/contrib/linux-kernel/test/Makefile index 9c77f0679..2908839fd 100644 --- a/contrib/linux-kernel/test/Makefile +++ b/contrib/linux-kernel/test/Makefile @@ -11,7 +11,7 @@ LINUX := ../linux LINUX_ZSTDLIB := $(LINUX)/lib/zstd -CPPFLAGS += -I$(LINUX)/include -I$(LINUX_ZSTDLIB) -Iinclude -DNDEBUG +CPPFLAGS += -I$(LINUX)/include -I$(LINUX_ZSTDLIB) -Iinclude -DNDEBUG -Wno-deprecated-declarations # Don't poison the workspace, it currently doesn't work with static allocation and workspace reuse CPPFLAGS += -DZSTD_ASAN_DONT_POISON_WORKSPACE diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index 8d2b1932e..25265e79a 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -30,6 +30,9 @@ CXXFLAGS ?= -O3 -Wall -Wextra -pedantic CPPFLAGS ?= LDFLAGS ?= +# PZstd uses legacy APIs +CFLAGS += -Wno-deprecated-declarations + # Include flags PZSTD_INC = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I. GTEST_INC = -isystem googletest/googletest/include diff --git a/contrib/seekable_format/examples/Makefile b/contrib/seekable_format/examples/Makefile index 543780f75..9df6b75fb 100644 --- a/contrib/seekable_format/examples/Makefile +++ b/contrib/seekable_format/examples/Makefile @@ -13,7 +13,7 @@ ZSTDLIB_PATH = ../../../lib ZSTDLIB_NAME = libzstd.a ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME) -CPPFLAGS += -I../ -I../../../lib -I../../../lib/common +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I../../../lib -I../../../lib/common CFLAGS ?= -O3 CFLAGS += -g diff --git a/contrib/seekable_format/examples/parallel_compression.c b/contrib/seekable_format/examples/parallel_compression.c index 69644d2b3..4118b0ad7 100644 --- a/contrib/seekable_format/examples/parallel_compression.c +++ b/contrib/seekable_format/examples/parallel_compression.c @@ -21,7 +21,6 @@ # define SLEEP(x) usleep(x * 1000) #endif -#define XXH_NAMESPACE ZSTD_ #include "xxhash.h" #include "pool.h" // use zstd thread pool for demo diff --git a/contrib/seekable_format/tests/Makefile b/contrib/seekable_format/tests/Makefile index 15eadb40e..d51deb3ea 100644 --- a/contrib/seekable_format/tests/Makefile +++ b/contrib/seekable_format/tests/Makefile @@ -13,7 +13,7 @@ ZSTDLIB_PATH = ../../../lib ZSTDLIB_NAME = libzstd.a ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME) -CPPFLAGS += -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common CFLAGS ?= -O3 CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wconversion \ diff --git a/contrib/seekable_format/zstdseek_compress.c b/contrib/seekable_format/zstdseek_compress.c index d92917a62..242bd2ac3 100644 --- a/contrib/seekable_format/zstdseek_compress.c +++ b/contrib/seekable_format/zstdseek_compress.c @@ -12,7 +12,6 @@ #include #define XXH_STATIC_LINKING_ONLY -#define XXH_NAMESPACE ZSTD_ #include "xxhash.h" #define ZSTD_STATIC_LINKING_ONLY @@ -83,7 +82,7 @@ static size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog* fl) ZSTD_frameLog* ZSTD_seekable_createFrameLog(int checksumFlag) { - ZSTD_frameLog* const fl = malloc(sizeof(ZSTD_frameLog)); + ZSTD_frameLog* const fl = (ZSTD_frameLog*)malloc(sizeof(ZSTD_frameLog)); if (fl == NULL) return NULL; if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(fl))) { @@ -108,7 +107,7 @@ size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog* fl) ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void) { - ZSTD_seekable_CStream* const zcs = malloc(sizeof(ZSTD_seekable_CStream)); + ZSTD_seekable_CStream* const zcs = (ZSTD_seekable_CStream*)malloc(sizeof(ZSTD_seekable_CStream)); if (zcs == NULL) return NULL; memset(zcs, 0, sizeof(*zcs)); @@ -177,7 +176,7 @@ size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl, if (fl->size == fl->capacity) { /* exponential size increase for constant amortized runtime */ size_t const newCapacity = fl->capacity * 2; - framelogEntry_t* const newEntries = realloc(fl->entries, + framelogEntry_t* const newEntries = (framelogEntry_t*)realloc(fl->entries, sizeof(framelogEntry_t) * newCapacity); if (newEntries == NULL) return ERROR(memory_allocation); diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c index ecf816c17..5eed02495 100644 --- a/contrib/seekable_format/zstdseek_decompress.c +++ b/contrib/seekable_format/zstdseek_decompress.c @@ -60,7 +60,6 @@ #include #define XXH_STATIC_LINKING_ONLY -#define XXH_NAMESPACE ZSTD_ #include "xxhash.h" #define ZSTD_STATIC_LINKING_ONLY @@ -176,7 +175,7 @@ struct ZSTD_seekable_s { ZSTD_seekable* ZSTD_seekable_create(void) { - ZSTD_seekable* const zs = malloc(sizeof(ZSTD_seekable)); + ZSTD_seekable* const zs = (ZSTD_seekable*)malloc(sizeof(ZSTD_seekable)); if (zs == NULL) return NULL; /* also initializes stage to zsds_init */ @@ -202,7 +201,7 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs) ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs) { - ZSTD_seekTable* const st = malloc(sizeof(ZSTD_seekTable)); + ZSTD_seekTable* const st = (ZSTD_seekTable*)malloc(sizeof(ZSTD_seekTable)); if (st==NULL) return NULL; st->checksumFlag = zs->seekTable.checksumFlag; diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 4371156e3..f961d4c39 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -1611,6 +1611,10 @@ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + explicitly specified. + start a new frame, using same parameters from previous frame. This is typically useful to skip dictionary loading stage, since it will re-use it in-place. Note that zcs must be init at least once before using ZSTD_resetCStream(). diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 6e18b68e5..15a5c7c8d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -559,6 +559,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) bounds.upperBound = (int)ZSTD_urm_enableRowMatchFinder; return bounds; + case ZSTD_c_deterministicRefPrefix: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + default: bounds.error = ERROR(parameter_unsupported); return bounds; @@ -622,6 +627,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_validateSequences: case ZSTD_c_splitBlocks: case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: default: return 0; } @@ -676,6 +682,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) case ZSTD_c_validateSequences: case ZSTD_c_splitBlocks: case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); @@ -897,6 +904,11 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, CCtxParams->useRowMatchFinder = (ZSTD_useRowMatchFinderMode_e)value; return CCtxParams->useRowMatchFinder; + case ZSTD_c_deterministicRefPrefix: + BOUNDCHECK(ZSTD_c_deterministicRefPrefix, value); + CCtxParams->deterministicRefPrefix = !!value; + return CCtxParams->deterministicRefPrefix; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } } @@ -1026,6 +1038,9 @@ size_t ZSTD_CCtxParams_getParameter( case ZSTD_c_useRowMatchFinder : *value = (int)CCtxParams->useRowMatchFinder; break; + case ZSTD_c_deterministicRefPrefix: + *value = (int)CCtxParams->deterministicRefPrefix; + break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } return 0; @@ -1802,7 +1817,7 @@ static int ZSTD_dictTooBig(size_t const loadedDictSize) * note : `params` are assumed fully validated at this stage. */ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, - ZSTD_CCtx_params params, + ZSTD_CCtx_params const* params, U64 const pledgedSrcSize, size_t const loadedDictSize, ZSTD_compResetPolicy_e const crp, @@ -1810,30 +1825,36 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, { ZSTD_cwksp* const ws = &zc->workspace; DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d", - (U32)pledgedSrcSize, params.cParams.windowLog, (int)params.useRowMatchFinder); - assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); zc->isFirstBlock = 1; - assert(params.useRowMatchFinder != ZSTD_urm_auto); - if (params.ldmParams.enableLdm) { + /* Set applied params early so we can modify them for LDM, + * and point params at the applied params. + */ + zc->appliedParams = *params; + params = &zc->appliedParams; + + assert(params->useRowMatchFinder != ZSTD_urm_auto); + if (params->ldmParams.enableLdm) { /* Adjust long distance matching parameters */ - ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); - assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); - assert(params.ldmParams.hashRateLog < 32); + ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams); + assert(params->ldmParams.hashLog >= params->ldmParams.bucketSizeLog); + assert(params->ldmParams.hashRateLog < 32); } - { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); + { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize)); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); - U32 const divider = (params.cParams.minMatch==3) ? 3 : 4; + U32 const divider = (params->cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; - size_t const buffOutSize = (zbuff == ZSTDb_buffered && params.outBufferMode == ZSTD_bm_buffered) + size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered) ? ZSTD_compressBound(blockSize) + 1 : 0; - size_t const buffInSize = (zbuff == ZSTDb_buffered && params.inBufferMode == ZSTD_bm_buffered) + size_t const buffInSize = (zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered) ? windowSize + blockSize : 0; - size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize); int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); int const dictTooBig = ZSTD_dictTooBig(loadedDictSize); @@ -1842,7 +1863,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const neededSpace = ZSTD_estimateCCtxSize_usingCCtxParams_internal( - ¶ms.cParams, ¶ms.ldmParams, zc->staticSize != 0, params.useRowMatchFinder, + ¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder, buffInSize, buffOutSize, pledgedSrcSize); int resizeWorkspace; @@ -1885,8 +1906,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_cwksp_clear(ws); /* init params */ - zc->appliedParams = params; - zc->blockState.matchState.cParams = params.cParams; + zc->blockState.matchState.cParams = params->cParams; zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; zc->consumedSrcSize = 0; zc->producedCSize = 0; @@ -1917,11 +1937,11 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); /* ldm bucketOffsets table */ - if (params.ldmParams.enableLdm) { + if (params->ldmParams.enableLdm) { /* TODO: avoid memset? */ size_t const numBuckets = - ((size_t)1) << (params.ldmParams.hashLog - - params.ldmParams.bucketSizeLog); + ((size_t)1) << (params->ldmParams.hashLog - + params->ldmParams.bucketSizeLog); zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets); ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets); } @@ -1937,16 +1957,16 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, FORWARD_IF_ERROR(ZSTD_reset_matchState( &zc->blockState.matchState, ws, - ¶ms.cParams, - params.useRowMatchFinder, + ¶ms->cParams, + params->useRowMatchFinder, crp, needsIndexReset, ZSTD_resetTarget_CCtx), ""); /* ldm hash table */ - if (params.ldmParams.enableLdm) { + if (params->ldmParams.enableLdm) { /* TODO: avoid memset? */ - size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; + size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog; zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); @@ -2014,7 +2034,8 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { - DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%zu", pledgedSrcSize); + DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); { ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams; unsigned const windowLog = params.cParams.windowLog; @@ -2031,7 +2052,7 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, cdict->dictContentSize, ZSTD_cpm_attachDict); params.cParams.windowLog = windowLog; params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */ - FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, /* loadedDictSize */ 0, ZSTDcrp_makeClean, zbuff), ""); assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); @@ -2076,7 +2097,8 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; assert(!cdict->matchState.dedicatedDictSearch); - DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%zu", pledgedSrcSize); + DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); { unsigned const windowLog = params.cParams.windowLog; assert(windowLog != 0); @@ -2084,7 +2106,7 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, params.cParams = *cdict_cParams; params.cParams.windowLog = windowLog; params.useRowMatchFinder = cdict->useRowMatchFinder; - FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, /* loadedDictSize */ 0, ZSTDcrp_leaveDirty, zbuff), ""); assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); @@ -2190,7 +2212,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_urm_auto); params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder; params.fParams = fParams; - ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, + ZSTD_resetCCtx_internal(dstCCtx, ¶ms, pledgedSrcSize, /* loadedDictSize */ 0, ZSTDcrp_leaveDirty, zbuff); assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); @@ -4045,11 +4067,12 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, if (!srcSize) return fhSize; /* do not generate an empty block if no input */ - if (!ZSTD_window_update(&ms->window, src, srcSize)) { + if (!ZSTD_window_update(&ms->window, src, srcSize, ms->forceNonContiguous)) { + ms->forceNonContiguous = 0; ms->nextToUpdate = ms->window.dictLimit; } if (cctx->appliedParams.ldmParams.enableLdm) { - ZSTD_window_update(&cctx->ldmState.window, src, srcSize); + ZSTD_window_update(&cctx->ldmState.window, src, srcSize, /* forceNonContiguous */ 0); } if (!frame) { @@ -4141,11 +4164,12 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, } DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder); - ZSTD_window_update(&ms->window, src, srcSize); + ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0); ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); + ms->forceNonContiguous = params->deterministicRefPrefix; if (loadLdmDict) { - ZSTD_window_update(&ls->window, src, srcSize); + ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0); ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); } @@ -4422,7 +4446,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } - FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize, + FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, dictContentSize, ZSTDcrp_makeClean, zbuff) , ""); { size_t const dictID = cdict ? @@ -4591,15 +4615,14 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, const void* dict,size_t dictSize, ZSTD_parameters params) { - ZSTD_CCtx_params cctxParams; DEBUGLOG(4, "ZSTD_compress_advanced"); FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); - ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, ZSTD_NO_CLEVEL); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, ZSTD_NO_CLEVEL); return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, - &cctxParams); + &cctx->simpleApiParams); } /* Internal */ @@ -4623,14 +4646,13 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_CCtx_params cctxParams; { ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict); assert(params.fParams.contentSizeFlag == 1); - ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); } DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize); - return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctxParams); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams); } size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, @@ -4963,15 +4985,15 @@ unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict) return cdict->dictID; } - -/* ZSTD_compressBegin_usingCDict_advanced() : - * cdict must be != NULL */ -size_t ZSTD_compressBegin_usingCDict_advanced( +/* ZSTD_compressBegin_usingCDict_internal() : + * Implementation of various ZSTD_compressBegin_usingCDict* functions. + */ +static size_t ZSTD_compressBegin_usingCDict_internal( ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) { ZSTD_CCtx_params cctxParams; - DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_internal"); RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!"); /* Initialize the cctxParams from the cdict */ { @@ -5003,23 +5025,46 @@ size_t ZSTD_compressBegin_usingCDict_advanced( ZSTDb_not_buffered); } + +/* ZSTD_compressBegin_usingCDict_advanced() : + * This function is DEPRECATED. + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_advanced( + ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) +{ + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, pledgedSrcSize); +} + /* ZSTD_compressBegin_usingCDict() : - * pledgedSrcSize=0 means "unknown" - * if pledgedSrcSize>0, it will enable contentSizeFlag */ + * cdict must be != NULL */ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; - DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag); - return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); } +/*! ZSTD_compress_usingCDict_internal(): + * Implementation of various ZSTD_compress_usingCDict* functions. + */ +static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); +} + +/*! ZSTD_compress_usingCDict_advanced(): + * This function is DEPRECATED. + */ size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) { - FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ - return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); } /*! ZSTD_compress_usingCDict() : @@ -5033,7 +5078,7 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; - return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index c0b017948..03974ae3a 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -219,6 +219,8 @@ struct ZSTD_matchState_t { U32* hashTable3; U32* chainTable; + U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */ + int dedicatedDictSearch; /* Indicates whether this matchState is using the * dedicated dictionary search structure. */ @@ -317,6 +319,9 @@ struct ZSTD_CCtx_params_s { /* Param for deciding whether to use row-based matchfinder */ ZSTD_useRowMatchFinderMode_e useRowMatchFinder; + /* Always load a dictionary in ext-dict mode (not prefix mode)? */ + int deterministicRefPrefix; + /* Internal use, for createCCtxParams() and freeCCtxParams() only */ ZSTD_customMem customMem; }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ @@ -340,6 +345,7 @@ struct ZSTD_CCtx_s { int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ ZSTD_CCtx_params requestedParams; ZSTD_CCtx_params appliedParams; + ZSTD_CCtx_params simpleApiParams; /* Param storage used by the simple API - not sticky. Must only be used in top-level simple API functions for storage. */ U32 dictID; size_t dictContentSize; @@ -1138,7 +1144,8 @@ MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) { * Returns non-zero if the segment is contiguous. */ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, - void const* src, size_t srcSize) + void const* src, size_t srcSize, + int forceNonContiguous) { BYTE const* const ip = (BYTE const*)src; U32 contiguous = 1; @@ -1148,7 +1155,7 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, assert(window->base != NULL); assert(window->dictBase != NULL); /* Check if blocks follow each other */ - if (src != window->nextSrc) { + if (src != window->nextSrc || forceNonContiguous) { /* not contiguous */ size_t const distanceFromBase = (size_t)(window->nextSrc - window->base); DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit); diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index cfd75cc49..22aa3e124 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -512,7 +512,7 @@ ZSTDMT_serialState_reset(serialState_t* serialState, if (dictSize > 0) { if (dictContentType == ZSTD_dct_rawContent) { BYTE const* const dictEnd = (const BYTE*)dict + dictSize; - ZSTD_window_update(&serialState->ldmState.window, dict, dictSize); + ZSTD_window_update(&serialState->ldmState.window, dict, dictSize, /* forceNonContiguous */ 0); ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, ¶ms.ldmParams); serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base); } else { @@ -569,7 +569,7 @@ static void ZSTDMT_serialState_update(serialState_t* serialState, assert(seqStore.seq != NULL && seqStore.pos == 0 && seqStore.size == 0 && seqStore.capacity > 0); assert(src.size <= serialState->params.jobSize); - ZSTD_window_update(&serialState->ldmState.window, src.start, src.size); + ZSTD_window_update(&serialState->ldmState.window, src.start, src.size, /* forceNonContiguous */ 0); error = ZSTD_ldm_generateSequences( &serialState->ldmState, &seqStore, &serialState->params.ldmParams, src.start, src.size); @@ -695,6 +695,10 @@ static void ZSTDMT_compressionJob(void* jobDescription) { size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob); if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError); } + if (!job->firstJob) { + size_t const err = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_deterministicRefPrefix, 0); + if (ZSTD_isError(err)) JOB_ERROR(err); + } { size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */ ZSTD_dtlm_fast, @@ -750,6 +754,12 @@ static void ZSTDMT_compressionJob(void* jobDescription) if (ZSTD_isError(cSize)) JOB_ERROR(cSize); lastCBlockSize = cSize; } } + if (!job->firstJob) { + /* Double check that we don't have an ext-dict, because then our + * repcode invalidation doesn't work. + */ + assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); + } ZSTD_CCtx_trace(cctx, 0); _endJob: diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index e0f293315..028be0bf0 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -1109,7 +1109,6 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, /* Regen sequences */ if (nbSeq) { seqState_t seqState; - size_t error = 0; dctx->fseEntropy = 1; { U32 i; for (i=0; ientropy.rep[i]; } RETURN_ERROR_IF( @@ -1173,22 +1172,17 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, assert(!ZSTD_isError(oneSeqSize)); if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); #endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); - BIT_reloadDStream(&(seqState.DStream)); op += oneSeqSize; - /* gcc and clang both don't like early returns in this loop. - * Instead break and check for an error at the end of the loop. - */ - if (UNLIKELY(ZSTD_isError(oneSeqSize))) { - error = oneSeqSize; + if (UNLIKELY(!--nbSeq)) break; - } - if (UNLIKELY(!--nbSeq)) break; + BIT_reloadDStream(&(seqState.DStream)); } /* check if reached exact end */ DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); - if (ZSTD_isError(error)) return error; RETURN_ERROR_IF(nbSeq, corruption_detected, ""); RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); /* save reps for next block */ @@ -1259,9 +1253,9 @@ ZSTD_decompressSequencesLong_body( /* Regen sequences */ if (nbSeq) { -#define STORED_SEQS 4 +#define STORED_SEQS 8 #define STORED_SEQS_MASK (STORED_SEQS-1) -#define ADVANCED_SEQS 4 +#define ADVANCED_SEQS STORED_SEQS seq_t sequences[STORED_SEQS]; int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); seqState_t seqState; diff --git a/lib/zdict.h b/lib/zdict.h index 4b8589f2f..75b05dbf4 100644 --- a/lib/zdict.h +++ b/lib/zdict.h @@ -36,6 +36,145 @@ extern "C" { # define ZDICTLIB_API ZDICTLIB_VISIBILITY #endif +/******************************************************************************* + * Zstd dictionary builder + * + * FAQ + * === + * Why should I use a dictionary? + * ------------------------------ + * + * Zstd can use dictionaries to improve compression ratio of small data. + * Traditionally small files don't compress well because there is very little + * repetion in a single sample, since it is small. But, if you are compressing + * many similar files, like a bunch of JSON records that share the same + * structure, you can train a dictionary on ahead of time on some samples of + * these files. Then, zstd can use the dictionary to find repetitions that are + * present across samples. This can vastly improve compression ratio. + * + * When is a dictionary useful? + * ---------------------------- + * + * Dictionaries are useful when compressing many small files that are similar. + * The larger a file is, the less benefit a dictionary will have. Generally, + * we don't expect dictionary compression to be effective past 100KB. And the + * smaller a file is, the more we would expect the dictionary to help. + * + * How do I use a dictionary? + * -------------------------- + * + * Simply pass the dictionary to the zstd compressor with + * `ZSTD_CCtx_loadDictionary()`. The same dictionary must then be passed to + * the decompressor, using `ZSTD_DCtx_loadDictionary()`. There are other + * more advanced functions that allow selecting some options, see zstd.h for + * complete documentation. + * + * What is a zstd dictionary? + * -------------------------- + * + * A zstd dictionary has two pieces: Its header, and its content. The header + * contains a magic number, the dictionary ID, and entropy tables. These + * entropy tables allow zstd to save on header costs in the compressed file, + * which really matters for small data. The content is just bytes, which are + * repeated content that is common across many samples. + * + * What is a raw content dictionary? + * --------------------------------- + * + * A raw content dictionary is just bytes. It doesn't have a zstd dictionary + * header, a dictionary ID, or entropy tables. Any buffer is a valid raw + * content dictionary. + * + * How do I train a dictionary? + * ---------------------------- + * + * Gather samples from your use case. These samples should be similar to each + * other. If you have several use cases, you could try to train one dictionary + * per use case. + * + * Pass those samples to `ZDICT_trainFromBuffer()` and that will train your + * dictionary. There are a few advanced versions of this function, but this + * is a great starting point. If you want to further tune your dictionary + * you could try `ZDICT_optimizeTrainFromBuffer_cover()`. If that is too slow + * you can try `ZDICT_optimizeTrainFromBuffer_fastCover()`. + * + * If the dictionary training function fails, that is likely because you + * either passed too few samples, or a dictionary would not be effective + * for your data. Look at the messages that the dictionary trainer printed, + * if it doesn't say too few samples, then a dictionary would not be effective. + * + * How large should my dictionary be? + * ---------------------------------- + * + * A reasonable dictionary size, the `dictBufferCapacity`, is about 100KB. + * The zstd CLI defaults to a 110KB dictionary. You likely don't need a + * dictionary larger than that. But, most use cases can get away with a + * smaller dictionary. The advanced dictionary builders can automatically + * shrink the dictionary for you, and select a the smallest size that + * doesn't hurt compression ratio too much. See the `shrinkDict` parameter. + * A smaller dictionary can save memory, and potentially speed up + * compression. + * + * How many samples should I provide to the dictionary builder? + * ------------------------------------------------------------ + * + * We generally recommend passing ~100x the size of the dictionary + * in samples. A few thousand should suffice. Having too few samples + * can hurt the dictionaries effectiveness. Having more samples will + * only improve the dictionaries effectiveness. But having too many + * samples can slow down the dictionary builder. + * + * How do I determine if a dictionary will be effective? + * ----------------------------------------------------- + * + * Simply train a dictionary and try it out. You can use zstd's built in + * benchmarking tool to test the dictionary effectiveness. + * + * # Benchmark levels 1-3 without a dictionary + * zstd -b1e3 -r /path/to/my/files + * # Benchmark levels 1-3 with a dictioanry + * zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary + * + * When should I retrain a dictionary? + * ----------------------------------- + * + * You should retrain a dictionary when its effectiveness drops. Dictionary + * effectiveness drops as the data you are compressing changes. Generally, we do + * expect dictionaries to "decay" over time, as your data changes, but the rate + * at which they decay depends on your use case. Internally, we regularly + * retrain dictionaries, and if the new dictionary performs significantly + * better than the old dictionary, we will ship the new dictionary. + * + * I have a raw content dictionary, how do I turn it into a zstd dictionary? + * ------------------------------------------------------------------------- + * + * If you have a raw content dictionary, e.g. by manually constructing it, or + * using a third-party dictionary builder, you can turn it into a zstd + * dictionary by using `ZDICT_finalizeDictionary()`. You'll also have to + * provide some samples of the data. It will add the zstd header to the + * raw content, which contains a dictionary ID and entropy tables, which + * will improve compression ratio, and allow zstd to write the dictionary ID + * into the frame, if you so choose. + * + * Do I have to use zstd's dictionary builder? + * ------------------------------------------- + * + * No! You can construct dictionary content however you please, it is just + * bytes. It will always be valid as a raw content dictionary. If you want + * a zstd dictionary, which can improve compression ratio, use + * `ZDICT_finalizeDictionary()`. + * + * What is the attack surface of a zstd dictionary? + * ------------------------------------------------ + * + * Zstd is heavily fuzz tested, including loading fuzzed dictionaries, so + * zstd should never crash, or access out-of-bounds memory no matter what + * the dictionary is. However, if an attacker can control the dictionary + * during decompression, they can cause zstd to generate arbitrary bytes, + * just like if they controlled the compressed data. + * + ******************************************************************************/ + /*! ZDICT_trainFromBuffer(): * Train a dictionary from an array of samples. @@ -64,7 +203,14 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap typedef struct { int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */ unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ - unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) */ + unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) + * NOTE: The zstd format reserves some dictionary IDs for future use. + * You may use them in private settings, but be warned that they + * may be used by zstd in a public dictionary registry in the future. + * These dictionary IDs are: + * - low range : <= 32767 + * - high range : >= (2^31) + */ } ZDICT_params_t; /*! ZDICT_finalizeDictionary(): diff --git a/lib/zstd.h b/lib/zstd.h index 8f0ba268d..fbc4a98d4 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -449,7 +449,8 @@ typedef enum { ZSTD_c_experimentalParam11=1008, ZSTD_c_experimentalParam12=1009, ZSTD_c_experimentalParam13=1010, - ZSTD_c_experimentalParam14=1011 + ZSTD_c_experimentalParam14=1011, + ZSTD_c_experimentalParam15=1012 } ZSTD_cParameter; typedef struct { @@ -1085,6 +1086,28 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY +/* Deprecation warnings : + * Should these warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. + */ +#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS +# define ZSTD_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZSTD_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) +# define ZSTD_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZSTD_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZSTD_DEPRECATED(message) ZSTDLIB_API __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") +# define ZSTD_DEPRECATED(message) ZSTDLIB_API +# endif +#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ + /**************************************************************************************** * experimental API (static linking only) **************************************************************************************** @@ -1620,18 +1643,20 @@ ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParame /*! ZSTD_compress_advanced() : * Note : this function is now DEPRECATED. * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2") +size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict,size_t dictSize, ZSTD_parameters params); /*! ZSTD_compress_usingCDict_advanced() : - * Note : this function is now REDUNDANT. + * Note : this function is now DEPRECATED. * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning in some future version */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict, @@ -1859,6 +1884,26 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre */ #define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14 +/* ZSTD_c_deterministicRefPrefix + * Default is 0 == disabled. Set to 1 to enable. + * + * Zstd produces different results for prefix compression when the prefix is + * directly adjacent to the data about to be compressed vs. when it isn't. + * This is because zstd detects that the two buffers are contiguous and it can + * use a more efficient match finding algorithm. However, this produces different + * results than when the two buffers are non-contiguous. This flag forces zstd + * to always load the prefix in non-contiguous mode, even if it happens to be + * adjacent to the data, to guarantee determinism. + * + * If you really care about determinism when using a dictionary or prefix, + * like when doing delta compression, you should select this option. It comes + * at a speed penalty of about ~2.5% if the dictionary and data happened to be + * contiguous, and is free if they weren't contiguous. We don't expect that + * intentionally making the dictionary and data contiguous will be worth the + * cost to memcpy() the data. + */ +#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 + /*! ZSTD_CCtx_getParameter() : * Get the requested compression parameter value, selected by enum ZSTD_cParameter, * and store it into int* value. @@ -2070,11 +2115,13 @@ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param /*! ZSTD_DCtx_setFormat() : + * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). * Instruct the decoder context about what kind of data to decode next. * This instruction is mandatory to decode data without a fully-formed header, * such ZSTD_f_zstd1_magicless for example. * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); +ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); /*! ZSTD_decompressStream_simpleArgs() : * Same as ZSTD_decompressStream(), @@ -2098,7 +2145,7 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( /*===== Advanced Streaming compression functions =====*/ /*! ZSTD_initCStream_srcSize() : - * This function is deprecated, and equivalent to: + * This function is DEPRECATED, and equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); @@ -2107,15 +2154,15 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( * pledgedSrcSize must be correct. If it is not known at init time, use * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, * "0" also disables frame content size field. It may be enabled in the future. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /*! ZSTD_initCStream_usingDict() : - * This function is deprecated, and is equivalent to: + * This function is DEPRECATED, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); @@ -2124,15 +2171,15 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, * dict == NULL or dictSize < 8, in which case no dict is used. * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /*! ZSTD_initCStream_advanced() : - * This function is deprecated, and is approximately equivalent to: + * This function is DEPRECATED, and is approximately equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * // Pseudocode: Set each zstd parameter and leave the rest as-is. * for ((param, value) : params) { @@ -2144,23 +2191,24 @@ ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. * pledgedSrcSize must be correct. * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_advanced(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /*! ZSTD_initCStream_usingCDict() : - * This function is deprecated, and equivalent to: + * This function is DEPRECATED, and equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_refCDict(zcs, cdict); - * + * * note : cdict will just be referenced, and must outlive compression session - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /*! ZSTD_initCStream_usingCDict_advanced() : * This function is DEPRECATED, and is approximately equivalent to: @@ -2175,18 +2223,21 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDi * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. * pledgedSrcSize must be correct. If srcSize is not known at init time, use * value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /*! ZSTD_resetCStream() : - * This function is deprecated, and is equivalent to: + * This function is DEPRECATED, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + * explicitly specified. * * start a new frame, using same parameters from previous frame. * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. @@ -2196,9 +2247,10 @@ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. * @return : 0, or an error code (which can be tested using ZSTD_isError()) - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); typedef struct { @@ -2285,8 +2337,7 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); ZSTD_CCtx object can be re-used multiple times within successive compression operations. Start by initializing a context. - Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, - or ZSTD_compressBegin_advanced(), for finer parameter control. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() Then, consume your input using ZSTD_compressContinue(). @@ -2311,15 +2362,17 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /*===== Buffer-less streaming compression functions =====*/ ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - +/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ /** Buffer-less streaming decompression (synchronous mode) diff --git a/programs/benchzstd.c b/programs/benchzstd.c index ccf705d29..49c03490d 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -67,18 +67,10 @@ static const size_t maxMemory = (sizeof(size_t)==4) ? /* ************************************* * console display ***************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush(NULL); } #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ -static const U64 g_refreshRate = SEC_TO_MICRO / 6; -static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; - -#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \ - if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \ - { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ - if (displayLevel>=4) fflush(stderr); } } } - /* ************************************* * Exceptions @@ -768,7 +760,7 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize, } { FILE* const f = fopen(fileNamesTable[n], "rb"); if (f==NULL) RETURN_ERROR_INT(10, "impossible to open file %s", fileNamesTable[n]); - DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); + DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]); if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); if (readSize != (size_t)fileSize) RETURN_ERROR_INT(11, "could not read %s", fileNamesTable[n]); diff --git a/programs/fileio.c b/programs/fileio.c index 604084cc7..ec9390839 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -25,9 +25,10 @@ ***************************************/ #include "platform.h" /* Large Files support, SET_BINARY_MODE */ #include "util.h" /* UTIL_getFileSize, UTIL_isRegularFile, UTIL_isSameFile */ -#include /* fprintf, fopen, fread, _fileno, stdin, stdout */ +#include /* fprintf, open, fdopen, fread, _fileno, stdin, stdout */ #include /* malloc, free */ #include /* strcmp, strlen */ +#include /* O_WRONLY */ #include #include /* errno */ #include /* INT_MAX */ @@ -73,6 +74,14 @@ #define FNSPACE 30 +/* Default file permissions 0666 (modulated by umask) */ +#if !defined(_WIN32) +/* These macros aren't defined on windows. */ +#define DEFAULT_FILE_PERMISSIONS (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) +#else +#define DEFAULT_FILE_PERMISSIONS (0666) +#endif + /*-************************************* * Macros ***************************************/ @@ -84,10 +93,10 @@ struct FIO_display_prefs_s { int displayLevel; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */ - U32 noProgress; + FIO_progressSetting_e progressSetting; }; -static FIO_display_prefs_t g_display_prefs = {2, 0}; +static FIO_display_prefs_t g_display_prefs = {2, FIO_ps_auto}; #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) @@ -96,10 +105,10 @@ static FIO_display_prefs_t g_display_prefs = {2, 0}; static const U64 g_refreshRate = SEC_TO_MICRO / 6; static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; -#define READY_FOR_UPDATE() (!g_display_prefs.noProgress && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) +#define READY_FOR_UPDATE() ((g_display_prefs.progressSetting != FIO_ps_never) && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) #define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); } #define DISPLAYUPDATE(l, ...) { \ - if (g_display_prefs.displayLevel>=l && !g_display_prefs.noProgress) { \ + if (g_display_prefs.displayLevel>=l && (g_display_prefs.progressSetting != FIO_ps_never)) { \ if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \ DELAY_NEXT_UPDATE(); \ DISPLAY(__VA_ARGS__); \ @@ -324,6 +333,7 @@ struct FIO_prefs_s { int excludeCompressedFiles; int patchFromMode; int contentSize; + int allowBlockDevices; }; /*-************************************* @@ -384,6 +394,7 @@ FIO_prefs_t* FIO_createPreferences(void) ret->testMode = 0; ret->literalCompressionMode = ZSTD_lcm_auto; ret->excludeCompressedFiles = 0; + ret->allowBlockDevices = 0; return ret; } @@ -419,7 +430,7 @@ void FIO_freeContext(FIO_ctx_t* const fCtx) void FIO_setNotificationLevel(int level) { g_display_prefs.displayLevel=level; } -void FIO_setNoProgress(unsigned noProgress) { g_display_prefs.noProgress = noProgress; } +void FIO_setProgressSetting(FIO_progressSetting_e setting) { g_display_prefs.progressSetting = setting; } /*-************************************* @@ -451,6 +462,8 @@ void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) { void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->excludeCompressedFiles = excludeCompressedFiles; } +void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockDevices = allowBlockDevices; } + void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) { if (blockSize && prefs->nbWorkers==0) DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); @@ -593,11 +606,12 @@ static int FIO_removeFile(const char* path) } /** FIO_openSrcFile() : - * condition : `srcFileName` must be non-NULL. + * condition : `srcFileName` must be non-NULL. `prefs` may be NULL. * @result : FILE* to `srcFileName`, or NULL if it fails */ -static FILE* FIO_openSrcFile(const char* srcFileName) +static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName) { stat_t statbuf; + int allowBlockDevices = prefs != NULL ? prefs->allowBlockDevices : 0; assert(srcFileName != NULL); if (!strcmp (srcFileName, stdinmark)) { DISPLAYLEVEL(4,"Using stdin for input \n"); @@ -613,6 +627,7 @@ static FILE* FIO_openSrcFile(const char* srcFileName) if (!UTIL_isRegularFileStat(&statbuf) && !UTIL_isFIFOStat(&statbuf) + && !(allowBlockDevices && UTIL_isBlockDevStat(&statbuf)) ) { DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName); @@ -631,7 +646,8 @@ static FILE* FIO_openSrcFile(const char* srcFileName) * @result : FILE* to `dstFileName`, or NULL if it fails */ static FILE* FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, - const char* srcFileName, const char* dstFileName) + const char* srcFileName, const char* dstFileName, + const int mode) { if (prefs->testMode) return NULL; /* do not open file in test mode */ @@ -658,7 +674,6 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, if (UTIL_isRegularFile(dstFileName)) { /* Check if destination file already exists */ - FILE* const fCheck = fopen( dstFileName, "rb" ); #if !defined(_WIN32) /* this test does not work on Windows : * `NUL` and `nul` are detected as regular files */ @@ -667,26 +682,39 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, dstFileName); } #endif - if (fCheck != NULL) { /* dst file exists, authorization prompt */ - fclose(fCheck); - if (!prefs->overwrite) { - if (g_display_prefs.displayLevel <= 1) { - /* No interaction possible */ - DISPLAY("zstd: %s already exists; not overwritten \n", - dstFileName); - return NULL; - } - DISPLAY("zstd: %s already exists; ", dstFileName); - if (UTIL_requireUserConfirmation("overwrite (y/n) ? ", "Not overwritten \n", "yY", fCtx->hasStdinInput)) - return NULL; + if (!prefs->overwrite) { + if (g_display_prefs.displayLevel <= 1) { + /* No interaction possible */ + DISPLAY("zstd: %s already exists; not overwritten \n", + dstFileName); + return NULL; } - /* need to unlink */ - FIO_removeFile(dstFileName); - } } + DISPLAY("zstd: %s already exists; ", dstFileName); + if (UTIL_requireUserConfirmation("overwrite (y/n) ? ", "Not overwritten \n", "yY", fCtx->hasStdinInput)) + return NULL; + } + /* need to unlink */ + FIO_removeFile(dstFileName); + } - { const int old_umask = UTIL_umask(0177); /* u-x,go-rwx */ - FILE* const f = fopen( dstFileName, "wb" ); - UTIL_umask(old_umask); + { +#if defined(_WIN32) + /* Windows requires opening the file as a "binary" file to avoid + * mangling. This macro doesn't exist on unix. */ + const int openflags = O_WRONLY|O_CREAT|O_TRUNC|O_BINARY; + const int fd = _open(dstFileName, openflags, mode); + FILE* f = NULL; + if (fd != -1) { + f = _fdopen(fd, "wb"); + } +#else + const int openflags = O_WRONLY|O_CREAT|O_TRUNC; + const int fd = open(dstFileName, openflags, mode); + FILE* f = NULL; + if (fd != -1) { + f = fdopen(fd, "wb"); + } +#endif if (f == NULL) { DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); } @@ -1376,24 +1404,25 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, (unsigned)(zfp.consumed >> 20), (unsigned)(zfp.produced >> 20), cShare ); - } else { /* summarized notifications if == 2 */ - DISPLAYLEVEL(2, "\r%79s\r", ""); /* Clear out the current displayed line */ + } else if (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always) { + /* Require level 2 or forcibly displayed progress counter for summarized updates */ + DISPLAYLEVEL(1, "\r%79s\r", ""); /* Clear out the current displayed line */ if (fCtx->nbFilesTotal > 1) { size_t srcFileNameSize = strlen(srcFileName); /* Ensure that the string we print is roughly the same size each time */ if (srcFileNameSize > 18) { const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15; - DISPLAYLEVEL(2, "Compress: %u/%u files. Current: ...%s ", - fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName); + DISPLAYLEVEL(1, "Compress: %u/%u files. Current: ...%s ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName); } else { - DISPLAYLEVEL(2, "Compress: %u/%u files. Current: %*s ", - fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName); + DISPLAYLEVEL(1, "Compress: %u/%u files. Current: %*s ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName); } } - DISPLAYLEVEL(2, "Read : %2u ", (unsigned)(zfp.consumed >> 20)); + DISPLAYLEVEL(1, "Read : %2u ", (unsigned)(zfp.consumed >> 20)); if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAYLEVEL(2, "/ %2u ", (unsigned)(fileSize >> 20)); - DISPLAYLEVEL(2, "MB ==> %2.f%%", cShare); + DISPLAYLEVEL(1, "MB ==> %2.f%%", cShare); DELAY_NEXT_UPDATE(); } @@ -1609,23 +1638,24 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, int closeDstFile = 0; int result; stat_t statbuf; - int transfer_permissions = 0; assert(ress.srcFile != NULL); if (ress.dstFile == NULL) { + int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; + if ( strcmp (srcFileName, stdinmark) + && UTIL_stat(srcFileName, &statbuf) + && UTIL_isRegularFileStat(&statbuf) ) { + dstFilePermissions = statbuf.st_mode; + } + closeDstFile = 1; DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName); - ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName); + ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ /* Must only be added after FIO_openDstFile() succeeds. * Otherwise we may delete the destination file if it already exists, * and the user presses Ctrl-C when asked if they wish to overwrite. */ addHandler(dstFileName); - - if ( strcmp (srcFileName, stdinmark) - && UTIL_stat(srcFileName, &statbuf) - && UTIL_isRegularFileStat(&statbuf) ) - transfer_permissions = 1; } result = FIO_compressFilename_internal(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); @@ -1645,11 +1675,6 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ ) { FIO_removeFile(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */ - } else if (transfer_permissions) { - DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: transferring permissions into dst: %s \n", dstFileName); - UTIL_setFileStat(dstFileName, &statbuf); - } else { - DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: do not transfer permissions into dst: %s \n", dstFileName); } } @@ -1708,7 +1733,7 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx, return 0; } - ress.srcFile = FIO_openSrcFile(srcFileName); + ress.srcFile = FIO_openSrcFile(prefs, srcFileName); if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ result = FIO_compressFilename_dstFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); @@ -1821,7 +1846,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, FIO_freeCResources(&ress); return 1; } - ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName); + ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); if (ress.dstFile == NULL) { /* could not open outFileName */ error = 1; } else { @@ -2140,7 +2165,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, /* Write block */ storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips); frameSize += outBuff.pos; - if (!fCtx->hasStdoutOutput) { + if (!fCtx->hasStdoutOutput || g_display_prefs.progressSetting == FIO_ps_always) { if (fCtx->nbFilesTotal > 1) { size_t srcFileNameSize = strlen(srcFileName); if (srcFileNameSize > 18) { @@ -2511,13 +2536,19 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, { int result; stat_t statbuf; - int transfer_permissions = 0; int releaseDstFile = 0; if ((ress.dstFile == NULL) && (prefs->testMode==0)) { + int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; + if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */ + && UTIL_stat(srcFileName, &statbuf) + && UTIL_isRegularFileStat(&statbuf) ) { + dstFilePermissions = statbuf.st_mode; + } + releaseDstFile = 1; - ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName); + ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); if (ress.dstFile==NULL) return 1; /* Must only be added after FIO_openDstFile() succeeds. @@ -2525,11 +2556,6 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, * and the user presses Ctrl-C when asked if they wish to overwrite. */ addHandler(dstFileName); - - if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */ - && UTIL_stat(srcFileName, &statbuf) - && UTIL_isRegularFileStat(&statbuf) ) - transfer_permissions = 1; } result = FIO_decompressFrames(fCtx, ress, srcFile, prefs, dstFileName, srcFileName); @@ -2547,8 +2573,6 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ ) { FIO_removeFile(dstFileName); /* remove decompression artefact; note: don't do anything special if remove() fails */ - } else if ( transfer_permissions /* file permissions correctly extracted from src */ ) { - UTIL_setFileStat(dstFileName, &statbuf); /* transfer file permissions from src into dst */ } } @@ -2571,7 +2595,7 @@ static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs return 1; } - srcFile = FIO_openSrcFile(srcFileName); + srcFile = FIO_openSrcFile(prefs, srcFileName); if (srcFile==NULL) return 1; ress.srcBufferLoaded = 0; @@ -2750,7 +2774,7 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, return 1; } if (!prefs->testMode) { - ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName); + ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); } for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { @@ -2921,7 +2945,7 @@ static InfoError getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName) { InfoError status; - FILE* const srcFile = FIO_openSrcFile(inFileName); + FILE* const srcFile = FIO_openSrcFile(NULL, inFileName); ERROR_IF(srcFile == NULL, info_file_error, "Error: could not open source file %s", inFileName); info->compressedSize = UTIL_getFileSize(inFileName); diff --git a/programs/fileio.h b/programs/fileio.h index 1f5db4b52..9d97ec8bf 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -68,6 +68,8 @@ void FIO_freeContext(FIO_ctx_t* const fCtx); typedef struct FIO_display_prefs_s FIO_display_prefs_t; +typedef enum { FIO_ps_auto, FIO_ps_never, FIO_ps_always } FIO_progressSetting_e; + /*-************************************* * Parameters ***************************************/ @@ -100,9 +102,10 @@ void FIO_setLiteralCompressionMode( FIO_prefs_t* const prefs, ZSTD_literalCompressionMode_e mode); -void FIO_setNoProgress(unsigned noProgress); +void FIO_setProgressSetting(FIO_progressSetting_e progressSetting); void FIO_setNotificationLevel(int level); void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles); +void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices); void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value); void FIO_setContentSize(FIO_prefs_t* const prefs, int value); diff --git a/programs/platform.h b/programs/platform.h index 3b8b505e1..b858e3b48 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -22,6 +22,7 @@ extern "C" { ****************************************/ #if defined(_MSC_VER) # define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ +# define _CRT_NONSTDC_NO_WARNINGS /* Disable C4996 complaining about posix function names */ # if (_MSC_VER <= 1800) /* 1800 == Visual Studio 2013 */ # define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before and */ # define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ diff --git a/programs/util.c b/programs/util.c index 0f857019a..8d190c62c 100644 --- a/programs/util.c +++ b/programs/util.c @@ -159,15 +159,6 @@ int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions) return chmod(filename, permissions); } -int UTIL_umask(int mode) { -#if PLATFORM_POSIX_VERSION > 0 - return umask(mode); -#else - /* do nothing, fake return value */ - return mode; -#endif -} - int UTIL_setFileStat(const char *filename, const stat_t *statbuf) { int res = 0; @@ -269,6 +260,17 @@ int UTIL_isFIFOStat(const stat_t* statbuf) return 0; } +/* UTIL_isBlockDevStat : distinguish named pipes */ +int UTIL_isBlockDevStat(const stat_t* statbuf) +{ +/* macro guards, as defined in : https://linux.die.net/man/2/lstat */ +#if PLATFORM_POSIX_VERSION >= 200112L + if (S_ISBLK(statbuf->st_mode)) return 1; +#endif + (void)statbuf; + return 0; +} + int UTIL_isLink(const char* infilename) { /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ @@ -321,9 +323,7 @@ U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles) static size_t readLineFromFile(char* buf, size_t len, FILE* file) { assert(!feof(file)); - /* Work around Cygwin problem when len == 1 it returns NULL. */ - if (len <= 1) return 0; - CONTROL( fgets(buf, (int) len, file) ); + if ( fgets(buf, (int) len, file) == NULL ) return 0; { size_t linelen = strlen(buf); if (strlen(buf)==0) return 0; if (buf[linelen-1] == '\n') linelen--; diff --git a/programs/util.h b/programs/util.h index af3778a2f..24cce4480 100644 --- a/programs/util.h +++ b/programs/util.h @@ -22,7 +22,7 @@ extern "C" { #include "platform.h" /* PLATFORM_POSIX_VERSION, ZSTD_NANOSLEEP_SUPPORT, ZSTD_SETPRIORITY_SUPPORT */ #include /* size_t, ptrdiff_t */ #include /* stat, utime */ -#include /* stat, chmod, umask */ +#include /* stat, chmod */ #include "../lib/common/mem.h" /* U64 */ @@ -143,6 +143,7 @@ int UTIL_setFileStat(const char* filename, const stat_t* statbuf); int UTIL_isRegularFileStat(const stat_t* statbuf); int UTIL_isDirectoryStat(const stat_t* statbuf); int UTIL_isFIFOStat(const stat_t* statbuf); +int UTIL_isBlockDevStat(const stat_t* statbuf); U64 UTIL_getFileSizeStat(const stat_t* statbuf); /** @@ -152,11 +153,6 @@ U64 UTIL_getFileSizeStat(const stat_t* statbuf); */ int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions); -/** - * Wraps umask(). Does nothing when the platform doesn't have that concept. - */ -int UTIL_umask(int mode); - /* * In the absence of a pre-existing stat result on the file in question, these * functions will do a stat() call internally and then use that result to diff --git a/programs/zstd.1 b/programs/zstd.1 index b68679893..563f3feb6 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -1,3 +1,5 @@ +.\" generated with Ronn/v0.7.3 +.\" http://github.com/rtomayko/ronn/tree/0.7.3 . .TH "ZSTD" "1" "May 2021" "zstd 1.4.10" "User Commands" . @@ -156,7 +158,7 @@ This is also used during compression when using with \-\-patch\-from=\. In this \fB\-o FILE\fR: save result into \fBFILE\fR . .IP "\(bu" 4 -\fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, etc\. +\fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, block devices, etc\. . .IP "\(bu" 4 \fB\-c\fR, \fB\-\-stdout\fR: force write to standard output, even if it is the console diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 1220a8140..9acea1b1d 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -202,7 +202,7 @@ the last one takes effect. save result into `FILE` * `-f`, `--force`: disable input and output checks. Allows overwriting existing files, input - from console, output to stdout, operating on links, etc. + from console, output to stdout, operating on links, block devices, etc. * `-c`, `--stdout`: force write to standard output, even if it is the console * `--[no-]sparse`: diff --git a/programs/zstdcli.c b/programs/zstdcli.c index d9d2c701e..239aaf406 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -148,7 +148,8 @@ static void usage(FILE* f, const char* programName) DISPLAY_F(f, " -D DICT: use DICT as Dictionary for compression or decompression \n"); DISPLAY_F(f, " -o file: result stored into `file` (only 1 output file) \n"); DISPLAY_F(f, " -f : disable input and output checks. Allows overwriting existing files,\n"); - DISPLAY_F(f, " input from console, output to stdout, operating on links, etc.\n"); + DISPLAY_F(f, " input from console, output to stdout, operating on links,\n"); + DISPLAY_F(f, " block devices, etc.\n"); DISPLAY_F(f, "--rm : remove source file(s) after successful de/compression \n"); DISPLAY_F(f, " -k : preserve source file(s) (default) \n"); DISPLAY_F(f, " -h/-H : display help/long help and exit \n"); @@ -166,7 +167,8 @@ static void usage_advanced(const char* programName) DISPLAYOUT( " -v : verbose mode; specify multiple times to increase verbosity \n"); DISPLAYOUT( " -q : suppress warnings; specify twice to suppress errors too \n"); - DISPLAYOUT( "--no-progress : do not display the progress counter \n"); + DISPLAYOUT( "--[no-]progress : forcibly display, or never display the progress counter.\n"); + DISPLAYOUT( " note: any (de)compressed output to terminal will mix with progress counter text. \n"); #ifdef UTIL_HAS_CREATEFILELIST DISPLAYOUT( " -r : operate recursively on directories \n"); @@ -724,6 +726,7 @@ int main(int const argCount, const char* argv[]) { int argNb, followLinks = 0, + allowBlockDevices = 0, forceStdin = 0, forceStdout = 0, hasStdout = 0, @@ -838,7 +841,7 @@ int main(int const argCount, const char* argv[]) if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; } if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; } if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; } - if (!strcmp(argument, "--force")) { FIO_overwriteMode(prefs); forceStdin=1; forceStdout=1; followLinks=1; continue; } + if (!strcmp(argument, "--force")) { FIO_overwriteMode(prefs); forceStdin=1; forceStdout=1; followLinks=1; allowBlockDevices=1; continue; } if (!strcmp(argument, "--version")) { printVersion(); CLEAN_RETURN(0); } if (!strcmp(argument, "--help")) { usage_advanced(programName); CLEAN_RETURN(0); } if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; } @@ -877,7 +880,8 @@ int main(int const argCount, const char* argv[]) if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; } if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; } if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; } - if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; } + if (!strcmp(argument, "--no-progress")) { FIO_setProgressSetting(FIO_ps_never); continue; } + if (!strcmp(argument, "--progress")) { FIO_setProgressSetting(FIO_ps_always); continue; } if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; } /* long commands with arguments */ @@ -1024,7 +1028,7 @@ int main(int const argCount, const char* argv[]) case 'D': argument++; NEXT_FIELD(dictFileName); break; /* Overwrite */ - case 'f': FIO_overwriteMode(prefs); forceStdin=1; forceStdout=1; followLinks=1; argument++; break; + case 'f': FIO_overwriteMode(prefs); forceStdin=1; forceStdout=1; followLinks=1; allowBlockDevices=1; argument++; break; /* Verbose mode */ case 'v': g_displayLevel++; argument++; break; @@ -1331,6 +1335,7 @@ int main(int const argCount, const char* argv[]) FIO_setNbFilesTotal(fCtx, (int)filenames->tableSize); FIO_determineHasStdinInput(fCtx, filenames); FIO_setNotificationLevel(g_displayLevel); + FIO_setAllowBlockDevices(prefs, allowBlockDevices); FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL); if (memLimit == 0) { if (compressionParams.windowLog == 0) { diff --git a/tests/Makefile b/tests/Makefile index c4d1472d0..85553007d 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -24,7 +24,7 @@ PRGDIR = ../programs PYTHON ?= python3 TESTARTEFACT := versionsTest -DEBUGLEVEL ?= 1 +DEBUGLEVEL ?= 2 export DEBUGLEVEL # transmit value to sub-makefiles DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL) CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ @@ -38,7 +38,7 @@ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ - -Wredundant-decls -Wmissing-prototypes + -Wredundant-decls -Wmissing-prototypes -Wno-deprecated-declarations CFLAGS += $(DEBUGFLAGS) CPPFLAGS += $(MOREFLAGS) @@ -131,7 +131,7 @@ zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ fullbench32: CPPFLAGS += -m32 -fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) +fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD) fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements fullbench fullbench32 : $(ZSTD_FILES) @@ -147,7 +147,7 @@ fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR # $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll $(LINK.c) $^ $(LDLIBS) -o $@$(EXT) -fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) +fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations fuzzer : LDFLAGS += $(MULTITHREAD_LD) fuzzer : $(ZSTDMT_OBJECTS) fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c diff --git a/tests/README.md b/tests/README.md index 1e40c46aa..cd255e904 100644 --- a/tests/README.md +++ b/tests/README.md @@ -8,7 +8,6 @@ This directory contains the following programs and scripts: - `paramgrill` : parameter tester for zstd - `test-zstd-speed.py` : script for testing zstd speed difference between commits - `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+) -- `zbufftest` : Test tool to check ZBUFF (a buffered streaming API) integrity - `zstreamtest` : Fuzzer test tool for zstd streaming API - `legacy` : Test tool to test decoding of legacy zstd frames - `decodecorpus` : Tool to generate valid Zstandard frames, for verifying decoder implementations diff --git a/tests/fuzz/.gitignore b/tests/fuzz/.gitignore index 8ef3a3efd..93d935a85 100644 --- a/tests/fuzz/.gitignore +++ b/tests/fuzz/.gitignore @@ -16,6 +16,7 @@ zstd_frame_info decompress_dstSize_tooSmall fse_read_ncount sequence_compression_api +seekable_roundtrip fuzz-*.log rt_lib_* d_lib_* diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile index 24b9f3468..ccb574b79 100644 --- a/tests/fuzz/Makefile +++ b/tests/fuzz/Makefile @@ -25,10 +25,11 @@ CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corp ZSTDDIR = ../../lib PRGDIR = ../../programs +CONTRIBDIR = ../../contrib FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(ZSTDDIR)/legacy \ - -I$(PRGDIR) -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS) + -I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS) FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef \ @@ -46,6 +47,9 @@ FUZZ_ROUND_TRIP_FLAGS := -DFUZZING_ASSERT_VALID_SEQUENCE FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h fuzz_data_producer.h FUZZ_SRC := $(PRGDIR)/util.c ./fuzz_helpers.c ./zstd_helpers.c ./fuzz_data_producer.c +SEEKABLE_HEADERS = $(CONTRIBDIR)/seekable_format/zstd_seekable.h +SEEKABLE_OBJS = $(CONTRIBDIR)/seekable_format/zstdseek_compress.c $(CONTRIBDIR)/seekable_format/zstdseek_decompress.c + ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c ZSTDCOMP_SRC := $(ZSTDDIR)/compress/*.c ZSTDDECOMP_SRC := $(ZSTDDIR)/decompress/*.c @@ -98,7 +102,8 @@ FUZZ_TARGETS := \ dictionary_stream_round_trip \ decompress_dstSize_tooSmall \ fse_read_ncount \ - sequence_compression_api + sequence_compression_api \ + seekable_roundtrip all: libregression.a $(FUZZ_TARGETS) @@ -192,6 +197,9 @@ fse_read_ncount: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_fse_read_ncount. sequence_compression_api: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_sequence_compression_api.o $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_sequence_compression_api.o $(LIB_FUZZING_ENGINE) -o $@ +seekable_roundtrip: $(FUZZ_HEADERS) $(SEEKABLE_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) $(SEEKABLE_OBJS) rt_fuzz_seekable_roundtrip.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) $(SEEKABLE_OBJS) rt_fuzz_seekable_roundtrip.o $(LIB_FUZZING_ENGINE) -o $@ + libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c d_fuzz_regression_driver.o $(AR) $(FUZZ_ARFLAGS) $@ d_fuzz_regression_driver.o diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py index a3431f365..d8dfa7782 100755 --- a/tests/fuzz/fuzz.py +++ b/tests/fuzz/fuzz.py @@ -62,6 +62,7 @@ TARGET_INFO = { 'decompress_dstSize_tooSmall': TargetInfo(InputType.RAW_DATA), 'fse_read_ncount': TargetInfo(InputType.RAW_DATA), 'sequence_compression_api': TargetInfo(InputType.RAW_DATA), + 'seekable_roundtrip': TargetInfo(InputType.RAW_DATA), } TARGETS = list(TARGET_INFO.keys()) ALL_TARGETS = TARGETS + ['all'] diff --git a/tests/fuzz/seekable_roundtrip.c b/tests/fuzz/seekable_roundtrip.c new file mode 100644 index 000000000..dcdcaae11 --- /dev/null +++ b/tests/fuzz/seekable_roundtrip.c @@ -0,0 +1,88 @@ +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd.h" +#include "zstd_seekable.h" +#include "fuzz_helpers.h" +#include "fuzz_data_producer.h" + +static ZSTD_seekable *stream = NULL; +static ZSTD_seekable_CStream *zscs = NULL; +static const size_t kSeekableOverheadSize = ZSTD_seekTableFooterSize; + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + size_t const compressedBufferSize = ZSTD_compressBound(size) + kSeekableOverheadSize; + uint8_t* compressedBuffer = (uint8_t*)malloc(compressedBufferSize); + uint8_t* decompressedBuffer = (uint8_t*)malloc(size); + + int const cLevel = FUZZ_dataProducer_int32Range(producer, ZSTD_minCLevel(), ZSTD_maxCLevel()); + unsigned const checksumFlag = FUZZ_dataProducer_int32Range(producer, 0, 1); + size_t const uncompressedSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size_t const offset = FUZZ_dataProducer_uint32Range(producer, 0, size - uncompressedSize); + size_t seekSize; + + if (!zscs) { + zscs = ZSTD_seekable_createCStream(); + FUZZ_ASSERT(zscs); + } + if (!stream) { + stream = ZSTD_seekable_create(); + FUZZ_ASSERT(stream); + } + + { /* Perform a compression */ + size_t const initStatus = ZSTD_seekable_initCStream(zscs, cLevel, checksumFlag, size); + size_t endStatus; + ZSTD_outBuffer out = { .dst=compressedBuffer, .pos=0, .size=compressedBufferSize }; + ZSTD_inBuffer in = { .src=src, .pos=0, .size=size }; + FUZZ_ASSERT(!ZSTD_isError(initStatus)); + + do { + size_t cSize = ZSTD_seekable_compressStream(zscs, &out, &in); + FUZZ_ASSERT(!ZSTD_isError(cSize)); + } while (in.pos != in.size); + + FUZZ_ASSERT(in.pos == in.size); + endStatus = ZSTD_seekable_endStream(zscs, &out); + FUZZ_ASSERT(!ZSTD_isError(endStatus)); + seekSize = out.pos; + } + + { /* Decompress at an offset */ + size_t const initStatus = ZSTD_seekable_initBuff(stream, compressedBuffer, seekSize); + size_t decompressedBytesTotal = 0; + size_t dSize; + + FUZZ_ZASSERT(initStatus); + do { + dSize = ZSTD_seekable_decompress(stream, decompressedBuffer, uncompressedSize, offset); + FUZZ_ASSERT(!ZSTD_isError(dSize)); + decompressedBytesTotal += dSize; + } while (decompressedBytesTotal < uncompressedSize && dSize > 0); + FUZZ_ASSERT(decompressedBytesTotal == uncompressedSize); + } + + FUZZ_ASSERT_MSG(!FUZZ_memcmp(src+offset, decompressedBuffer, uncompressedSize), "Corruption!"); + + free(decompressedBuffer); + free(compressedBuffer); + FUZZ_dataProducer_free(producer); + +#ifndef STATEFUL_FUZZING + ZSTD_seekable_free(stream); stream = NULL; + ZSTD_seekable_freeCStream(zscs); zscs = NULL; +#endif + return 0; +} diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c index 31bf4c20d..4d889deeb 100644 --- a/tests/fuzz/zstd_helpers.c +++ b/tests/fuzz/zstd_helpers.c @@ -97,6 +97,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, producer); setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, producer); setRand(cctx, ZSTD_c_splitBlocks, 0, 1, producer); + setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer); if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) { setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer); } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 2c6c60d38..46147a24b 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -766,6 +766,50 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : testing dict compression for determinism : ", testNb++); + { + size_t const testSize = 1024; + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + char* dict = (char*)malloc(2 * testSize); + int ldmEnabled, level; + + RDG_genBuffer(dict, testSize, 0.5, 0.5, seed); + RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed); + memcpy(dict + testSize, CNBuffer, testSize); + for (level = 1; level <= 5; ++level) { + for (ldmEnabled = 0; ldmEnabled <= 1; ++ldmEnabled) { + size_t cSize0; + XXH64_hash_t compressedChecksum0; + + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ldmEnabled)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1)); + + CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize)); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize); + CHECK_Z(cSize); + CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, testSize, compressedBuffer, cSize, dict, testSize)); + + cSize0 = cSize; + compressedChecksum0 = XXH64(compressedBuffer, cSize, 0); + + CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize)); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize); + CHECK_Z(cSize); + + if (cSize != cSize0) goto _output_error; + if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error; + } + } + + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + free(dict); + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++); { ZSTD_CCtx* cctx = ZSTD_createCCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx(); @@ -1738,10 +1782,7 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++); { size_t const testSize = CNBuffSize / 3; - { ZSTD_parameters p = ZSTD_getParams(2, testSize, dictSize); - p.fParams.contentSizeFlag = 1; - CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) ); - } + CHECK( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) ); CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) ); CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), @@ -2562,12 +2603,8 @@ static int basicUnitTests(U32 const seed, double compressibility) int const compressionLevel = -1; assert(cctx != NULL); - { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, srcSize, 0); - size_t const cSize_1pass = ZSTD_compress_advanced(cctx, - compressedBuffer, compressedBufferSize, - CNBuffer, srcSize, - NULL, 0, - params); + { size_t const cSize_1pass = ZSTD_compress(compressedBuffer, compressedBufferSize, + CNBuffer, srcSize, compressionLevel); if (ZSTD_isError(cSize_1pass)) goto _output_error; CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); diff --git a/tests/playTests.sh b/tests/playTests.sh index bfb2e378f..fa748c0cf 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -124,6 +124,23 @@ case "$UNAME" in Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;; esac +GET_PERMS="stat -c %a" +case "$UNAME" in + Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;; +esac + +assertFilePermissions() { + STAT1=$($GET_PERMS "$1") + STAT2=$2 + [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)" +} + +assertSamePermissions() { + STAT1=$($GET_PERMS "$1") + STAT2=$($GET_PERMS "$2") + [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)" +} + DIFF="diff" case "$UNAME" in SunOS) DIFF="gdiff" ;; @@ -192,7 +209,7 @@ println "test : compress to stdout" zstd tmp -c > tmpCompressed zstd tmp --stdout > tmpCompressed # long command format println "test : compress to named file" -rm tmpCompressed +rm -f tmpCompressed zstd tmp -o tmpCompressed test -f tmpCompressed # file must be created println "test : force write, correct order" @@ -346,7 +363,7 @@ rm -f tmplog zstd tmp -f -o "$INTOVOID" 2>&1 | grep -v "Refusing to remove non-regular file" println "test : --rm on stdin" println a | zstd --rm > $INTOVOID # --rm should remain silent -rm tmp +rm -f tmp zstd -f tmp && die "tmp not present : should have failed" test ! -f tmp.zst # tmp.zst should not be created println "test : -d -f do not delete destination when source is not present" @@ -354,7 +371,7 @@ touch tmp # create destination file zstd -d -f tmp.zst && die "attempt to decompress a non existing file" test -f tmp # destination file should still be present println "test : -f do not delete destination when source is not present" -rm tmp # erase source file +rm -f tmp # erase source file touch tmp.zst # create destination file zstd -f tmp && die "attempt to compress a non existing file" test -f tmp.zst # destination file should still be present @@ -368,7 +385,7 @@ println "\n===> decompression only tests " dd bs=1048576 count=1 if=/dev/zero of=tmp zstd -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst" $DIFF -s tmp1 tmp -rm tmp* +rm -f tmp* println "\n===> compress multiple files" @@ -415,7 +432,7 @@ zstd -f tmp* test -f tmp1.zst test -f tmp2.zst test -f tmp3.zst -rm tmp1 tmp2 tmp3 +rm -f tmp1 tmp2 tmp3 println "decompress tmp* : " zstd -df ./*.zst test -f tmp1 @@ -430,7 +447,7 @@ zstd -dc tmpall* > tmpdec test -f tmpdec # should check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3)) println "compress multiple files including a missing one (notHere) : " zstd -f tmp1 notHere tmp2 && die "missing file not detected!" -rm tmp* +rm -f tmp* if [ "$isWindows" = false ] ; then @@ -445,6 +462,96 @@ if [ "$isWindows" = false ] ; then rm -rf tmp* fi +println "\n===> zstd created file permissions tests" +if [ "$isWindows" = false ] ; then + rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out # todo: remove + + ORIGINAL_UMASK=$(umask) + umask 0000 + + datagen > tmp1 + datagen > tmp2 + assertFilePermissions tmp1 666 + assertFilePermissions tmp2 666 + + println "test : copy 666 permissions in file -> file compression " + zstd -f tmp1 -o tmp1.zst + assertSamePermissions tmp1 tmp1.zst + println "test : copy 666 permissions in file -> file decompression " + zstd -f -d tmp1.zst -o tmp1.out + assertSamePermissions tmp1.zst tmp1.out + + rm -f tmp1.zst tmp1.out + + println "test : copy 400 permissions in file -> file compression (write to a read-only file) " + chmod 0400 tmp1 + assertFilePermissions tmp1 400 + zstd -f tmp1 -o tmp1.zst + assertSamePermissions tmp1 tmp1.zst + println "test : copy 400 permissions in file -> file decompression (write to a read-only file) " + zstd -f -d tmp1.zst -o tmp1 + assertSamePermissions tmp1.zst tmp1 + + rm -f tmp1.zst tmp1.out + + println "test : check created permissions from stdin input in compression " + zstd -f -o tmp1.zst < tmp1 + assertFilePermissions tmp1.zst 666 + println "test : check created permissions from stdin input in decompression " + zstd -f -d -o tmp1.out < tmp1.zst + assertFilePermissions tmp1.out 666 + + rm -f tmp1.zst tmp1.out + + println "test : check created permissions from multiple inputs in compression " + zstd -f tmp1 tmp2 -o tmp1.zst + assertFilePermissions tmp1.zst 666 + println "test : check created permissions from multiple inputs in decompression " + cp tmp1.zst tmp2.zst + zstd -f -d tmp1.zst tmp2.zst -o tmp1.out + assertFilePermissions tmp1.out 666 + + rm -f tmp1.zst tmp2.zst tmp1.out tmp2.out + + println "test : check permissions on pre-existing output file in compression " + chmod 0600 tmp1 + touch tmp1.zst + chmod 0400 tmp1.zst + zstd -f tmp1 -o tmp1.zst + assertFilePermissions tmp1.zst 600 + println "test : check permissions on pre-existing output file in decompression " + chmod 0400 tmp1.zst + touch tmp1.out + chmod 0200 tmp1.out + zstd -f -d tmp1.zst -o tmp1.out + assertFilePermissions tmp1.out 400 + + rm -f tmp1.zst tmp1.out + + umask 0666 + chmod 0666 tmp1 tmp2 + + println "test : respect umask when copying permissions in file -> file compression " + zstd -f tmp1 -o tmp1.zst + assertFilePermissions tmp1.zst 0 + println "test : respect umask when copying permissions in file -> file decompression " + chmod 0666 tmp1.zst + zstd -f -d tmp1.zst -o tmp1.out + assertFilePermissions tmp1.out 0 + + rm -f tmp1.zst tmp1.out + + println "test : respect umask when compressing from stdin input " + zstd -f -o tmp1.zst < tmp1 + assertFilePermissions tmp1.zst 0 + println "test : respect umask when decompressing from stdin input " + chmod 0666 tmp1.zst + zstd -f -d -o tmp1.out < tmp1.zst + assertFilePermissions tmp1.out 0 + + rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out + umask $ORIGINAL_UMASK +fi if [ -n "$DEVNULLRIGHTS" ] ; then # these tests requires sudo rights, which is uncommon. @@ -459,6 +566,22 @@ if [ -n "$DEVNULLRIGHTS" ] ; then ls -las $INTOVOID | grep "rw-rw-rw-" fi +if [ -n "$READFROMBLOCKDEVICE" ] ; then + # This creates a temporary block device, which is only possible on unix-y + # systems, is somewhat invasive, and requires sudo. For these reasons, you + # have to specifically ask for this test. + println "\n===> checking that zstd can read from a block device" + datagen -g65536 > tmp.img + sudo losetup -fP tmp.img + LOOP_DEV=$(losetup -a | grep 'tmp\.img' | cut -f1 -d:) + [ -z "$LOOP_DEV" ] && die "failed to get loopback device" + sudoZstd $LOOP_DEV -c > tmp.img.zst && die "should fail without -f" + sudoZstd -f $LOOP_DEV -c > tmp.img.zst + zstd -d tmp.img.zst -o tmp.img.copy + sudo losetup -d $LOOP_DEV + $DIFF -s tmp.img tmp.img.copy || die "round trip failed" + rm -f tmp.img tmp.img.zst tmp.img.copy +fi println "\n===> compress multiple files into an output directory, --output-dir-flat" println henlo > tmp1 @@ -651,16 +774,16 @@ $DIFF helloworld.tmp result.tmp ln -s helloworld.zst helloworld.link.zst $EXE_PREFIX ./zstdcat helloworld.link.zst > result.tmp $DIFF helloworld.tmp result.tmp -rm zstdcat -rm result.tmp +rm -f zstdcat +rm -f result.tmp println "testing zcat symlink" ln -sf "$ZSTD_BIN" zcat $EXE_PREFIX ./zcat helloworld.zst > result.tmp $DIFF helloworld.tmp result.tmp $EXE_PREFIX ./zcat helloworld.link.zst > result.tmp $DIFF helloworld.tmp result.tmp -rm zcat -rm ./*.tmp ./*.zstd +rm -f zcat +rm -f ./*.tmp ./*.zstd println "frame concatenation tests completed" @@ -722,7 +845,7 @@ zstd -d -v -f tmpSparseCompressed -o tmpSparseRegenerated zstd -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated ls -ls tmpSparse* # look at file size and block size on disk $DIFF tmpSparse2M tmpSparseRegenerated -rm tmpSparse* +rm -f tmpSparse* println "\n===> stream-size mode" @@ -868,7 +991,7 @@ then println "- Create dictionary with multithreading enabled" zstd --train -T0 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict fi -rm tmp* dictionary +rm -f tmp* dictionary println "\n===> fastCover dictionary builder : advanced options " @@ -910,7 +1033,7 @@ zstd -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c test -f tmpDict zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c test -f dictionary -rm tmp* dictionary +rm -f tmp* dictionary println "\n===> legacy dictionary builder " @@ -938,7 +1061,7 @@ zstd -o tmpDict --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c test -f tmpDict zstd --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c test -f dictionary -rm tmp* dictionary +rm -f tmp* dictionary println "\n===> integrity tests " @@ -1010,7 +1133,7 @@ if [ $GZIPMODE -eq 1 ]; then gzip -t -v tmp.gz gzip -f tmp zstd -d -f -v tmp.gz - rm tmp* + rm -f tmp* else println "gzip binary not detected" fi @@ -1027,7 +1150,7 @@ if [ $GZIPMODE -eq 1 ]; then zstd -f tmp cat tmp.gz tmp.zst tmp.gz tmp.zst | zstd -d -f -o tmp truncateLastByte tmp.gz | zstd -t > $INTOVOID && die "incomplete frame not detected !" - rm tmp* + rm -f tmp* else println "gzip mode not supported" fi @@ -1058,7 +1181,7 @@ if [ $LZMAMODE -eq 1 ]; then lzma -Q -f -k --lzma1 tmp zstd -d -f -v tmp.xz zstd -d -f -v tmp.lzma - rm tmp* + rm -f tmp* println "Creating symlinks" ln -s "$ZSTD_BIN" ./xz ln -s "$ZSTD_BIN" ./unxz @@ -1075,8 +1198,8 @@ if [ $LZMAMODE -eq 1 ]; then ./xz -d tmp.xz lzma -Q tmp ./lzma -d tmp.lzma - rm xz unxz lzma unlzma - rm tmp* + rm -f xz unxz lzma unlzma + rm -f tmp* else println "xz binary not detected" fi @@ -1095,7 +1218,7 @@ if [ $LZMAMODE -eq 1 ]; then cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | zstd -d -f -o tmp truncateLastByte tmp.xz | zstd -t > $INTOVOID && die "incomplete frame not detected !" truncateLastByte tmp.lzma | zstd -t > $INTOVOID && die "incomplete frame not detected !" - rm tmp* + rm -f tmp* else println "xz mode not supported" fi @@ -1114,7 +1237,7 @@ if [ $LZ4MODE -eq 1 ]; then lz4 -t -v tmp.lz4 lz4 -f -m tmp # ensure result is sent into tmp.lz4, not stdout zstd -d -f -v tmp.lz4 - rm tmp* + rm -f tmp* else println "lz4 binary not detected" fi @@ -1130,7 +1253,7 @@ if [ $LZ4MODE -eq 1 ]; then zstd -f tmp cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | zstd -d -f -o tmp truncateLastByte tmp.lz4 | zstd -t > $INTOVOID && die "incomplete frame not detected !" - rm tmp* + rm -f tmp* else println "\nlz4 mode not supported" fi @@ -1164,7 +1287,7 @@ rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 tmp1.zstd datagen > tmp tar cf tmp.tar tmp zstd tmp.tar -o tmp.tzst -rm tmp.tar +rm -f tmp.tar zstd -d tmp.tzst [ -e tmp.tar ] || die ".tzst failed to decompress to .tar!" rm -f tmp.tar tmp.tzst @@ -1243,7 +1366,7 @@ then ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting= ZSTD_NBTHREADS=2 zstd -f mt_tmp # correct usage ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single thread - rm mt_tmp* + rm -f mt_tmp* println "\n===> ovLog tests " datagen -g2MB > tmp @@ -1267,7 +1390,7 @@ else println "\n===> no multithreading, skipping zstdmt tests " fi -rm tmp* +rm -f tmp* println "\n===> zstd --list/-l single frame tests " datagen > tmp1 @@ -1300,9 +1423,9 @@ zstd -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE zstd --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file" -rm $TEST_DATA_FILE -rm $FULL_COMPRESSED_FILE -rm $TRUNCATED_COMPRESSED_FILE +rm -f $TEST_DATA_FILE +rm -f $FULL_COMPRESSED_FILE +rm -f $TRUNCATED_COMPRESSED_FILE println "\n===> zstd --list/-l errors when presented with stdin / no files" zstd -l && die "-l must fail on empty list of files" @@ -1346,7 +1469,7 @@ zstd -D tmp1 tmp2 -c | zstd --trace tmp.trace -t -D tmp1 zstd -b1e10i0 --trace tmp.trace tmp1 zstd -b1e10i0 --trace tmp.trace tmp1 tmp2 tmp3 -rm tmp* +rm -f tmp* println "\n===> zstd long distance matching tests " diff --git a/tests/regression/results.csv b/tests/regression/results.csv index 68150a615..b94d5501e 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -97,11 +97,11 @@ github, uncompressed literals, compress github, uncompressed literals optimal, compress cctx, 134064 github, huffman literals, compress cctx, 175568 github, multithreaded with advanced params, compress cctx, 141102 -silesia, level -5, zstdcli, 6882553 -silesia, level -3, zstdcli, 6568424 -silesia, level -1, zstdcli, 6183451 +silesia, level -5, zstdcli, 6737655 +silesia, level -3, zstdcli, 6444725 +silesia, level -1, zstdcli, 6178508 silesia, level 0, zstdcli, 4849600 -silesia, level 1, zstdcli, 5314210 +silesia, level 1, zstdcli, 5313252 silesia, level 3, zstdcli, 4849600 silesia, level 4, zstdcli, 4787018 silesia, level 5, zstdcli, 4707842 @@ -111,16 +111,16 @@ silesia, level 9, zstdcli, silesia, level 13, zstdcli, 4482183 silesia, level 16, zstdcli, 4360299 silesia, level 19, zstdcli, 4283285 -silesia, long distance mode, zstdcli, 4840792 +silesia, long distance mode, zstdcli, 4840806 silesia, multithreaded, zstdcli, 4849600 -silesia, multithreaded long distance mode, zstdcli, 4840792 -silesia, small window log, zstdcli, 7111012 +silesia, multithreaded long distance mode, zstdcli, 4840806 +silesia, small window log, zstdcli, 7095967 silesia, small hash log, zstdcli, 6526189 silesia, small chain log, zstdcli, 4912245 -silesia, explicit params, zstdcli, 4795887 +silesia, explicit params, zstdcli, 4795856 silesia, uncompressed literals, zstdcli, 5128030 silesia, uncompressed literals optimal, zstdcli, 4317944 -silesia, huffman literals, zstdcli, 5331216 +silesia, huffman literals, zstdcli, 5326316 silesia, multithreaded with advanced params, zstdcli, 5128030 silesia.tar, level -5, zstdcli, 6738934 silesia.tar, level -3, zstdcli, 6448419 @@ -137,9 +137,9 @@ silesia.tar, level 13, zstdcli, silesia.tar, level 16, zstdcli, 4356831 silesia.tar, level 19, zstdcli, 4264491 silesia.tar, no source size, zstdcli, 4861508 -silesia.tar, long distance mode, zstdcli, 4853153 +silesia.tar, long distance mode, zstdcli, 4853226 silesia.tar, multithreaded, zstdcli, 4861512 -silesia.tar, multithreaded long distance mode, zstdcli, 4853153 +silesia.tar, multithreaded long distance mode, zstdcli, 4853226 silesia.tar, small window log, zstdcli, 7101576 silesia.tar, small hash log, zstdcli, 6529290 silesia.tar, small chain log, zstdcli, 4917022 @@ -187,16 +187,16 @@ github, uncompressed literals, zstdcli, github, uncompressed literals optimal, zstdcli, 159227 github, huffman literals, zstdcli, 144465 github, multithreaded with advanced params, zstdcli, 167915 -github.tar, level -5, zstdcli, 46751 -github.tar, level -5 with dict, zstdcli, 44444 -github.tar, level -3, zstdcli, 43541 -github.tar, level -3 with dict, zstdcli, 41116 -github.tar, level -1, zstdcli, 42469 -github.tar, level -1 with dict, zstdcli, 41200 +github.tar, level -5, zstdcli, 46860 +github.tar, level -5 with dict, zstdcli, 44575 +github.tar, level -3, zstdcli, 43758 +github.tar, level -3 with dict, zstdcli, 41451 +github.tar, level -1, zstdcli, 42494 +github.tar, level -1 with dict, zstdcli, 41135 github.tar, level 0, zstdcli, 38445 github.tar, level 0 with dict, zstdcli, 37999 -github.tar, level 1, zstdcli, 39346 -github.tar, level 1 with dict, zstdcli, 38297 +github.tar, level 1, zstdcli, 39269 +github.tar, level 1 with dict, zstdcli, 38284 github.tar, level 3, zstdcli, 38445 github.tar, level 3 with dict, zstdcli, 37999 github.tar, level 4, zstdcli, 38471 @@ -217,16 +217,16 @@ github.tar, level 19, zstdcli, github.tar, level 19 with dict, zstdcli, 32899 github.tar, no source size, zstdcli, 38442 github.tar, no source size with dict, zstdcli, 38004 -github.tar, long distance mode, zstdcli, 39726 +github.tar, long distance mode, zstdcli, 39730 github.tar, multithreaded, zstdcli, 38445 -github.tar, multithreaded long distance mode, zstdcli, 39726 -github.tar, small window log, zstdcli, 199432 +github.tar, multithreaded long distance mode, zstdcli, 39730 +github.tar, small window log, zstdcli, 198544 github.tar, small hash log, zstdcli, 129874 github.tar, small chain log, zstdcli, 41673 github.tar, explicit params, zstdcli, 41227 github.tar, uncompressed literals, zstdcli, 41126 github.tar, uncompressed literals optimal, zstdcli, 35392 -github.tar, huffman literals, zstdcli, 38804 +github.tar, huffman literals, zstdcli, 38781 github.tar, multithreaded with advanced params, zstdcli, 41126 silesia, level -5, advanced one pass, 6737607 silesia, level -3, advanced one pass, 6444677 @@ -249,9 +249,9 @@ silesia, level 13, advanced silesia, level 16, advanced one pass, 4360251 silesia, level 19, advanced one pass, 4283237 silesia, no source size, advanced one pass, 4849552 -silesia, long distance mode, advanced one pass, 4840744 +silesia, long distance mode, advanced one pass, 4840738 silesia, multithreaded, advanced one pass, 4849552 -silesia, multithreaded long distance mode, advanced one pass, 4840744 +silesia, multithreaded long distance mode, advanced one pass, 4840758 silesia, small window log, advanced one pass, 7095919 silesia, small hash log, advanced one pass, 6526141 silesia, small chain log, advanced one pass, 4912197 @@ -281,9 +281,9 @@ silesia.tar, level 13, advanced silesia.tar, level 16, advanced one pass, 4356827 silesia.tar, level 19, advanced one pass, 4264487 silesia.tar, no source size, advanced one pass, 4861425 -silesia.tar, long distance mode, advanced one pass, 4847735 +silesia.tar, long distance mode, advanced one pass, 4847754 silesia.tar, multithreaded, advanced one pass, 4861508 -silesia.tar, multithreaded long distance mode, advanced one pass, 4853149 +silesia.tar, multithreaded long distance mode, advanced one pass, 4853222 silesia.tar, small window log, advanced one pass, 7101530 silesia.tar, small hash log, advanced one pass, 6529232 silesia.tar, small chain log, advanced one pass, 4917041 @@ -511,9 +511,9 @@ github.tar, level 19 with dict copy, advanced github.tar, level 19 with dict load, advanced one pass, 32676 github.tar, no source size, advanced one pass, 38441 github.tar, no source size with dict, advanced one pass, 37995 -github.tar, long distance mode, advanced one pass, 39722 +github.tar, long distance mode, advanced one pass, 39757 github.tar, multithreaded, advanced one pass, 38441 -github.tar, multithreaded long distance mode, advanced one pass, 39722 +github.tar, multithreaded long distance mode, advanced one pass, 39726 github.tar, small window log, advanced one pass, 198540 github.tar, small hash log, advanced one pass, 129870 github.tar, small chain log, advanced one pass, 41669 @@ -543,9 +543,9 @@ silesia, level 13, advanced silesia, level 16, advanced one pass small out, 4360251 silesia, level 19, advanced one pass small out, 4283237 silesia, no source size, advanced one pass small out, 4849552 -silesia, long distance mode, advanced one pass small out, 4840744 +silesia, long distance mode, advanced one pass small out, 4840738 silesia, multithreaded, advanced one pass small out, 4849552 -silesia, multithreaded long distance mode, advanced one pass small out, 4840744 +silesia, multithreaded long distance mode, advanced one pass small out, 4840758 silesia, small window log, advanced one pass small out, 7095919 silesia, small hash log, advanced one pass small out, 6526141 silesia, small chain log, advanced one pass small out, 4912197 @@ -575,9 +575,9 @@ silesia.tar, level 13, advanced silesia.tar, level 16, advanced one pass small out, 4356827 silesia.tar, level 19, advanced one pass small out, 4264487 silesia.tar, no source size, advanced one pass small out, 4861425 -silesia.tar, long distance mode, advanced one pass small out, 4847735 +silesia.tar, long distance mode, advanced one pass small out, 4847754 silesia.tar, multithreaded, advanced one pass small out, 4861508 -silesia.tar, multithreaded long distance mode, advanced one pass small out, 4853149 +silesia.tar, multithreaded long distance mode, advanced one pass small out, 4853222 silesia.tar, small window log, advanced one pass small out, 7101530 silesia.tar, small hash log, advanced one pass small out, 6529232 silesia.tar, small chain log, advanced one pass small out, 4917041 @@ -805,9 +805,9 @@ github.tar, level 19 with dict copy, advanced github.tar, level 19 with dict load, advanced one pass small out, 32676 github.tar, no source size, advanced one pass small out, 38441 github.tar, no source size with dict, advanced one pass small out, 37995 -github.tar, long distance mode, advanced one pass small out, 39722 +github.tar, long distance mode, advanced one pass small out, 39757 github.tar, multithreaded, advanced one pass small out, 38441 -github.tar, multithreaded long distance mode, advanced one pass small out, 39722 +github.tar, multithreaded long distance mode, advanced one pass small out, 39726 github.tar, small window log, advanced one pass small out, 198540 github.tar, small hash log, advanced one pass small out, 129870 github.tar, small chain log, advanced one pass small out, 41669 @@ -837,9 +837,9 @@ silesia, level 13, advanced silesia, level 16, advanced streaming, 4360251 silesia, level 19, advanced streaming, 4283237 silesia, no source size, advanced streaming, 4849516 -silesia, long distance mode, advanced streaming, 4840744 +silesia, long distance mode, advanced streaming, 4840738 silesia, multithreaded, advanced streaming, 4849552 -silesia, multithreaded long distance mode, advanced streaming, 4840744 +silesia, multithreaded long distance mode, advanced streaming, 4840758 silesia, small window log, advanced streaming, 7112062 silesia, small hash log, advanced streaming, 6526141 silesia, small chain log, advanced streaming, 4912197 @@ -869,9 +869,9 @@ silesia.tar, level 13, advanced silesia.tar, level 16, advanced streaming, 4356834 silesia.tar, level 19, advanced streaming, 4264392 silesia.tar, no source size, advanced streaming, 4861423 -silesia.tar, long distance mode, advanced streaming, 4847735 +silesia.tar, long distance mode, advanced streaming, 4847754 silesia.tar, multithreaded, advanced streaming, 4861508 -silesia.tar, multithreaded long distance mode, advanced streaming, 4853149 +silesia.tar, multithreaded long distance mode, advanced streaming, 4853222 silesia.tar, small window log, advanced streaming, 7118769 silesia.tar, small hash log, advanced streaming, 6529235 silesia.tar, small chain log, advanced streaming, 4917021 @@ -1099,9 +1099,9 @@ github.tar, level 19 with dict copy, advanced github.tar, level 19 with dict load, advanced streaming, 32676 github.tar, no source size, advanced streaming, 38438 github.tar, no source size with dict, advanced streaming, 38000 -github.tar, long distance mode, advanced streaming, 39722 +github.tar, long distance mode, advanced streaming, 39757 github.tar, multithreaded, advanced streaming, 38441 -github.tar, multithreaded long distance mode, advanced streaming, 39722 +github.tar, multithreaded long distance mode, advanced streaming, 39726 github.tar, small window log, advanced streaming, 199558 github.tar, small hash log, advanced streaming, 129870 github.tar, small chain log, advanced streaming, 41669 diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index f2b16db2c..56f77232f 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -321,7 +321,9 @@ static int basicUnitTests(U32 seed, double compressibility) /* Basic compression test using dict */ DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); - CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1 /* cLevel */) ); + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) ); + CHECK_Z( ZSTD_CCtx_loadDictionary(zc, CNBuffer, dictSize) ); outBuff.dst = (char*)(compressedBuffer)+cSize; assert(compressedBufferSize > cSize); outBuff.size = compressedBufferSize - cSize; @@ -368,7 +370,7 @@ static int basicUnitTests(U32 seed, double compressibility) } /* Attempt bad compression parameters */ - DISPLAYLEVEL(3, "test%3i : use bad compression parameters : ", testNb++); + DISPLAYLEVEL(3, "test%3i : use bad compression parameters with ZSTD_initCStream_advanced : ", testNb++); { size_t r; ZSTD_parameters params = ZSTD_getParams(1, 0, 0); params.cParams.minMatch = 2; @@ -539,7 +541,10 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK\n"); /* _srcSize compression test */ DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); - CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) ); + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize) ); outBuff.dst = (char*)(compressedBuffer); outBuff.size = compressedBufferSize; outBuff.pos = 0; @@ -559,7 +564,10 @@ static int basicUnitTests(U32 seed, double compressibility) /* wrong _srcSize compression test */ DISPLAYLEVEL(3, "test%3i : too large srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1); - ZSTD_initCStream_srcSize(zc, 1, CNBufferSize+1); + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize+1) ); outBuff.dst = (char*)(compressedBuffer); outBuff.size = compressedBufferSize; outBuff.pos = 0; @@ -574,7 +582,10 @@ static int basicUnitTests(U32 seed, double compressibility) /* wrong _srcSize compression test */ DISPLAYLEVEL(3, "test%3i : too small srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1); - ZSTD_initCStream_srcSize(zc, 1, CNBufferSize-1); + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize-1) ); outBuff.dst = (char*)(compressedBuffer); outBuff.size = compressedBufferSize; outBuff.pos = 0; @@ -587,9 +598,9 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "test%3i : wrong srcSize !contentSizeFlag : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1); - { ZSTD_parameters params = ZSTD_getParams(1, CNBufferSize, 0); - params.fParams.contentSizeFlag = 0; - CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, CNBufferSize - MIN(CNBufferSize, 200 KB))); + { CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, 0) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize - MIN(CNBufferSize, 200 KB)) ); outBuff.dst = (char*)compressedBuffer; outBuff.size = compressedBufferSize; outBuff.pos = 0; @@ -609,7 +620,9 @@ static int basicUnitTests(U32 seed, double compressibility) /* use 1 */ { size_t const inSize = 513; DISPLAYLEVEL(5, "use1 "); - ZSTD_initCStream_advanced(zc, NULL, 0, ZSTD_getParams(19, inSize, 0), inSize); /* needs btopt + search3 to trigger hashLog3 */ + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) ); inBuff.src = CNBuffer; inBuff.size = inSize; inBuff.pos = 0; @@ -626,7 +639,9 @@ static int basicUnitTests(U32 seed, double compressibility) /* use 2 */ { size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */ DISPLAYLEVEL(5, "use2 "); - ZSTD_initCStream_advanced(zc, NULL, 0, ZSTD_getParams(19, inSize, 0), inSize); /* needs btopt + search3 to trigger hashLog3 */ + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) ); inBuff.src = CNBuffer; inBuff.size = inSize; inBuff.pos = 0; @@ -1274,7 +1289,7 @@ static int basicUnitTests(U32 seed, double compressibility) if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error; DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++); + DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly with ZSTD_initCStream_advanced : ", testNb++); { ZSTD_parameters params = ZSTD_getParams(5, 0, 0); params.fParams.contentSizeFlag = 1; CHECK_Z( ZSTD_initCStream_advanced(zc, NULL, 0, params, 0) ); @@ -1290,7 +1305,8 @@ static int basicUnitTests(U32 seed, double compressibility) cSize = outBuff.pos; if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error; - ZSTD_resetCStream(zc, 0); /* resetCStream should treat 0 as unknown */ + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) ); outBuff.dst = compressedBuffer; outBuff.size = compressedBufferSize; outBuff.pos = 0; @@ -1434,7 +1450,8 @@ static int basicUnitTests(U32 seed, double compressibility) CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN)); CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level)); CHECK(level != 11, "Compression level does not match"); - ZSTD_resetCStream(zc, ZSTD_CONTENTSIZE_UNKNOWN); + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) ); CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level)); CHECK(level != 11, "Compression level does not match"); } @@ -1444,7 +1461,8 @@ static int basicUnitTests(U32 seed, double compressibility) { ZSTD_parameters const params = ZSTD_getParams(9, 0, 0); CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN)); CHECK(badParameters(zc, params), "Compression parameters do not match"); - ZSTD_resetCStream(zc, ZSTD_CONTENTSIZE_UNKNOWN); + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) ); CHECK(badParameters(zc, params), "Compression parameters do not match"); } DISPLAYLEVEL(3, "OK \n"); @@ -1836,8 +1854,9 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double co && oldTestLog /* at least one test happened */ && resetAllowed) { maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2); maxTestSize = MIN(maxTestSize, srcBufferSize-16); - { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; - CHECK_Z( ZSTD_resetCStream(zc, pledgedSrcSize) ); + { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); } } else { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; @@ -1855,11 +1874,13 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double co dict = srcBuffer + dictStart; } { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; - ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize); - params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; - params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; - params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1; - CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) ); + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, cLevel) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, FUZ_rand(&lseed) & 1) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); + CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) ); } } /* multi-segments compression test */ @@ -2215,6 +2236,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, } if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_deterministicRefPrefix, FUZ_rand(&lseed) & 1, opaqueAPI) ); /* Apply parameters */ if (opaqueAPI) { diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index 127f6114b..5993e51ba 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -264,9 +264,22 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, ZSTD_outBuffer outBuffer; ZSTD_CStream* zbc = ZSTD_createCStream(); size_t rSize; + ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); + + if (!cctxParams) EXM_THROW(1, "ZSTD_createCCtxParams() allocation failure"); if (zbc == NULL) EXM_THROW(1, "ZSTD_createCStream() allocation failure"); - rSize = ZSTD_initCStream_advanced(zbc, dictBuffer, dictBufferSize, zparams, avgSize); - if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initCStream_advanced() failed : %s", ZSTD_getErrorName(rSize)); + + { int initErr = 0; + initErr |= ZSTD_isError(ZSTD_CCtx_reset(zbc, ZSTD_reset_session_only)); + initErr |= ZSTD_isError(ZSTD_CCtxParams_init_advanced(cctxParams, zparams)); + initErr |= ZSTD_isError(ZSTD_CCtx_setParametersUsingCCtxParams(zbc, cctxParams)); + initErr |= ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(zbc, avgSize)); + initErr |= ZSTD_isError(ZSTD_CCtx_loadDictionary(zbc, dictBuffer, dictBufferSize)); + + ZSTD_freeCCtxParams(cctxParams); + if (initErr) EXM_THROW(1, "CCtx init failed!"); + } + do { U32 blockNb; for (blockNb=0; blockNbzbc == NULL) return Z_STREAM_ERROR; if (!pledgedSrcSize) pledgedSrcSize = zwc->pledgedSrcSize; - { ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize); - size_t initErr; + { unsigned initErr = 0; + ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize); + ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); + if (!cctxParams) return Z_STREAM_ERROR; LOG_WRAPPERC("pledgedSrcSize=%d windowLog=%d chainLog=%d hashLog=%d searchLog=%d minMatch=%d strategy=%d\n", (int)pledgedSrcSize, params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.minMatch, params.cParams.strategy); - initErr = ZSTD_initCStream_advanced(zwc->zbc, dict, dictSize, params, pledgedSrcSize); - if (ZSTD_isError(initErr)) return Z_STREAM_ERROR; + + initErr |= ZSTD_isError(ZSTD_CCtx_reset(zwc->zbc, ZSTD_reset_session_only)); + initErr |= ZSTD_isError(ZSTD_CCtxParams_init_advanced(cctxParams, params)); + initErr |= ZSTD_isError(ZSTD_CCtx_setParametersUsingCCtxParams(zwc->zbc, cctxParams)); + initErr |= ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(zwc->zbc, pledgedSrcSize)); + initErr |= ZSTD_isError(ZSTD_CCtx_loadDictionary(zwc->zbc, dict, dictSize)); + + ZSTD_freeCCtxParams(cctxParams); + if (initErr) return Z_STREAM_ERROR; } return Z_OK;